From 4ddd2d01b94d7fb2841df4d36c0b72bb0214bbbc Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Sat, 1 Aug 2026 14:13:58 -0400 Subject: [PATCH] rollback patches --- .github/actions/build-emitters/action.yml | 39 + .github/actions/fetch-toolchain/action.yml | 51 + .github/actions/stage-setup-sdk/action.yml | 71 + CONTRIBUTING.md | 20 +- README.md | 206 +- cmake/toolchain-mingw-w64.cmake | 46 + docs/GAME_PROJECT_SETUP.md | 228 +++ docs/LOCAL_CODEGEN_SDK.md | 149 ++ docs/ROLLBACK_MOTK_HOOKUP.md | 1019 +++++++++- docs/ci/README.md | 64 + docs/ci/setup-release.yml.example | 23 + docs/ci/templates/game.gitignore | 63 + docs/ci/templates/setup-release.yml | 252 +++ host/psxrecomp_codegen_host.c | 1969 ++++++++++++++++++++ host/psxrecomp_codegen_host.h | 55 + lib/recomp-net | 2 +- psxrecomp_cli.py | 1342 +++++++++++++ recompiler/CMakeLists.txt | 44 +- runtime/include/boot_state.h | 10 +- runtime/include/cdrom.h | 8 + runtime/include/gpu.h | 7 + runtime/include/interrupts.h | 7 +- runtime/include/netplay_rb_post.h | 27 + runtime/include/psx_netplay_rb.h | 35 +- runtime/runtime.cmake | 287 ++- runtime/src/boot_state.c | 43 +- runtime/src/cdrom.c | 26 +- runtime/src/crc32.c | 68 +- runtime/src/gpu.c | 90 +- runtime/src/interrupts.c | 62 +- runtime/src/main.cpp | 102 +- runtime/src/mdec.c | 10 +- runtime/src/netplay_state_digest.c | 10 + runtime/src/psx_netplay.c | 511 ++++- runtime/src/psx_netplay_rb.c | 1129 +++++++++-- runtime/src/psx_selfcheck.c | 4 +- runtime/tests/test_rb_post_tip_filter.c | 45 + tools/bundle_mingw_dlls.sh | 240 +++ tools/ci/build_emitters.sh | 106 ++ tools/ci/clear_generated.sh | 25 + tools/ci/normalize_version.sh | 50 + tools/ci/record_pins.sh | 38 + tools/fetch_toolchain.sh | 133 ++ tools/package_setup_host.sh | 294 +++ tools/prepare_disc.py | 587 ++++++ tools/stage_setup_sdk.sh | 266 +++ tools/toolchain_pack.py | 404 ++++ 47 files changed, 9941 insertions(+), 326 deletions(-) create mode 100644 .github/actions/build-emitters/action.yml create mode 100644 .github/actions/fetch-toolchain/action.yml create mode 100644 .github/actions/stage-setup-sdk/action.yml create mode 100644 cmake/toolchain-mingw-w64.cmake create mode 100644 docs/GAME_PROJECT_SETUP.md create mode 100644 docs/LOCAL_CODEGEN_SDK.md create mode 100644 docs/ci/README.md create mode 100644 docs/ci/setup-release.yml.example create mode 100644 docs/ci/templates/game.gitignore create mode 100644 docs/ci/templates/setup-release.yml create mode 100644 host/psxrecomp_codegen_host.c create mode 100644 host/psxrecomp_codegen_host.h create mode 100755 psxrecomp_cli.py create mode 100644 runtime/include/netplay_rb_post.h create mode 100644 runtime/tests/test_rb_post_tip_filter.c create mode 100755 tools/bundle_mingw_dlls.sh create mode 100755 tools/ci/build_emitters.sh create mode 100755 tools/ci/clear_generated.sh create mode 100755 tools/ci/normalize_version.sh create mode 100755 tools/ci/record_pins.sh create mode 100755 tools/fetch_toolchain.sh create mode 100755 tools/package_setup_host.sh create mode 100644 tools/prepare_disc.py create mode 100755 tools/stage_setup_sdk.sh create mode 100644 tools/toolchain_pack.py diff --git a/.github/actions/build-emitters/action.yml b/.github/actions/build-emitters/action.yml new file mode 100644 index 00000000..65750313 --- /dev/null +++ b/.github/actions/build-emitters/action.yml @@ -0,0 +1,39 @@ +name: Build psxrecomp emitters +description: Configure and build psxrecomp-game + psxrecomp-bios via tools/ci/build_emitters.sh + +inputs: + framework: + description: Path to the psxrecomp checkout + required: false + default: psxrecomp + build-dir: + description: Out-of-tree build directory for the emitters + required: false + default: build-recompiler + toolchain-dir: + description: > + Portable cmake-clang-v1 root (llvm-mingw on Windows). Empty uses + PSXRECOMP_TOOLCHAIN_DIR / TOOLCHAIN_DIR from the environment. + required: false + default: '' + +runs: + using: composite + steps: + - shell: bash + run: | + set -euo pipefail + SCRIPT="${{ inputs.framework }}/tools/ci/build_emitters.sh" + if [[ ! -f "${SCRIPT}" ]]; then + echo "::error::missing ${SCRIPT}" + exit 1 + fi + chmod +x "${SCRIPT}" + args=( + --framework "${{ inputs.framework }}" + --build-dir "${{ inputs.build-dir }}" + ) + if [[ -n "${{ inputs.toolchain-dir }}" ]]; then + args+=(--toolchain-dir "${{ inputs.toolchain-dir }}") + fi + bash "${SCRIPT}" "${args[@]}" diff --git a/.github/actions/fetch-toolchain/action.yml b/.github/actions/fetch-toolchain/action.yml new file mode 100644 index 00000000..fc24ab2e --- /dev/null +++ b/.github/actions/fetch-toolchain/action.yml @@ -0,0 +1,51 @@ +name: Fetch psxrecomp portable toolchain +description: > + Download cmake-clang-v1 from retcomm-toolchains and export TOOLCHAIN_DIR / + PSXRECOMP_TOOLCHAIN_DIR / BPE_TOOLCHAIN_DIR for later packaging steps. + +inputs: + artifact: + description: Platform pack id (linux-x64, windows-x64, macos-arm64, macos-x64) + required: true + framework: + description: Path to the psxrecomp checkout (contains tools/fetch_toolchain.sh) + required: false + default: psxrecomp + dl-dir: + description: Download cache directory + required: false + default: .cache/toolchain-dl + out-dir: + description: Unpack directory + required: false + default: .cache/toolchain-pack + repo: + description: GitHub owner/name for toolchain releases + required: false + default: TechnicallyComputers/retcomm-toolchains + +outputs: + toolchain-dir: + description: Absolute path to the unpacked toolchain root (bin/ present) + value: ${{ steps.fetch.outputs.toolchain-dir }} + +runs: + using: composite + steps: + - id: fetch + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + SCRIPT="${{ inputs.framework }}/tools/fetch_toolchain.sh" + if [[ ! -f "${SCRIPT}" ]]; then + echo "::error::missing ${SCRIPT}" + exit 1 + fi + chmod +x "${SCRIPT}" + bash "${SCRIPT}" \ + --artifact "${{ inputs.artifact }}" \ + --dl-dir "${{ inputs.dl-dir }}" \ + --out-dir "${{ inputs.out-dir }}" \ + --repo "${{ inputs.repo }}" diff --git a/.github/actions/stage-setup-sdk/action.yml b/.github/actions/stage-setup-sdk/action.yml new file mode 100644 index 00000000..03ed8871 --- /dev/null +++ b/.github/actions/stage-setup-sdk/action.yml @@ -0,0 +1,71 @@ +name: Stage psxrecomp setup SDK +description: > + Copy emitters, OpenBIOS profiles, retcomm-sdk.json, portable toolchain, and + Windows MinGW runtime DLLs into an existing setup-host stage directory. + +inputs: + stage: + description: Absolute or workspace-relative stage directory + required: true + framework: + description: Path to the psxrecomp checkout + required: false + default: psxrecomp + sdk-overlay: + description: Legacy optional overlay (prefer CLI/tools inside the submodule) + required: false + default: '' + recompiler-build: + description: Build dir containing psxrecomp-game / psxrecomp-bios + required: false + default: build-recompiler + toolchain-dir: + description: Unpacked toolchain root (defaults to env PSXRECOMP_TOOLCHAIN_DIR / TOOLCHAIN_DIR / BPE_TOOLCHAIN_DIR) + required: false + default: '' + host-exe: + description: Optional host .exe path for Windows DLL bundling + required: false + default: '' + runtime-bin: + description: MinGW bin dir for runtime DLLs (Windows) + required: false + default: '' + allow-no-toolchain: + description: If true, warn instead of failing when toolchain is unset + required: false + default: 'false' + +runs: + using: composite + steps: + - shell: bash + run: | + set -euo pipefail + SCRIPT="${{ inputs.framework }}/tools/stage_setup_sdk.sh" + if [[ ! -f "${SCRIPT}" ]]; then + echo "::error::missing ${SCRIPT}" + exit 1 + fi + chmod +x "${SCRIPT}" + args=( + --stage "${{ inputs.stage }}" + --framework "${{ inputs.framework }}" + --recompiler-build "${{ inputs.recompiler-build }}" + ) + if [[ -n "${{ inputs.sdk-overlay }}" ]]; then + args+=(--sdk-overlay "${{ inputs.sdk-overlay }}") + fi + if [[ -n "${{ inputs.toolchain-dir }}" ]]; then + args+=(--toolchain-dir "${{ inputs.toolchain-dir }}") + fi + if [[ "${{ inputs.allow-no-toolchain }}" == "true" ]]; then + args+=(--allow-no-toolchain) + fi + if [[ -n "${{ inputs.host-exe }}" ]]; then + args+=(--host-exe "${{ inputs.host-exe }}") + fi + if [[ -n "${{ inputs.runtime-bin }}" ]]; then + args+=(--runtime-bin "${{ inputs.runtime-bin }}") + fi + bash "${SCRIPT}" "${args[@]}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ac8ab37a..26bf7cea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,20 +16,22 @@ New to the codebase? Read [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) and (`recompiler/`), the runtime engine (`runtime/`), shared tools, and docs. Framework changes go here. - **Per-game repos** (`mstan/TombaRecomp`, `mstan/MegaManX6Recomp`, - `mstan/ApeEscapeRecomp`, …) — one per title. Each contains only that game's - config, seeds, and build glue and links this framework in as a submodule at - `psxrecomp/`. Game-specific work goes in the game repo; nothing game-specific - belongs in the framework. + `mstan/ApeEscapeRecomp`, …) — one per title. Each keeps **game code at the + repo root** and pins this framework (and usually `recomp-ui`) as **root-level + submodules** (`psxrecomp/`, `recomp-ui/`). Game-specific work goes in the game + repo; nothing game-specific belongs in the framework. Each game repo pins an **exact framework commit** — the git submodule pointer at `psxrecomp/` (some older game repos also carry a human-readable `psxrecomp-v4.pin` record; the submodule pointer is the source of truth). The framework evolves on its own cadence; a game only moves to a newer framework when someone deliberately -bumps that submodule pointer. See -[Linking the framework](docs/BUILDING.md#linking-the-framework) for the submodule -+ local-junction dev setup, and -[Framework changes and the pin](#framework-changes-and-the-pin) below for how a -framework change actually reaches a game. +bumps that submodule pointer. + +**Starting a new title or shipping a setup-host zip?** Follow +[`docs/GAME_PROJECT_SETUP.md`](docs/GAME_PROJECT_SETUP.md) (layout, CI template +under `docs/ci/templates/setup-release.yml`, release checklist). Also see +[Linking the framework](docs/BUILDING.md#linking-the-framework) and +[Framework changes and the pin](#framework-changes-and-the-pin) below. ## The core rules diff --git a/README.md b/README.md index 2178391d..fd141a7e 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,10 @@ standalone BIOS runtime supporting OpenBIOS and a compatible retail BIOS** — see [Release Package](#release-package) below. Bringing up a title of your own? Start with -[`CONTRIBUTING.md`](CONTRIBUTING.md) and open an issue — community projects are -listed here alongside the rest. +[`docs/GAME_PROJECT_SETUP.md`](docs/GAME_PROJECT_SETUP.md) (submodules, +setup-host CI template, release checklist), then +[`CONTRIBUTING.md`](CONTRIBUTING.md). Community projects are listed here +alongside the rest. ## What It Is @@ -95,15 +97,22 @@ Three things sit on that foundation: execution. PSXRecomp is a **framework**. Game-specific projects live in their own -repositories and link this one in as a **git submodule** to build a game binary. +repositories with **`psxrecomp/` and `recomp-ui/` as root-level submodules** +and game code (`game.toml`, seeds, CMake) at the repo root. See +[`docs/GAME_PROJECT_SETUP.md`](docs/GAME_PROJECT_SETUP.md). **New here?** The fastest way in: -[`docs/EXECUTION_MODEL.md`](docs/EXECUTION_MODEL.md) (how a game actually -runs — static / native-overlay / interpreter), then -[`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md), -[`docs/BUILDING.md`](docs/BUILDING.md), -[`docs/MOD_PACKAGES.md`](docs/MOD_PACKAGES.md) (versioned runtime mods), -[`CONTRIBUTING.md`](CONTRIBUTING.md). + +| Path | Doc | +|------|-----| +| How a game runs | [`docs/EXECUTION_MODEL.md`](docs/EXECUTION_MODEL.md) | +| Architecture | [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | +| Build the framework | [`docs/BUILDING.md`](docs/BUILDING.md) | +| **Ship a game repo** (submodules + CI + release checklist) | [`docs/GAME_PROJECT_SETUP.md`](docs/GAME_PROJECT_SETUP.md) | +| Setup-host CI template | [`docs/ci/templates/setup-release.yml`](docs/ci/templates/setup-release.yml) | +| Local Generate & rebuild CLI | [`docs/LOCAL_CODEGEN_SDK.md`](docs/LOCAL_CODEGEN_SDK.md) | +| Mods | [`docs/MOD_PACKAGES.md`](docs/MOD_PACKAGES.md) | +| Contributing | [`CONTRIBUTING.md`](CONTRIBUTING.md) | ## Which PlayStation BIOS does it use? @@ -180,10 +189,170 @@ turned off as easily as it was turned on. The mods panel in the launcher

-Tomba! ships three today — widescreen, skip-FMV, and a warp debug menu that -re-enables the developers' own hidden menu. Full format in +### Add a mod + +Start with a source directory containing a manifest and, when needed, payload +files: + +```text +my-mod/ +├── manifest.toml +├── README.txt +└── assets/ + └── replacement.bin +``` + +A minimal guarded executable patch looks like this: + +```toml +format_version = 1 +id = "example.quick-start" +version = "1.0.0" +name = "Quick-start example" +author = "Your name" +description = "One independently toggleable gameplay change." +resolver = "declarative" +save_compatibility = "shared" + +[[target]] +game_id = "SLUS-00000" +# Replace this with the SHA-256 of the exact supported stock disc image. +disc_sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[[feature]] +id = "quick-start" +name = "Quick Start" +description = "Skips the game's startup delay." +group = "Gameplay" +default_enabled = false + +[[patch]] +feature = "quick-start" +target = "main_exe" +address = 0x80041234 +expected = "2a 00 02 24" +replace = "00 00 02 24" +``` + +`main_exe` addresses are PSX guest virtual addresses. The complete `expected` +bytes are checked after the BIOS loads the executable and before any write is +made, so a package fails closed on the wrong revision. For disc changes use +`disc_raw` or `disc_user`; for a larger asset use a file-backed `[[overlay]]` +with payload and expected-range SHA-256 hashes. + +From a PSXRecomp checkout, pack the directory into a deterministic archive: + +```sh +python tools/psxmod_pack.py my-mod my-mod-1.0.0.psxmod +``` + +Install the resulting `.psxmod` through the launcher's Mods manager. A game can +also ship reviewed, default-disabled packages by placing unpacked sources at +`mods/preloaded/packages///` and copying +`mods/preloaded` beside the game executable as `mods`; see Tomba's build wiring +below. + +Choose the narrowest mechanism that describes the change: + +| Change | Package mechanism | +|---|---| +| Fixed code or data bytes | Guarded `[[patch]]` on `main_exe`, `disc_raw`, or `disc_user` | +| A player-selectable boolean, choice, or number | Feature-local `[[option]]` plus `when`, `replace_from`, sparse `fields`, or `when_integer` | +| Artwork, script, audio, or another large disc asset | Hashed file-backed `[[overlay]]`; do not rebuild the player's stock image | +| Host setting or live game behavior | Trusted static `[[plugin]]`, compiled into the game and selected by a stable id | +| Several features composing one shared table, bitfield, routine, or allocation | Game-owned `resolver = "builtin:"`, only when declarative operations cannot express the composition | + +Format versions 2–4 add bounded integers, ordered constraints, linked MIPS +`LUI`/`ORI` values, sparse owned fields, and integer predicates. Format 5 adds +trusted static plugins. The full schemas and resolution rules are in [`docs/MOD_PACKAGES.md`](docs/MOD_PACKAGES.md). +The enabled feature set is resolved before boot. Changing it may require a +relaunch, but never asks the player to patch a disc or compile the game. + +### Trusted game code + +A `.psxmod` never supplies or loads native code. If a feature needs a per-VBlank +callback or must configure a host-side facility, the game registers an +implementation that is already linked into its executable: + +```c +#include "mod_plugins.h" + +static void example_vblank(void) { + if (psx_mod_game_started()) + psx_mod_write_byte(0x1F8001B4u, 1u); +} + +PSX_MOD_CONSTRUCTOR(register_example_mod) { + (void)psx_mod_register_vblank_plugin( + "example.quick-start", example_vblank); +} +``` + +The manifest activates it with: + +```toml +format_version = 5 + +[[plugin]] +feature = "quick-start" +id = "example.quick-start" +``` + +Activation callbacks are for one-time pre-renderer configuration; VBlank +callbacks are deterministic guest-time work. Plugins receive only the narrow +services in +[`runtime/include/mod_plugins.h`](runtime/include/mod_plugins.h), can read +validated feature options, and are selected by registry id—not by a library +path or symbol supplied by an archive. Prefer ordinary patches and overlays +whenever they are enough. + +### Converting an existing patch or patcher + +Large legacy mod suites should be converted feature by feature: + +1. Pin the exact clean disc revision and keep the original patcher or patched + output as a byte-for-byte **test oracle**, not as the runtime payload. +2. Split its UI into independent features and typed, feature-local options. + Enabling one feature must not silently select another. +3. Map each owned change back to the stock image: executable writes become + guarded `main_exe` patches, small disc writes become guarded disc patches, + and large assets become content-addressed overlays. +4. Use sparse fields when features own adjacent bytes in one record. Introduce + a trusted built-in resolver only when independent selections must synthesize + one shared semantic object. +5. Test every feature alone, representative combinations, all-off identity, + wrong-revision rejection, collision diagnostics, and byte parity with the + oracle. Record authorship and redistribution permission for every imported + asset. + +Do not ship a prepatched disc or use the legacy `derived_disc`/VCDIFF path for +new packages. It remains conversion scaffolding only. + +### Real examples + +- [Tomba's merged catalog](https://github.com/mstan/TombaRecomp/tree/master/mods/preloaded/packages) + is the compact starting point. Its + [trusted plugins](https://github.com/mstan/TombaRecomp/tree/master/src/mods) + show activation callbacks, guest-VBlank behavior, launcher options, display + features, controller policy, and skip-FMV behavior; its + [CMake file](https://github.com/mstan/TombaRecomp/blob/master/CMakeLists.txt) + shows how to compile the plugin sources and preload the package catalog. +- [Mega Man X6's unmerged Tweaks package branch](https://github.com/mstan/MegaManX6Recomp/tree/integrate/mmx6-mod-packages-current) + is the large conversion example. The + [package generators](https://github.com/mstan/MegaManX6Recomp/tree/integrate/mmx6-mod-packages-current/tools) + preserve patcher parity while producing guarded stock-targeted operations and + content-addressed assets; the + [trusted resolvers](https://github.com/mstan/MegaManX6Recomp/tree/integrate/mmx6-mod-packages-current/src/mods) + demonstrate deterministic composition of shared records and injected code. + The earlier + [`feature/mmx6-tweaks`](https://github.com/mstan/MegaManX6Recomp/tree/feature/mmx6-tweaks) + branch records the patcher-parity and write-classification work that feeds + this conversion; its patched-disc/pre-bake experiments are history, not the + recommended package delivery model. Both branches are review material rather + than a released compatibility promise. + ## Swap language on the fly A general localization layer captures the game's own strings out of memory and @@ -549,13 +718,16 @@ the selected faithfully recompiled BIOS — OpenBIOS or retail BIOS — is the baseline and oracle, generated code is never hand-edited (fix the recompiler and regenerate), and a change proves itself against the Beetle oracle / on screen rather than by assertion. Game-specific work lives in the game repos, which pin -an exact framework commit as a submodule. +exact framework and UI commits as root-level submodules. -Read [`CONTRIBUTING.md`](CONTRIBUTING.md) before opening a PR — it covers the core -rules, how to verify a change, the regression checklist across the known games, -and how a framework fix reaches a game through its pin. Bugs and build problems go -to GitHub issues (include `gcc -v` / OS / generator for build failures); design -discussion happens in the **R.A.I.D.** Discord (invite below). +- New title / setup-host release: + [`docs/GAME_PROJECT_SETUP.md`](docs/GAME_PROJECT_SETUP.md) +- Framework PRs: [`CONTRIBUTING.md`](CONTRIBUTING.md) (rules, verification, + regression checklist, how a fix reaches a game through its pin) + +Bugs and build problems go to GitHub issues (include `gcc -v` / OS / generator +for build failures); design discussion happens in the **R.A.I.D.** Discord +(invite below). ## License diff --git a/cmake/toolchain-mingw-w64.cmake b/cmake/toolchain-mingw-w64.cmake new file mode 100644 index 00000000..73cc8043 --- /dev/null +++ b/cmake/toolchain-mingw-w64.cmake @@ -0,0 +1,46 @@ +# Cross-compile toolchain: Linux host → Windows x86_64 (MinGW-w64). +# +# Usage from a game repo: +# cmake -DCMAKE_TOOLCHAIN_FILE=psxrecomp/cmake/toolchain-mingw-w64.cmake ... +# +# Requires (Arch / CachyOS packages): +# mingw-w64-gcc mingw-w64-sdl2 (and cmake, ninja) +# Debian/Ubuntu equivalents: +# g++-mingw-w64-x86-64 + a MinGW SDL2 (distro or self-built) + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86_64) + +set(MINGW_TRIPLE "x86_64-w64-mingw32" CACHE STRING "MinGW target triple") + +set(CMAKE_C_COMPILER "${MINGW_TRIPLE}-gcc") +set(CMAKE_CXX_COMPILER "${MINGW_TRIPLE}-g++") +set(CMAKE_RC_COMPILER "${MINGW_TRIPLE}-windres") +set(CMAKE_RANLIB "${MINGW_TRIPLE}-ranlib") +set(CMAKE_AR "${MINGW_TRIPLE}-ar") +set(CMAKE_STRIP "${MINGW_TRIPLE}-strip") + +# Prefer the triple-prefixed pkg-config so we do not pick up host Linux SDL2. +find_program(_MINGW_PKG_CONFIG NAMES "${MINGW_TRIPLE}-pkg-config" pkg-config) +if(_MINGW_PKG_CONFIG) + set(PKG_CONFIG_EXECUTABLE "${_MINGW_PKG_CONFIG}" CACHE FILEPATH + "pkg-config for MinGW sysroot" FORCE) +endif() + +# Isolate pkg-config from the host's .pc files. +set(ENV{PKG_CONFIG_PATH} "") +if(DEFINED ENV{MINGW_PKG_CONFIG_LIBDIR} AND NOT "$ENV{MINGW_PKG_CONFIG_LIBDIR}" STREQUAL "") + set(ENV{PKG_CONFIG_LIBDIR} "$ENV{MINGW_PKG_CONFIG_LIBDIR}") +elseif(EXISTS "/usr/${MINGW_TRIPLE}/lib/pkgconfig") + set(ENV{PKG_CONFIG_LIBDIR} "/usr/${MINGW_TRIPLE}/lib/pkgconfig") +endif() + +set(CMAKE_FIND_ROOT_PATH "/usr/${MINGW_TRIPLE}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +# Large generated TUs can exceed COFF section limits on older binutils. +string(APPEND CMAKE_C_FLAGS_INIT " -Wa,-mbig-obj") +string(APPEND CMAKE_CXX_FLAGS_INIT " -Wa,-mbig-obj") diff --git a/docs/GAME_PROJECT_SETUP.md b/docs/GAME_PROJECT_SETUP.md new file mode 100644 index 00000000..bf105208 --- /dev/null +++ b/docs/GAME_PROJECT_SETUP.md @@ -0,0 +1,228 @@ +# Bringing up a game project on PSXRecomp + +This guide is for **title developers**: you keep game code and config in your +own repository, pin this framework and `recomp-ui` as **root-level submodules**, +and (optionally) ship a **setup-host** zip that users generate from a legal disc +locally — the same model as Bomberman Party Edition / Masters of Teräs Käsi. + +Related docs: + +| Doc | When | +|-----|------| +| [`BUILDING.md`](BUILDING.md) | Local build of the framework / BIOS runtime | +| [`LOCAL_CODEGEN_SDK.md`](LOCAL_CODEGEN_SDK.md) | CLI contract for Generate & rebuild | +| [`ci/README.md`](ci/README.md) | Shared CI scripts and composite actions | +| [`CONTRIBUTING.md`](../CONTRIBUTING.md) | Framework contribution rules | + +--- + +## Recommended repository layout + +Everything game-specific lives at the **root** of *your* title repo. Framework +and UI are submodules next to that code — not nested under each other. + +The RetComM / Generate & rebuild CLI (`psxrecomp_cli.py`, `tools/prepare_disc.py`, +pack helpers) ships **inside** the `psxrecomp` submodule. There is no separate +`psxrecomp-sdk/` tree. + +```text +YourGameRecomp/ # your git repo +├── .gitmodules +├── CMakeLists.txt # thin: psxrecomp_add_game_runtime(...) +├── game.toml # disc / recompiler / runtime config +├── VERSION # release pin (e.g. 0.1.0) +├── seeds/ # function-start seeds for psxrecomp-game +├── codegen_setup.c / .h # title PsxrecompCodegenHostConfig (+ apply hooks) +├── launcher_assets/ # boxart / fonts / brand (recomp-ui) +├── scripts/ +│ └── package_setup_release.sh # thin wrapper → package_setup_host.sh +├── .github/workflows/ +│ └── release.yml # from docs/ci/templates/setup-release.yml +├── psxrecomp/ # submodule → framework + CLI + codegen host + CI tools +│ └── host/psxrecomp_codegen_host.* +├── recomp-ui/ # submodule → launcher UI +├── generated/ # local only — gitignore (not in CI setup zip) +└── prepared_disc/ / disc/ # local disc working tree — gitignore +``` + +Minimal `CMakeLists.txt` shape: + +```cmake +set(PSXRECOMP_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/psxrecomp") +include("${PSXRECOMP_ROOT}/runtime/runtime.cmake") +psxrecomp_add_game_runtime(psx-runtime + ENABLE_NETPLAY_IF_PRESENT + WINDOW_TITLE "My Game Recompiled" + GEN_MARKER "generated/SLUS_01234_dispatch.c" + GEN_FULL_GLOB "generated/SLUS_01234_full_*.c" + CODEGEN_SETUP_SOURCES codegen_setup.c + DEFAULT_GAME_CONFIG_PATH "game.toml" + LAUNCHER_BOXART "${CMAKE_CURRENT_SOURCE_DIR}/launcher_assets/img/boxart.tga" +) +``` +### Add the submodules + +```bash +cd YourGameRecomp +git submodule add https://github.com/mstan/psxrecomp.git psxrecomp +git submodule add https://github.com/TechnicallyComputers/recomp-ui.git recomp-ui +git submodule update --init --recursive +``` + +Pin **exact commits** (do not track `main` floating). Bump deliberately when you +want a newer framework or UI. + +Clone for contributors: + +```bash +git clone --recurse-submodules https://github.com/you/YourGameRecomp.git +``` + +--- + +## Local development loop + +1. **Legal disc** — place your dump where `game.toml` expects it (or pass + `--disc` to the CLI / wizard). Never commit disc images or retail BIOS dumps. +2. **Build emitters** (once per machine / after recompiler changes): + + ```bash + ./psxrecomp/tools/ci/build_emitters.sh + ``` + +3. **Generate game (+ OpenBIOS) C** — either the setup wizard, or: + + ```bash + python3 psxrecomp/psxrecomp_cli.py generate \ + --config game.toml --project-root . --disc /path/to/dump.cue + ``` + +4. **Build the playable runtime**: + + ```bash + cmake -S . -B build-release -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build build-release --target psx-runtime -j"$(nproc)" + ``` + +Details: [`LOCAL_CODEGEN_SDK.md`](LOCAL_CODEGEN_SDK.md), [`BUILDING.md`](BUILDING.md). + +--- + +## Setup-host releases (self-build zip) + +CI ships a zip **without** `generated/` game C, retail BIOS dumps, or an +embedded `toolchain/`. The zip includes: + +- Setup host exe (`recomp-ui` + generate/rebuild wizard) +- Your game sources (`game.toml`, seeds, CMake, host glue, …) +- `psxrecomp/` (runtime + CLI + OpenBIOS profiles + emitters) +- `recomp-ui/` sources (needed to rebuild) +- On Windows: MinGW runtime DLLs beside the host and emitters + +Players (or [RetComM](https://github.com/TechnicallyComputers/RetComM-Launcher)) +run Generate & rebuild locally. RetComM / the wizard download +`cmake-clang-v1` from +[retcomm-toolchains](https://github.com/TechnicallyComputers/retcomm-toolchains) +(or accept an offline zip / `RETCOMM_TOOLCHAIN_DIR`). Pass +`--embed-toolchain` to `package_setup_host.sh` only for special offline-first +packs. + +Do **not** set `PSX_PGO` in CI. PGO stays user-local when `[pgo] enabled = true`. + +--- + +## Shared tools (use these; do not reimplement) + +| Tool | Role | +|------|------| +| `tools/ci/normalize_version.sh` | `vX.Y.Z` → `VERSION` / `TAG` | +| `tools/ci/clear_generated.sh` | Wipe `generated/` for setup-host CI | +| `tools/ci/record_pins.sh` | Log submodule SHAs | +| `tools/ci/build_emitters.sh` | Build `psxrecomp-game` + `psxrecomp-bios` | +| `tools/fetch_toolchain.sh` | Optional download/unpack (embed packs only) | +| `tools/stage_setup_sdk.sh` | Emitters + OpenBIOS + optional `toolchain/` + MinGW DLLs | +| `tools/bundle_mingw_dlls.sh` | Windows runtime DLL copy | +| `tools/package_setup_host.sh` | Full setup-host zip (title args) | + +Composite actions (from the game repo after checkout): + +```yaml +- uses: ./psxrecomp/.github/actions/build-emitters +# Optional — only when packaging with --embed-toolchain: +# - uses: ./psxrecomp/.github/actions/fetch-toolchain +# with: +# artifact: ${{ matrix.artifact }} +- uses: ./psxrecomp/.github/actions/stage-setup-sdk + with: + stage: dist/stage-setup-${{ matrix.artifact }} + recompiler-build: build-recompiler + allow-no-toolchain: 'true' +``` + +--- + +## CI workflow template + +1. Copy the template into your title repo: + + ```bash + mkdir -p .github/workflows scripts + cp psxrecomp/docs/ci/templates/setup-release.yml .github/workflows/release.yml + ``` + +2. Replace every `YOUR_*` placeholder. +3. Add a thin `scripts/package_setup_release.sh` that calls + `psxrecomp/tools/package_setup_host.sh` with your exe name / zip prefix + (see Bomberman Party Edition for an example). + +Full action reference: [`ci/README.md`](ci/README.md). + +--- + +## Bundled release checklist + +Use this before tagging a setup-host release that matches other titles +(BPE / MotK / RetComM). + +### Repository + +- [ ] `psxrecomp/` and `recomp-ui/` are root-level submodules on pinned commits +- [ ] CLI lives in the submodule (`psxrecomp/psxrecomp_cli.py`) — no sibling sdk +- [ ] `game.toml` has disc identity (`[prepare_disc]` hashes/sizes) and boot EXE +- [ ] Seeds cover the boot path; `VERSION` matches the release you will tag +- [ ] Disc images, `generated/`, and `SCPH1001.BIN` are gitignored +- [ ] Setup-host CMake path builds with **no** game C and **no** BIOS backends + (e.g. `-DPSXRECOMP_ALLOW_NO_BIOS=ON` + your title’s force-setup option) +- [ ] Thin `codegen_setup.c` + `psxrecomp_add_game_runtime` (codegen host is in + `psxrecomp/host/`; CI may use `-DPSXRECOMP_FORCE_SETUP_HOST=ON`) +- [ ] Optional: start from `docs/ci/templates/game.gitignore` + +### Packaging (shared helpers) + +- [ ] CI uses `./psxrecomp/.github/actions/build-emitters` +- [ ] Packager calls `psxrecomp/tools/package_setup_host.sh` (lean zip by + default; optional `--embed-toolchain`) or `stage_setup_sdk.sh` after a + custom stage +- [ ] Staged tree includes `psxrecomp/psxrecomp_cli.py` +- [ ] Staged `psxrecomp/bios/` has `OpenBIOS.toml`, `openbios.bin`, + `OpenBIOS.LICENSE`, `SCPH1001.toml` — and **no** retail `.BIN` +- [ ] Windows zip has `libstdc++-6.dll` + `libgcc_s_seh-1.dll` next to both + emitters (and host deps such as `zlib1.dll` next to the exe) +- [ ] Zip mtimes are normalized (packager `touch`) so Ninja does not see + “future” files after extract + +### Zip contents smoke test + +- [ ] Linux / macOS / Windows artifacts named consistently + (`YOUR_PREFIX--.zip`) +- [ ] Extract → run host → Generate with a legal disc succeeds end-to-end +- [ ] After rebuild, game launches; saves/settings land beside the exe +- [ ] RetComM install/update (if you publish a catalog entry) uses this **same** + zip — no separate tools pack required + +### Publish + +- [ ] Tag `vX.Y.Z`; GitHub Release attaches all four platform zips +- [ ] Release notes say the zip is a setup host (no disc / no retail BIOS / + no pre-generated game C) +- [ ] Catalog / RetComM entry points at the release assets when ready diff --git a/docs/LOCAL_CODEGEN_SDK.md b/docs/LOCAL_CODEGEN_SDK.md new file mode 100644 index 00000000..2369cbe2 --- /dev/null +++ b/docs/LOCAL_CODEGEN_SDK.md @@ -0,0 +1,149 @@ +# Local codegen SDK + +Headless contract for regenerating an existing psxrecomp game project from a +user-supplied disc. Intended for `recomp-ui` setup flows and RetComM launcher +automation. This does **not** redistribute disc images. + +PGO (optional) runs **only on the user’s machine** during local rebuild when +`game.toml` has `[pgo] enabled = true`. CI must not set `PSX_PGO`. + +## Setup host (CI without game/BIOS generated C) + +Games can ship a **setup host**: `psx-runtime` linked **without** game C and +**without** BIOS backends (`-DPSXRECOMP_FORCE_SETUP_HOST=ON`, or legacy +`-DBPE_FORCE_SETUP_HOST=ON`) via `psxrecomp_add_game_runtime(...)`. CI never +needs BIOS dumps or private assets. First-run Generate emits OpenBIOS (from +bundled `openbios.bin`) and optional SCPH1001 (player dump), then game C, then +rebuild links everything. + +| Piece | Role | +|-------|------| +| Setup exe | `recomp-ui` + codegen host; opens Generate & rebuild | +| Game zip `psxrecomp/` | submodule tree: CLI, tools, emitters, OpenBIOS profiles | +| `psxrecomp_cli.py` | Generate / rebuild / verify-disc (in the submodule) | +| `tools/package_setup_host.sh` | Universal setup-host zip packager | +| `tools/ci/*.sh` | normalize version, clear generated, record pins, build emitters | +| `tools/fetch_toolchain.sh` | Optional: download/unpack `cmake-clang-v1` (CI embed or local) | +| `tools/toolchain_pack.py` | CLI: resolve / download / unpack into `toolchain/` + shared cache | +| `tools/stage_setup_sdk.sh` | Pack: emitters, OpenBIOS checks, optional `toolchain/`, MinGW DLLs | +| `tools/bundle_mingw_dlls.sh` | Windows: copy MinGW runtime DLLs next to host + emitters | +| Project `toolchain/` | Wizard/CLI install target (`bin/cmake`, stamp `.psxrecomp-bin`) | +| Shared toolchain cache | `~/.local/share/psxrecomp/toolchains/cmake-clang-v1/` (or RetComM’s) | +| `docs/ci/` | Composite actions + [`templates/setup-release.yml`](ci/templates/setup-release.yml) | +| `docs/GAME_PROJECT_SETUP.md` | Submodules, CI template usage, bundled-release checklist | +| Game sources | `game.toml`, seeds, `CMakeLists.txt` at repo root; `psxrecomp/`, `recomp-ui/` submodules | + +RetComM harvests emitters into the SDK cache and **downloads** `cmake-clang-v1` +(no separate tools zip; game zips stay lean). The setup host installs the +portable pack with **host-native** `curl` + `tar`/`unzip` into +`%LOCALAPPDATA%/psxrecomp/toolchains/…` (or XDG), not through Python — so +Microsoft Store Python AppData redirection cannot hide cmake. Offline builds +accept a zip via the wizard file picker. Generate/rebuild still need Python 3 +for `psxrecomp_cli.py`. + +## Commands + +```bash +python psxrecomp/psxrecomp_cli.py verify-disc \ + --config game.toml --disc path/to/dump.bin [--json-progress] + +python psxrecomp/psxrecomp_cli.py generate \ + --config game.toml --project-root . --disc path/to/dump.iso \ + [--bios path/to/SCPH1001.BIN] [--force-bios] \ + [--skip-hash-check] [--force-prepare] [--json-progress] + +python psxrecomp/psxrecomp_cli.py ensure-toolchain \ + [--project-root .] [--from-zip cmake-clang-v1-linux-x64.zip] [--no-download] + +python psxrecomp/psxrecomp_cli.py rebuild \ + --config game.toml --project-root . \ + --build-dir build-release --target psx-runtime \ + --exe-basename Bomberman_Party_Edition_Recompiled \ + [--disc path/to/game.cue] [--toolchain-zip path/to/pack.zip] \ + [--no-toolchain-download] [--no-pgo] [--force-pgo] [--json-progress] + +python psxrecomp/psxrecomp_cli.py pgo-train \ + --config game.toml --build-dir build-release \ + --exe-basename Bomberman_Party_Edition_Recompiled \ + [--disc …] [--train-secs 120] [--train-runs 3] [--json-progress] +``` + +`generate` normalizes the dump via `tools/prepare_disc.py` when needed, then +runs `psxrecomp-game --config game.toml` into `[recompiler] out_dir`. + +`rebuild` runs cmake. If `[pgo] enabled = true` (and `--no-pgo` not set), it +instruments, trains (`scenario` / timed boot), then rebuilds with profiles. + +## Exit codes + +| Code | Meaning | +|------|---------| +| 0 | success | +| 1 | runtime / generation / build failure | +| 2 | usage / argument error | +| 3 | disc verification failure | + +## JSONL progress (`--json-progress`) + +Stdout is reserved for one JSON object per line. Useful events: + +| `event` | Notes | +|---------|--------| +| `phase` | `verify`, `prepare_disc`, `emit`, `build`, `pgo_*`, `done` | +| `disc` | digests after verification | +| `log` | mirrored tool chatter | +| `result` | final payload | +| `error` | `message`, `code` | + +## Portable recomp-ui host (`psxrecomp/host/`) + +Prefer `psxrecomp_add_game_runtime(...)` — it compiles +`psxrecomp/host/psxrecomp_codegen_host.c` and sets `PSX_HAS_GAME_CODEGEN`. + +Titles keep a thin root `codegen_setup.c` that fills +`PsxrecompCodegenHostConfig` and exposes apply/relaunch hooks: + +```c +psxrecomp_codegen_host_apply(&gi, &my_cfg); + +/* after recomp_launcher_run_window: */ +if (lr == RECOMP_LAUNCHER_RESULT_RELAUNCH) + psxrecomp_codegen_host_relaunch_or_exit(disc_path); +``` + +### Modular `[pgo]` (game.toml) — opt-in + +PGO runs only when the title sets `enabled = true`. Framework defaults when +enabled: **60s × 2 runs**, `mute_host_audio = true`, `hide_video = true`. +Games may lengthen trains in their own `game.toml`. + +```toml +[pgo] +enabled = true # required to opt in +train_secs = 60 # optional override +train_runs = 2 # optional override +mute_host_audio = true # default: discard SDL output (SPU still runs) +hide_video = true # default: --headless (no on-screen FMV) +scenario = "boot_timed" # title-authored workload hint +``` + +Omit the section or set `enabled = false` to skip. During train the CLI/UI show +a **WARNING** — do not cancel the process (or close a visible window if +`hide_video = false`). + +`hide_video` keeps guest MDEC/FMV decode in the profile but shows nothing on +screen (avoids abrasive/flickering FMV). Use `--pgo-video` only when you need +host present paths in the profile. + +### Env overrides + +| Env | Role | +|-----|------| +| `PSXRECOMP_PROJECT_ROOT` / game-specific | project root | +| `PSXRECOMP_BUILD_DIR` / game-specific | cmake build dir | +| `PSXRECOMP_FORCE_SETUP` / game-specific | force setup wizard | +| `PSXRECOMP_GAME` | path to `psxrecomp-game` binary | +| `PSX_HOST_MUTE=1` | discard host SDL audio (SPU still runs; set by PGO train) | +| `PSX_HEADLESS=1` | no SDL window (set by PGO train when `hide_video`) | +| `SDL_VIDEODRIVER=dummy` | set by default with headless train | +| `PYTHON` / `CMAKE` | tool overrides | diff --git a/docs/ROLLBACK_MOTK_HOOKUP.md b/docs/ROLLBACK_MOTK_HOOKUP.md index 299d9b23..6c54e486 100644 --- a/docs/ROLLBACK_MOTK_HOOKUP.md +++ b/docs/ROLLBACK_MOTK_HOOKUP.md @@ -108,12 +108,42 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. baseline) coalesce press+release into one episode. Tip-extend from TipHold/Verify clears the POST handshake; follower FOLLOW mirrors initiator rereplay; TipHold ignores stale rexmit POSTs. - Post-TipHold `choose_load_tick` must not pick Live tip snaps above - `agreed_through` unless `hash_confirm` confirms them (else follower - NACK past frontier). + Tip-extend during Seal/AwaitingBaseline resigns+extends only (no + rereplay) so the initiator cannot solo-Replay before ready GO. + TipHold tip-extend **stays TipHold** (library); rereplay only if Live + already invented past the prior tip. FOLLOW mirrors that gate. + Tip-extend rereplay reloads the **prior tip** on both peers (not + mismatch-1→pin fallback vs FOLLOW old_t). + Tip-extend snap apply always `arm_rereplay_after_load` (hc_prime + + `sim=reload+1`) — poll_snap used to apply without arming, leaving the + TipHold Live tip clock and invent FRAME_COMMITs → false + `resim core diverge` (matched sealed fin, stale peer invent). + TipHold Live does **not** emit FRAME_COMMIT; tip-extend Replay drops + in-flight invent commits until the peer's sealed resim matches. + **TipHold invent-cap (menu D-pad storm):** Live stalls once + `sim > tip + tip_seal_slack` (MotK forces slack **0**). Stops invent + racing ahead of the tip and tip-extend rereplays on every held edge. + Quiet finalize counts `tip_runway` (MotK 24) pump frames at the invent + cap when sim cannot reach the old `tip_hold_until`. + **Seal span cap (menu hang):** peer-seal completion is a `uint64` + bitmask (`RNET_RB_PEER_SEAL_MASK_BITS` = 64). Tip-extend past + `seal_base+63` fails; TipHold used to `begin_refused` until runway + then open ep N+1 alone → `seal timeout`. Fix: TipHold span-cap + tip-hold-commits immediately and `begin_rewind` opens a fresh + episode; FOLLOW yields TipHold on a new-epoch SYNC. Self-test: + `rollback_episode_test` (extend to span=64 OK, span=65 fails). + **Stale POST after tip-extend:** `RB_POST` carries `target_tick`; + Verify ignores peer POSTs whose tip ≠ episode tip (tip-extend left + POST@T in the queue while Verify@T+1 → false diverge / peer verify + timeout). Self-test: `test_rb_post_tip_filter` + `rb_wire_test` + tip binding. + Post-TipHold `choose_load_tick` / follower frontier hard-cap at + `agreed_through` — ignore `hash_confirm` above that watermark (Live + invent + PC-cleared FRAME_COMMIT false-confirmed forked tip snaps). - `tip_seal_slack` (default 2) sets initial seal headroom; - `tip_runway` (default 12) caps TipHold + suggest slack; + `tip_seal_slack` (MotK 0 / library default 2) sets invent headroom past + tip + initial seal; `tip_runway` (MotK 24 / library default 12) caps + TipHold quiet + suggest slack; **light tip** (`RNET_RB_CORR_LIGHT_TIP`) skips ready-ACK RTT when load is at the shared frontier and depth ≤ 8. Do **not** soft-promote releases. @@ -129,14 +159,19 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. Do **not** freeze `cdrom_advance` during Replay (FMV skip resim hung). Promote wire into hist **before** `begin_rewind` seal. FMV media + **digest-gated post-FMV lockstep**: no invent + refuse - begin/follow for at least MIN=90 ticks (title Start was - invent→`0x8006CDA0` episode hang); hold until hash_confirm matched - CONFIRM=16 contiguous ticks or MAX=180 (cross-game cutover). Dense - tip snaps during media/lockstep/+32 after invent unlock so the first - invent-miss loads near the mismatch (was 954→944). Diag `FIRST CORE - DIVERGE` at first post-lockstep press is invent≠wire (e.g. `ffff`→ - `ffbf` Cross), not residual FMV non-determinism — live digs matched - through lockstep. + begin/follow for at least MIN=180 ticks (~3s; was 90 — invent≠Cross + at +14 opened a `0x8006CDA0` tip storm); hold until hash_confirm + matched CONFIRM=16 contiguous ticks or MAX=300. On RELEASE: invent + stays off **UNLOCK_GRACE=64** more ticks (do not clamp until→sim) and + arm `promote_sweep` so sticky hold-last poison cannot invent at + unlock+1. Invent stays **hold-last** after unlock (idle invent + re-mismatches every held D-pad tick — char-select freeze class); a + `pub=ffbf wire=ffff` FIRST CORE at unlock is a normal press/release + mispredict and light tips handle it. Dense tip snaps during + media/lockstep/+32 after invent unlock. + MotK TipHold: `tip_runway=24`, invent slack **0** (Live stalls at tip) + so tip-extend rarely rereplays — menu D-pad coalesce without the + prior-tip reload FPS cliff. Symmetric ready: follower ACK → initiator GO → both Replay. **`flush_resume` releases `gpu_vblank_flush_present` reentrancy guard** before longjmp (stuck `s_flushing_present` blocked all later presents / @@ -147,7 +182,11 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. post-present). Phantom fin@load before the latched VBlank IRQ put peers on opposite ping-pong edges (`v0=1` vs countdown, ~5-cycle skew). First arm is `load+1`; `hc` primed after load. Netplay flushes deferred present - **after** IRQ delivery when both are due (post-RFE digest). + **after** IRQ delivery when both are due (post-RFE digest). Also: do + **not** flush present on BB edges while `in_exception` (handler sees + IEc clear → old path `finish_frame` mid-BIOS before RFE restore; + soak irqctx `restored=0/reason=0`, sealed Cross resim forked + `v0=1` vs countdown / cyc±14). **Returning-leaf flush_resume (`0x8006A9F8` → PC=0):** prefer IRQ/sticky/ `$ra` over bare function-entry snap PCs; `flush_resume` clears deferred present + bb_defer nest; sentinel same-thread path must not publish @@ -155,14 +194,29 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. - [x] **Abort cooldown from live sim** (before realign rewinds the clock). Old path armed `until` from rewound tip → uncapped catch-up burned a short window in ms → char-select episode storms (`STALE COOLDOWN`). - Failed episode: 60 ticks from live (120 on streak≥2; 90 if no tip). + Failed episode: **24** ticks from live (**48** on streak≥2; was 60/120). + Begin **SPAN CAP=24**: deep post-abort catch-up is chunked (commit → + next episode) instead of one Replay across the whole cooldown + (soak depths 63→128 felt like "pushing further back"). Reconcile promotes wire for the whole abort window (`promote-no-resim`). Clean commit does **not** arm cooldown. +- [x] **Present only at MotK wait CDA0:** netplay `gpu_vblank_flush_present` + skips while resume/check PC is `0x8006CD54` (leave pending until + `0x8006CDA0`) so sealed idle resim cannot digest opposite ping-pong + edges (cyc±9 / v0 fork after mid-handler present fix). - [x] Netplay forces **software GPU** — GL/VK `glReadPixels` VRAM readback was forking peer snaps (core matched, pin zlib ~220KB apart) and mid-resim cores; baseline/POST also agree on `av=` (GPU+VRAM) via dig_b / POST input_digest - [x] SW GPU **before window** + late `force_sw` builds `SDL_Renderer` after GL - teardown (first match was black: cleared `g_gl_active` with null present) + teardown (first match was black: cleared `g_gl_active` with null present). + Rematch: only lock SW after `ensure_sw_sdl_present` succeeds; retry if + prior force left a null renderer (empty window after lobby rematch). +- [x] Mid-FMV tip load: `cdrom_resync_deadlines_after_restore`; do not wipe SPU + CD FIFO while XA/FMV pending; FMV media includes `cdrom_fmv_stream_pending` + (XA mode+reading) so invent-off arms before first MDEC colour decode; + `mdec_recently_active` uses guest-cycle age (host `s_frame_count` lied + under present-skip / Replay). RB frontend hook resets depth24 cutover + when media is already live (avoid permanent cutover blank). - [x] `flush_resume` releases present-flush reentrancy guard (no latched-VBlank re-arm — that phantom-finished load and forked menu wait resim). Symmetric ready GO. Live snaps stay on during media. Replay arms @@ -231,6 +285,16 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. Cause: flush_resume armed deferred present for latched I_STAT → phantom finish_frame before latched VBlank IRQ. Fix: arm `load+1` only; no latched re-arm; present-after-IRQ when delivery is due. +- [x] **Mid-handler present flush:** sealed Cross tip still forked fin@load+2 + (`v0=1` vs countdown, cyc±14) with matched pads/SIO/baseline; irqctx + stuck at `restored=0/reason=0`. Cause: handler BB edges flushed deferred + present while `in_exception` (IEc clear). Fix: skip present flush until + outer delivery returns; `gpu_vblank_flush_present` also refuses + `in_exception`. +- [x] **CDA0-only present + abort span cap:** after mid-handler fix, tip+1 + idle sealed resim still forked cyc±9 on CD54↔CDA0; abort cooldown + then opened 63–128-tick catch-up Replays. Present defers at CD54; + abort cool 24/48; begin SPAN CAP 24 chunks catch-up. - [x] **Returning-leaf resume crash:** after load+1 fix, Start-class episode resumed at `0x8006A9F8` (`jr $ra` leaf) with IEc=0 → top-level `execution completed, PC=0` (cps: `final_ra=0x8006CCA0`). Fix: resume @@ -357,7 +421,495 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. aborted on cpu-only MotK wait-loop PC forks (`CD54`↔`CDA0`) despite matched ram/clk. Fix: `netplay_hc_heal_stale_gap`, refuse loads past the shared frontier, canonicalize wait-loop snap PC to `CDA0`. + **Agreed tip aged out of snap ring:** after tip-hold commit, Live can + advance ~900 ticks with matching FRAME_COMMITs (`hc=1824`) while + `agreed_through=919` falls below `oldest` — begin REFUSED forever + (`no confirmed snap`), invent≠wire never resims, cores fork with + matched clocks. Fix: `heal_agreed_watermark_if_aged_out` raises + agreed to the highest hc-confirmed snap still in the ring (only when + *no* agreed-era snap remains — TipHold false-confirm guard intact). `rb live dig local` is a breadcrumb (not peer match). + **BOOTSTRAP when tip-hold never ran:** doomed early FMV abort left + `agreed` invalid and `hc` stuck below ring oldest (live core forks + block `heal_stale_gap`). Remote saw host P1 on the wire (`pub=ffff + wire=ffbf`) but `begin REFUSED` forever — P2 corrections still opened + on host. Fix: seed agreed to newest interval snap (`BOOTSTRAP` / + `HEAL-FORCE`); choose_load/follow fall through when watermark `< oldest`. + **Post-HEAL resim CD54↔CDA0 (cyc±3 / −43):** after `agreed HEAL→944`, + identical baseline + pads still forked fin@945 — dig2 IRQ at CDA0 + (`cyc=…569`, countdown v0) vs dig1 at CD54 (`…572`, v0=1); dig2 later + matched dig1 on a warm retry. Cause: `interrupts_resync` zeroed + `cycles_since_vblank` (not in snap/digest) while timers/LCF kept mid-frame + phase; CDA0-only present treated latch `pc==0` as presentable; Replay + tip-apply from BB poll mutated CPU under live CPS. Fix: persist csv in + `BS_SEC_IRQ` (12B), stop zeroing on resync, fold csv into clk digest, + present only at CDA0 (sticky/last fallback), episode loads only via + `try_admit`. + **Post-csv residual (clk/tim +9, double fin):** after csv persist, sealed + tip still aborted at resim 1202 with matched GPR/cpu/ram/av/SIO but + irqctx cyc ±9; dig1 `fin@1202` and `fin@1203` shared identical dig/cyc + (double `finish_frame`, no guest advance). Cause: sticky I_STAT kept + `delivery_needed` so entry skipped present while post-IRQ flush also + skipped on CD54 → deferred pending stacked across a full VBlank, then + one CDA0 drain ran finish twice; sticky-unknown defer (latch cleared) + worsened accumulation. Fix: attempt present at entry even when delivery + is due (CDA0 gate no-ops CD54); coalesce `s_present_pending` to 1; + sticky CDA0 with cleared latches allows coalesce present. + **Arm→first-present +2 VB on long Replay:** post-HEAL short tips matched, + but cold span-cap load (ep9 952→976) forked fin@953 — dig1 arm+2.03 VB + vs dig2 arm+1.00 VB, pads matched. Cause: entry `gpu_vblank_flush_present` + ran before `s_last_interrupt_check_pc` was updated, so MotK gate saw stale + `last==CD54` on every CDA0 edge and no-op'd; present only drained on + post-IRQ at a CDA0 delivery. First post-arm VB taken at CD54 → full extra + period. Fix: publish edge PC before entry flush; gate prefers compiled + CDA0 pc over stale last. + **Non-det fin@946 CD54 vs CDA0 (resim storm):** matched baseline + pads; + dig1 alone flipped cores across warm retries — IRQ at CD54 (`v0=1`) vs + present-before-IRQ at CDA0 (countdown). Fix: hold VBlank-only delivery at + CD54 while deferred present is armed (deliver after CDA0 present); MotK + gate requires explicit CDA0 edge (no sticky-only allow); flush_resume + pins sticky to canonical CDA0. + **FMV1→FMV2 cutover stall:** soak matched through FMV arm then guest + collapsed (~5 ms) with TURN admit wait and no `FMV settle` — media stayed + “live” (depth24) while MDEC idled. MotK present gate treated sticky wait + PC as in_wait and blocked non-CDA0 edges (FMV/cutover). Fix: gate only + CD54 edges (sticky only when latch cleared); depth24 1/4 present-skip and + FRAME_COMMIT skip only while MDEC hot; `rb FMV media` heartbeat every 32. + **±1 VB replay fork at MotK wait (abort@902):** identical committed + baseline 901 (`e6a64d47`) + idle pads still forked the first resim tick — + abort cyc `509725487` vs `509160985` (Δ≈564480 = exactly one VBlank), + wait countdown `r21=0x704` vs `0x705`, `v0=5c88` vs `5c89`. One peer + delivered one extra VBlank during replay. Cause: the CD54 delivery-hold + was gated on `gpu_vblank_present_pending()` — `s_present_pending` is + host-only state (not in the snap), so peers entered replay with + different hold behaviour. Fix: hold VBlank-only delivery at the CD54 + edge in netplay **unconditionally** (edge PC + I_STAT are + guest-deterministic; CDA0 delivers a few instructions later). + The FIRST CORE @896 (`pub=ffbf wire=ffff`) preceding it was a normal + press/release mispredict — ep1–2 committed fine; the storm was this + replay fork. UNLOCK_GRACE stays 64. + + **Residual Δ8-cycle replay fork (abort@940) — I-cache tags:** with the + unconditional CD54 hold in, the next soak forked far smaller: identical + baseline `f83f685b` / arm cyc `530611223` on both peers, fin@940 cyc + `531175713` vs `531175721` (Δ8), `v0=5c83` vs `5c86` — VBlank taken a + few wait-loop iterations apart, both at CDA0. Cause: the R3000A + I-cache fetch-cost tags (`g_psx_icache_tv`, `psx_icache.c`) are + host-persistent and were NOT in the snapshot — each peer replayed with + the cache its own live/retry timeline had built, so fetch-miss cycles + differed and IRQ delivery drifted. (Selfcheck/overlay replay already + shadow this state; RB didn't.) Fix: new `BS_SEC_ICACHE` snapshot + section (1024 tag words) written/applied by `boot_state.c` — warm + loads, retries, and the exchanged baseline blob all share the exact + fetch-timing state — and the tags are folded into the `clk` digest + partition so any cache asymmetry surfaces at compare time instead of + as a mid-resim abort. Section is optional on load (old blobs load + untouched); netplay peers are bit-identical builds anyway. + + **Post-fix soak (2026-07-31): determinism CLEAN, storm is now pure + episode cost.** Full soak with BS_SEC_ICACHE: **zero aborts, zero + resim diverges** — 40+ consecutive episodes all committed first try. + But menu play ran 0.44–0.8x: every button press/release edge + (`pub=ffef/ffbf` vs `wire=ffff`, wire 1–2 ticks late) opens a paired + episode (baseline pin 3.6MB + seal/POST handshake + tip-hold). + The follow-up soak with `guest=`/`admit=` FPS split corrected the + first theory: the link was **direct LAN all along** (ICE selected + `typ host` candidates despite `force_turn=1` in the config banner) + and `admit` (network wait) is mostly <5 ms/f. The real costs: + (a) **guest CPU** — the MotK menu scene costs ~16.6 ms/f of guest + work under netplay (software GPU forced + idle_skip forced off), + i.e. zero headroom at 60fps, so every 2–4-tick resim burst drops + frames (guest hits 20–29 ms/f during storms; pre-menu boot ran + 119 fps at guest 6–8 ms/f — it's the scene, not the engine); + (b) **mispredict frequency** — with delay=2 and both sims hovering + ±1 tick apart (lead=0 admission), the wire for tick T routinely + seals 1–2 ticks late on every edge, and each episode further delays + input relay (self-sustaining storm). Next: raise input delay to 3–4 + (margin for sim skew; kills most menu invents on LAN), and/or LAN + `PSX_NET_PREDICTION=0` (stall instead of invent — RTT≈0 makes it + free). Longer pole: shrink menu guest cost (hardware GPU needs a + deterministic VRAM snapshot path; sw rasterizer + full busy-wait + emulation is the ceiling), and drop baseline exchange/POST verify + from sealed-input-only episodes now that determinism is proven + (zero aborts across two full soaks). + + **Stall-before-invent grace (implemented):** `np_try_admit_rollback` + invented hold-last on the FIRST miss of a remote wire row; the row + then landed 1–2 ticks later and the mispredicted edge opened an + episode. New `np_invent_grace_stall` (psx_netplay.c): before + inventing, stall the admit up to **PSX_RB_INVENT_GRACE_MS (default + 30 ms)** from the first miss of that wire tick — the admit loop + pumps the session and waits on UDP at ~1 ms granularity. Adaptive + off: 45 consecutive expiries (peer genuinely lagging beyond the + budget, real WAN) disables the grace for 2 s so it never stacks a + per-tick stall on top of real latency. Host-side pacing only — + invented values are unchanged, guest determinism unaffected. + Set `PSX_RB_INVENT_GRACE_MS=0` to disable. + The first cut used 8 ms and failed: the follow-up soak showed the + lateness is **sim skew, not link latency** — ALL late edges were + on the peer running 1–2 ticks ahead (22×late-1, 5×late-2; the + other peer had zero), and 8 ms is less than one tick of skew, so + the grace expired on every edge, tripped adaptive-off, and the + storm returned ("stable for a few seconds then nonstop"). The + grace is really a **rate governor**: while the ahead peer stalls, + the behind peer keeps simulating and catches up, so a budget that + covers ~2 ticks keeps the sims aligned and, in steady state, + inputs arrive before the seal with no stall at all. + **Tick-period scaling (2026-07-31):** the first full-match soak + showed the fixed 30 ms budget being outrun in gameplay: the fight + scene runs 35–45 fps (guest 9–20 ms/f + admit), so a tick lasts + ~25 ms and 2 ticks of skew is 50 ms — edges mispredicted again + (~40 episodes/match, "stormy" feel; lateness profile 27×late-0 / + 12×late-1 on the ahead peer). The grace now measures the live + tick period (EMA over consecutive tracked wire ticks) and uses + `max(PSX_RB_INVENT_GRACE_MS, 2.5 × tick EMA)`, capped 100 ms, as + the effective budget, so it self-scales to whatever rate the sim + actually runs. At 60 fps the EMA path gives ~42 ms; at 40 fps + ~62 ms. Adaptive-off (45 expiries → OFF 2 s) still guards real + WAN latency. + **Gap gate (2026-07-31, same day):** the scaled grace fixed the + storms (soak: zero episodes, zero aborts, deterministic match) + but serialized the sims: stalling on ANY missing row meant + neither peer ever ran ahead, so each admit waited out the other + peer's guest frame — the FPS logs showed each side's admit ms/f + mirroring the OTHER side's guest ms/f (8+12 vs 13+9, both + summing ~23 ms → 40–50 fps all match with no episodes at all). + Fix: the grace is only invoked when `wire > + highest_remote_wire + 1` — a 1-tick gap is the normal phase + offset of two pipelined sims and is invented through freely + (hold-last only mispredicts on real edges, and episodes are + cheap now that the agreed watermark tracks live). The stall — + the rate governor — engages only at >=2 ticks of genuine skew, + which was the storm condition it was built for. + + **Timesync micro-throttle (2026-07-31, follow-up soak):** with + the gap gate, idle play held a clean 60 fps (admit ~0) — but + every input burst dropped fps to 34–48. Episode forensics: 29 + episodes/side, 28/29 already on the light path, loads shallow + (2–9 ticks). The cost is the replay itself — a paired 2–9 tick + resim at ~13–16 ms guest/tick on BOTH peers per button edge — + and the episodes all came from the same source as ever: the + ahead peer running ~1 tick of phase ahead and inventing each + edge just before the real row landed (lateness 23xlate-0 / + 6xlate-1, all on one side). New `np_timesync_throttle` + (psx_netplay.c), GGPO-style: advantage = wire_need − highest + received remote wire row, EMA'd once per tick; above +0.5 tick + the ahead peer shaves up to 3 ms per admitted tick until the + phase closes. Exactly one side sees positive advantage, so one + side paces down briefly and then rows arrive before the seal — + no invent, no episode. Capped at 3 ms/tick (worst case ~20% + pace, never lockstep); skipped during episodes; adaptive off + (engaged ~3 s straight → OFF 5 s) because WAN transit inflates + the advantage and can't be closed by pacing. `PSX_RB_TIMESYNC=0` + disables. + **Advantage metric was wrong (2026-08-01 soak) — replaced with + mispredict-driven pacing debt.** The soak showed BOTH peers + measuring ~+0.6 ticks of "advantage" (10/16 and 11/16): each + side samples wire_need at the start of its own tick while the + peer's newest row was sent at the start of *its* tick, so the + metric's natural operating point is ~+0.5, not 0. Both sides + throttled, neither could close it, both tripped the off-guard + (`advantage 17/16 not closing`), and the mispredicts continued + (33xlate-1, 43 episodes/side, fights at 0.54–0.7x). Rework: the + unambiguous "I am the ahead peer" signal is the mispredict + itself — only the ahead side reconciles real rows that arrived + AFTER it invented them. `np_timesync_note_late()` is called from + `np_rollback_reconcile_wire` on every predicted row whose real + row lands (matching or not, so held/idle rows feed the loop and + alignment happens before an edge); each event adds ~half a tick + of pacing debt (capped 2 ticks) and the admit path shaves it at + <=3 ms/tick. Steady state aligned = no invents = no debt = zero + cost. Adaptive off: debt continuously nonzero ~5 s (WAN transit + — every row late regardless of phase) → OFF 10 s. + Longer pole (documented, not built): one-sided + episodes — the follower's live state through a sealed-only span + is already the post-replay state (hc stores its per-tick + digests), so it could answer baseline/POST from stored data and + never rewind, halving system replay cost. Not attempted yet: + protocol-level change on freshly stabilized machinery. + + **Two bugs found in a matched-pair soak (2026-08-01), both fixed:** + 1. `note_late()` was called unconditionally on every predicted-row + resolution, not gated on `pads_differ`. Under normal input-delay + prediction EVERY invented row eventually resolves against a real + wire row whether or not the guess was right — that's just how + delay-based prediction works — so the debt signal was mostly + noise. Moved the call after the `pads_differ` computation and + gated on it; the busier peer's rewind-request count (28 vs 12 + across a matched pair) now cleanly identifies which side is + actually mispredicting. + 2. The off-guard used "debt continuously nonzero for 5s" → OFF. + Soak evidence it was actively harmful: 24/36 episode commits and + 19/28 rewind-requests on the busier peer landed AFTER the + off-guard fired mid-storm — an active mispredict burst legitimately + keeps debt elevated (each edge tops it up before the last slice + fully drains), that isn't pacing failing. Replaced with a streak + counter: debt landing AT THE CAP with no room to have drained + between hits, 12 times in a row, before disabling for 10s. This + is the actual "pacing can't help, this is transit" signal. + + **Depth24 (FMV) present path batched (2026-08-01):** FMV frames + showed guest cost roughly double the non-FMV baseline (15–22 ms/f + vs 7–13 ms/f) with admit cost a minor fraction — a local compute + cost, not a netplay one. `gpu_display_pixel_argb` was called once + per output pixel, and for `depth24` did 3 separate `gpu_vram_byte()` + calls each recomputing the row/wrap math fresh — 3x the per-pixel + work of the 16bpp path, as a function-call chain instead of a + straight-line loop. New `gpu_depth24_present_row()` (gpu.c) hoists + the per-row invariants (vy, base byte offset, how many pixels fall + inside the 2048-byte VRAM row before the black-fill tail) out of + the inner loop and inlines the byte extraction — same byte-order + shifts as `gpu_vram_byte` (no host-endianness assumption), same + output. Wired into both depth24 present call sites in `main.cpp` + (Vulkan CPU-readout path and the generic non-hires path); the + 16bpp path is untouched. + + **Unrelated build-system bug found while soaking:** + `runtime.cmake`'s `PSXRECOMP_RUNTIME_SOURCES` list was missing + `netplay_snap_ring.c`, `netplay_state_digest.c`, + `netplay_hash_confirm.c`, `netplay_input_hist.c`, + `psx_netplay_rb.c`, and `psx_selfcheck.c` — a stale local edit + unrelated to netplay work, likely from the `psxrecomp_add_game_runtime` + scaffolding added alongside it. Every rebuild in this session up to + this point silently linked STALE prebuilt `.o`s for those 6 units + from before the edit (their sources hadn't changed, so ninja never + flagged them) — nothing this session touched in those files, so no + correctness impact, but a fresh `cmake` reconfigure (e.g. after + editing `runtime.cmake` itself, or on a clean clone) would have + dropped rollback netplay from the link entirely. Restored the six + lines. + + **Post-A/B soak (2026-08-01) — "never-ending storm" traced to a + round-start input-density burst, not a regression.** Full match + soak with the `pads_differ` gate + pegged-streak off-guard + batched + depth24 present: **zero core diverges, zero aborts**, both peers' + episode lists byte-identical (34 commits, ticks 159 and + 1006..1400), full recovery to a sustained 60.0 fps for 450+ ticks + until voluntary disconnect (`sdl_window_close` / `peer_disconnect`, + not a stall). ICE selected `typ host` on both sides (confirmed LAN, + not TURN) so link latency is ruled out. All 34 episodes land in one + contiguous window starting the tick *right after* `FMV lockstep + RELEASE sim=939` hands control back to normal 2-tick-delay + prediction — i.e. the moment the round actually starts and both + controllers begin moving/attacking at once. Timesync debt during + that window stayed tiny (1–8 ms, cap is ~34 ms) and the pacer never + neared the cap, so this is not a phase-skew problem the pacer could + close faster — it is genuine simultaneous real input density from + two controllers exceeding what a 2-tick delay can predict, and it + self-resolves once the opening exchange calms down (~30 real + seconds at 40–70 fps here). Distinct from earlier storms: no + cascade, no divergence, no re-arming after it clears. Also + reconfirmed the pre-existing FMV admit cost: 11–15 ms/f *admit* + (network/lockstep wait) during FMV playback vs 6–11 ms/f guest + (render, post Fix B) — the FMV frame-rate ceiling is now the + lockstep admit wait, not GPU present cost. Next candidates, not yet + tried: (a) `PSX_NET_DELAY=3` to cut mispredict frequency in exactly + these high-density round-start windows (flagged 2026-07-31, never + soaked); (b) shrink FMV lockstep per-tick admit wait now that + render cost is off the table. + + **Residual gameplay determinism fork (open, 1 hit / 4000 ticks):** + soak 2026-07-31 aborted once at sim=4034 (fight scene). Forensics: + identical baseline (epoch=40 load=4033 core=fc779568 both peers), + identical sealed pads (s0=ffff s1=ffbf), identical fin cycle count + (2277677049), and clk/tim/av/cd/dma/spad/sio/aux all matched — + only **cpu** (50d03880 vs 01ccbeaf) and **ram** (88a8e19f vs + 58f7e8de) forked. So clocks, IRQ timing, icache, and devices were + bit-identical and the GPRs/RAM still diverged — a genuine + non-clock replay fork (GTE state? uninitialised read? resim-order + RAM write?). Needs a repro before digging; realign recovered it + in-band (cooldown, no cascade). + + **Char-select storm = frozen agreed watermark (fixed):** with the + 30 ms grace, menu D-pad was clean (6 late edges all soak, was 29) + — but char-select stormed with the peers' live digs IDENTICAL + through the whole window. Cause: after a tip-hold commit set + `g_agreed_valid`, the hard watermark only advanced at episode + commits; hundreds of hc-confirmed clean live ticks were ignored, + so every new edge reloaded from an ancient snap (soak: + `mismatch=2135 load=1361`) and SPAN CAP crawled 24 ticks per + episode while Live galloped ahead — permanently behind. Fix: + `advance_agreed_watermark_from_hc()` — outside episodes/TipHold/ + pending loads, raise agreed to the newest hc-confirmed interval + snap in the ring (span_lo collapses to the new watermark); called + from `choose_load_tick` and the follower REFUSE gate (the follower + must advance too or the initiator's valid frontier loads get + NACKed). The historical false-confirm hazard behind the hard cap + (TipHold Live invent FRAME_COMMITs with cleared PCs) is + structurally gone — TipHold Live emits no FRAME_COMMITs and the + core digest folds full CPU split + csv + icache. Edges now load + from ~one interval back and replay a handful of ticks (light + tips), like the main-menu path. + + **Rematch black screen (fixed):** second lobby launch armed + invent-lockstep at `sim=0` with `FMV media … depth24=0 mdec=0 xa=0 + fmv_pend=1 cd_reading=1 mode=e0 present_pend=1` and never printed + another FPS line. Cause: `rb_fmv_media_active` treated + `cdrom_fmv_stream_pending()` (`reading && mode&0x48`) as media — + MotK boot/LoadExe sets XA mode bits on ordinary CD reads, so + rematch locked invent before the first frame. Also soft-exit left + `s_present_pending` / flush reentrancy sticky (`gpu_init` does not + clear them). Fix: media = depth24 | mdec hysteresis | real XA + stream only; `present_session_reset` clears deferred present; + rb shutdown clears sticky BB PC + agreed watermark. + + **Episode cost cut: redundant full-bus digests off the per-tick hot + path (2026-08-01).** `delay=3` and `delay=4` soaks showed the same + "chain of small, back-to-back episodes" storm shape as `delay=2` — + increasing the prediction buffer didn't help, so the fix target moved + from network/prediction tuning to what a replayed tick actually + *costs*. Found: `log_resim_tick_audit()`'s `"fin"` call (fired once + per replayed tick, i.e. every tick of every episode, not just the + last) unconditionally computed the *full* bus digest breakdown — + `netplay_core_digest_parts` (a full 2 MiB RDRAM CRC plus GPR/COP0/GTE/ + clock/timer CRCs), a full 1 MiB VRAM CRC (`netplay_av_digest`), plus + CD-ROM/aux/scratchpad/DMA/SIO digests — purely to print it. Worse, on + the *last* tick of every episode, `enter_verify_at_tip()` then + recomputed core/aux/av **again**, independently, for the real POST + digest sent over the wire — so the commit tick paid for two full + sweeps back to back. For a storm with dozens of 3–5 tick episodes + this was megabytes of pure CRC32 work landing on exactly the ticks + already busy re-simulating, compounding guest ms/frame during the + exact window players felt as "stormy." + Fix (no protocol/behavior change, diagnostics only): + `log_resim_tick_audit()` now always takes the cheap path (register + + clock/timer + RAM core digest only, same as it already did for the + `"arm"` tag) for every replayed tick, `"fin"` included — the per-tick + line still shows core/sealed-pad state for triage, just not the full + breakdown. The full breakdown moved *into* `enter_verify_at_tip()` + itself, computed exactly once, from the exact `dig_cpu` used for the + real POST digest, and reused for both diagnostic prints (`rb post + parts`, `rb post sent`) and the wire send — eliminating the double + sweep on every commit. Net effect: full-bus digest cost drops from + "every replayed tick, twice on the last one" to "once per episode." + Also added a `replay=NN%%` field to the `PSX_NETPLAY_TIMING=1` `[FPS]` + line (`psx_netplay_rb_take_replay_ticks()`) so a soak can directly + confirm what fraction of the window was spent resimulating instead of + inferring it from episode density in the raw log. Considered a + `PSX_RB_VERBOSE_AUDIT` opt-out flag for the remaining once-per-episode + full audit print, but skipped it: at episode-count frequency (not + tick-count) the residual cost is already small, and every prior + determinism bug in this doc (I-cache tags, csv, the Δ8-cycle fork) + was found *from* this exact print, so defaulting it off risked + trading a small, now-rare cost for reduced forensics on the next one. + Not yet re-soaked; next step is confirming this shrinks the storm's + guest ms/f without changing episode count/shape. + + **Re-soak (2026-08-01): confirmed genuine, self-clearing, input-density + storm — plus two follow-up ideas investigated, one landed, one + shelved.** `replay=NN%%` (new telemetry above) confirmed 50-60% of + simulated ticks were spent in Replay during bursts of rapid main-menu + D-pad taps, dropping to 0% between bursts — bursty and self-clearing, + not a runaway cascade (zero core diverges/aborts across two full + logs). Root-caused every rewind to a genuine Up/Down press/release + edge (`wire rewind-request ... pub=ffff wire=ffbf` etc.) — confirmed + as *inherent* to the current contract: `np_try_admit_rollback()` + invents via hold-last **immediately, with zero wait**, whenever the + local sim is ≤1 tick ahead of the remote's last-confirmed tip (the + normal steady-state operating point on a fast LAN) — raising + `delay`/`P` does not change this, since the code never even checks + for the real value before inventing at that gap. A discrete edge is + therefore always wrong under hold-last and always forces a rewind + per the "button deltas always rewind" contract, independent of + delay/prediction settings — investigated and shelved, not worth + testing further. + Also investigated **one-sided episodes** (let the follower answer + POST from already-recorded live history instead of doing a full + snapshot-load + replay, when its own simulation was never the one + that mispredicted). Found a real obstacle: the live per-tick history + ring (`s_part_ring` in `psx_netplay.c`) only refreshes VRAM/CD/aux + every 32 ticks (`np_emit_frame_commit`'s `crumb` gate) specifically + *to avoid* a permanent per-frame VRAM-CRC cost — so answering POST + correctly for an arbitrary mismatched tick from stored history would + need that data fresh every tick, trading a rare, bursty, ~50ms/f + spike for a permanent ~1-2ms/f tax on every frame of ordinary play. + Not a free win; shelved pending a cheaper way to get exact per-tick + VRAM state (e.g. dirty-region VRAM CRC instead of full-frame). + **What did land:** found `accept_peer_baseline()` unconditionally + `fprintf`+`fflush`(stderr) on *every* received baseline packet, + including every redundant copy of `RB_BASELINE_BURST=8` UDP + retransmits sent from up to ~9 call sites across an episode's + lifecycle — soak logs showed ~25 identical "rb peer baseline" lines + (2932 across 117 episodes in one log) for state that only actually + changes once per episode. Each `fflush` is a blocking syscall, firing + repeatedly right on the ticks already busy resimulating. Fixed: + the accepted state (`g_peer_baseline_ok`/digest/av/aux/ready) still + updates unconditionally on every copy (protocol behavior unchanged), + but the diagnostic print is now gated on the (epoch, load, dig_m, + dig_a, dig_b, dig_c) tuple actually changing since the last log — + pure I/O-volume reduction, zero effect on decision logic. Not yet + re-soaked. + + **Engine-level digest cost (2026-08-01): Tier 1 landed, Tiers 2/3 + planned.** The above cuts were all "stop doing expensive work more + often than needed" within the netplay layer. A separate axis is + "make the expensive work itself cheaper" at the engine level, + independent of rollback protocol logic — `netplay_av_digest()` CRCs + the full 1 MiB VRAM buffer and `netplay_core_digest_parts()` CRCs the + full 2 MiB RAM buffer from scratch every time either is called, with + no dirty-region shortcut. Three tiers identified, ordered by + risk/complexity: + + **Tier 1 — faster CRC32 (landed).** `runtime/src/crc32.c` used the + textbook byte-at-a-time Sarwate table method: one table lookup per + byte, each depending on the previous byte's output CRC (a serial + dependency chain that stops the CPU overlapping consecutive steps). + Replaced with **slicing-by-8**: 8 of those serially-dependent steps + are restated as 8 *independent* table lookups (each depends only on + an input byte) XORed together, letting the CPU run them + out-of-order. Pure implementation swap — same polynomial, same + bytes in, bit-identical output — verified by fuzzing the new + implementation against the old one over 27k+ cases (exhaustive + small lengths 0-40 from 32 offsets × 2 starting-CRC values, 20k + random large buffers/offsets/starting-CRCs, 5k chained/incremental + folds matching how `netplay_state_digest.c` folds `core`/`aux` + together) before replacing it, since a subtle CRC bug here would be + exactly the class of silent-corruption bug this doc has spent months + chasing. Measured ~5x faster on both VRAM (1 MiB: 1.90ms → 0.38ms) + and RAM (2 MiB: 3.78ms → 0.76ms) sized buffers on the dev host. No + protocol/wire-format implication — the digest is an opaque + comparison value never compared across differently-built peers, and + both peers already have to run the identical binary. + + **Tier 2 — dirty-region digest (planned, not started).** Convert + "CRC the whole buffer" into "CRC only what changed since the last + digest." **VRAM is the tractable target**: audited every write to + the 1 MiB `vram[]` array in `gpu.c` and found only **3 call sites** + in the whole GPU emulation (CPU→VRAM transfer ~L2071, pixel + plot/fill/line/poly ~L2930, depth24/16bpp write ~L4744) — a small, + fully auditable surface where each site could mark a per-scanline + or per-tile "touched" bitmap, and `netplay_av_digest()` would CRC + only touched regions since the last checkpoint. This is also the + cheaper alternative flagged above as the unblocker for "one-sided + episodes" (exact per-tick VRAM state without a permanent per-frame + CRC tax). **RAM is not a good near-term target the same way**: guest + stores happen from essentially every recompiled `sw`/`sh`/`sb` site + across the whole program — thousands of call sites, not 3 — so + reliable RAM dirty-tracking means threading write-tracking through + the recompiler/interpreter's store path itself, a much larger + change. (Note: `dirty_ram_interp.h`'s existing per-page dirty + bitmap is an unrelated subsystem — self-modifying-code detection for + the static recompiler, CLAUDE.md Rule 18 — not a digest optimization + and not reusable for this without its own risk analysis.) **Risk to + respect:** a missed write site doesn't crash, it silently produces a + digest that doesn't reflect true state — masking a real divergence + or manufacturing a phantom one. Should ship with a "shadow-verify + against a full recompute" mode during bring-up, not on faith. + + **Tier 3 — Merkle/hash-tree digest (planned, longer-horizon idea).** + Replace the flat CRC with fixed-size blocks (e.g. 4 KiB) each hashed, + plus a tree of hashes-of-hashes up to a root digest; a write + invalidates only its own block's hash and that hash's ancestors + (same underlying tracking problem as Tier 2, organized as a tree + instead of a flat bitmap). Beyond the performance win, this buys + **free divergence localization**: every forensic hunt in this doc + (I-cache tags, the Δ8-cycle fork, the open cpu/ram-forked-but- + everything-else-matched case) currently starts from "two 32-bit + CRCs disagree" and requires manually narrowing via partition + digests / sub-CRCs built up over sessions. A hash tree turns that + narrowing into an O(log N) walk from the two disagreeing roots down + to the differing leaf. Bigger lift than Tier 2; revisit after Tier 2 + ships and its audit tooling exists to reuse. - [x] FMV / depth24: defer rewind (promote wire only); follow NACKs; RB snap load uses light frontend hook (no FMV cutover/present thrash) - [x] Multitap / N-slot: history + seal tables sized to `slot_count` @@ -418,3 +970,438 @@ RTT (BattleShip tiers / phase_lock); optional Manual Input Delay / Prediction. Reference: `lib/recomp-net/docs/rollback.md`, `include/recomp_net/rollback.h`, `include/recomp_net/input_contract.h`, `tests/rollback_episode_test.c`. + +--- + +## 12. Portable policy: size "wait before invent" budgets from RTT, not local tick cadence + +Found on MotK (2026-08-01), but the mistake is generic to any rollback +netcode with a "stall briefly, hoping the real remote row arrives, before +falling back to prediction" grace mechanism — worth checking for on every +future engine port before it ships. Written up here so it travels with the +rollback playbook rather than staying MotK-specific tribal knowledge. + +**The mistake.** MotK's `np_invent_grace_stall()` (`runtime/src/psx_netplay.c`) +decides how long to block waiting for a genuinely-late remote row before +giving up and hold-last-inventing it. Its budget was `max(floor_ms, +2.5 × measured_local_tick_period)`, capped at 100ms — i.e. it used an EMA of +**how long this peer's own ticks have recently been taking** as a proxy for +"how patient should I be." That metric is the wrong one to feed a stall +gate: local tick period gets *longer* precisely when the engine is already +struggling (an active resim burst, a heavy scene, GC/OS jitter) — none of +which has anything to do with how long the *network* actually needs. The +result is a stall whose size floats up right when the system is already +behind, adding pure idle wait on top of the very slowdown it's reacting to. +Confirmed on a genuine direct-LAN connection (ICE selected `typ host` +candidates both sides, sub-ms RTT by any reasonable estimate): the +"grace" stall was measured landing at 70-90ms per occurrence — sized as if +this were a lossy WAN link, not because the link needed it, but because the +formula was reading the engine's own local distress as the loneliness of a +lonely remote packet. + +**Why it partially masqueraded as "working as intended."** Lowering the +env-configured floor from 30ms to 5ms *did* measurably help (a same-scene +soak comparison showed replay-time share dropping ~31%→~25% and total +rewind episodes dropping ~113→~91 over an equal-length session) — so the +mechanism isn't useless, and the fix isn't "delete it." But the improvement +was partial and the storms remained *intermittent* rather than +disappearing, because the `2.5×` scaling term overrides a low floor the +moment tick cadence degrades even slightly — exactly the condition a live +storm creates. A fix that only tunes the floor constant will always leave +this residual: the floor is not the part doing the damage during a burst, +the scaling term is. + +**The general policy for the next engine.** When a rollback implementation +needs a "how long do I wait for a late remote row before I predict it" +budget: + +1. **Size it from measured network RTT, not from local simulation cadence.** + Most rollback netcodes already track RTT for something else (auto + delay/prediction tiering, connection-quality UI, etc.) — reuse that + signal. A LAN connection with sub-ms RTT should get a budget in the + low single-digit milliseconds; a WAN connection with 40ms RTT + legitimately needs more. Local tick period answers "how is my engine + doing", not "how is the link doing", and a wait-for-network gate should + only ever ask the second question. +2. **Audit every adaptive stall/pacing input for the feedback question:** + *"If this stall makes my own performance worse, does that get fed back + into a bigger version of the same stall?"* Any signal derived from + wall-clock frame/tick timing on the *same host that is currently + stalling* is a candidate for this trap. It doesn't have to be obvious — + here it was buried inside an otherwise-reasonable "scale grace to the + observed tick rate so slow scenes don't false-positive" rationale that + made sense in isolation but interacted badly with the stall's own + side effect on that same tick rate. +3. **Keep "wait for late data" (invent grace) and "close phase skew" + (timesync pacing/advantage throttle — MotK's `np_timesync_throttle`) + as separate mechanisms with separate inputs**, even though both are + "make the admit loop briefly wait." They answer different questions + (is the data late vs. am I running ahead of my peer) and conflating + their signals — e.g. letting both key off the same local-tick-cadence + EMA — multiplies the risk of the feedback trap in (2) instead of + isolating it to one place that's easy to reason about and cap. +4. **Give the mechanism a real ceiling independent of any adaptive term**, + and log both the configured floor and the actual applied budget + separately, so a soak can tell "the operator's floor setting" apart + from "what the scaling term inflated it to" — this distinction is what + made the MotK case diagnosable at all (see `psxrecomp: rb invent grace + N ms` at startup vs. the per-stall budget computed in + `np_invent_grace_stall`, which never prints and had to be reconstructed + from the tick-period EMA logged by the neighboring `np_timesync_throttle` + print). +5. **Validate any fix by holding episode/rewind count and correctness + (zero `post core/aux diverge` aborts) constant while comparing perceived + latency/`replay%`** — the goal is moving the budget's *source signal*, + not just its magnitude; a naive floor-only tune (as done here first, for + speed) is a legitimate stop-gap but should be labeled as partial, not + as the fix, in whatever tracking doc records it. + +**Status for MotK itself: implemented (2026-08-01), not yet re-soaked.** + +- [x] Replace `np_invent_grace_stall`'s `2.5 × local_tick_ema` scaling term + with a budget derived from measured session RTT, capped low. No + existing live RTT signal existed anywhere in the netcode (checked + `RNetSessionStats`, the ICE layer, and the lobby's one-shot pre-match + LAN probe — none fit: the first two don't exist, the third measures a + LAN-list endpoint that isn't necessarily this match's peer and doesn't + update during play). Rather than add a wire-protocol change (new + ping/pong message) or thread a value through the recomp-ui lobby UI + (out of scope, and that submodule has its own in-flight uncommitted + netplay-launch refactor — touching the same structs risked colliding + with it), sourced RTT for free from data already being timestamped: + `enter_verify_at_tip()` (`psx_netplay_rb.c`) already records + `g_verify_wait_ms` when it sends this peer's POST, for the existing + verify-timeout mechanism. Added `g_rb_rtt_ema_ms`, sampled at the + point the peer's POST is accepted — `now - g_verify_wait_ms` when we + sent ours first (the common case) is dominated by one-way transit of + the peer's packet, not local compute, since both sides enter Verify + around the same real-world moment (triggered by the same + mismatch/rewind-request). EMA'd (3:1), sanity-bounded (samples over + 2s discarded as stale/bogus), exposed via + `psx_netplay_rb_rtt_estimate_ms()`, reset to 0 on session + start/shutdown (rematch) alongside the rest of the per-session RB + state. This piggybacks on an existing, already-occurring exchange — + no new wire message, no protocol version concern. `np_invent_grace_stall` + now sizes its budget as `1.5 × rtt` once a sample exists (hard-capped + at 60ms regardless of RTT), falling back to the old tick-cadence term + (now 1.5 ticks / 40ms cap, down from 2.5 ticks / 100ms) only in the + brief startup window before the first episode has round-tripped. The + env-configured floor (`PSX_RB_INVENT_GRACE_MS`) default also dropped + 30→8ms, since the soak showed 5ms already worked and the floor now + matters far less than the (fixed) scaling term. +- [x] Log the applied per-stall budget directly, not just the configured + floor: `psxrecomp: rb invent grace budget=N ms (floor=F rtt=R + tick_ema=T) slot=S wire=W`, printed once per tracked wire tick + (deduped so a multi-ms stall doesn't spam one line per retry). +- [x] Re-soak after RTT-sourced invent grace (2026-08-01): steady-state + improved (`replay%` ~7% overall / ~3.6% quiet post-FMV; invent budgets + 8–22 ms). Remaining cliffs were **not** invent-grace — they were the + Verify POST-loss / tip-extend NACK storm in §13. Note: sampled "rtt" + reads ~15–26 ms on direct LAN because the POST handshake wait includes + peer Verify compute asymmetry, not pure UDP transit — still a better + budget signal than local tick cadence, but not a true ping. + +Touched: `runtime/src/psx_netplay_rb.c` (`g_rb_rtt_ema_ms`, sampling in the +peer-POST-accepted path, `psx_netplay_rb_rtt_estimate_ms()`, reset in +`psx_netplay_rb_shutdown()`), `runtime/include/psx_netplay_rb.h` (new +declaration), `runtime/src/psx_netplay.c` (`np_invent_grace_stall()` +budget formula + floor default + applied-budget log line). Builds clean, +no new lints. `np_timesync_throttle` intentionally left untouched per +policy point 3 above (separate mechanism, already has its own small fixed +cap and adaptive-off). + +--- + +## 13. Verify POST loss → unilateral tip-hold → tip-extend NACK storm (2026-08-01) + +**Symptoms (RTT-budget soak):** long quiet stretches at 60 fps / `replay≈0%`, +punctuated by cliffs (`8.1` then `2.7` fps, `admit=108–351 ms`). Not invent- +grace — budgets stayed at `22 ms (rtt=15)`. + +**Root cause (matched-pair logs at tip=1256 and tip=1437):** + +1. Both peers finish Replay and send identical POSTs. +2. Peer A receives B's POST first → `enter_tip_hold` → advances + `agreed_through` to the tip → **stops retransmitting its own POST** + (rexmit gate was `phase==Verify && local_post && !peer_post_ok`). +3. Peer B never got A's first POST (single UDP loss, even on direct LAN + `typ host`) → sits in Verify until `RB_VERIFY_TIMEOUT_MS` (4s) → + `ABORT — verify timeout (peer POST missing)` → realigns to last + *confirmed* tip (e.g. 1425). +4. `RB_RESOLVED` that A sent on tip-hold was ignored for Verify exit — + `take_rb_resolved` only called `rnet_rb_set_peer_convergence`. +5. A then opens tip-extend / fresh episodes with `load=` the unconfirmed tip + (1437). B refuses: `follow REFUSED … past frontier=1425` + NACK. +6. A's NACK handler was plain `abort_episode` — **no watermark demotion, + no cooldown** — so A immediately reopened `load=1437` → epochs 7..11 + back-to-back NACK abort storm (the `2.7 fps` cliff). + +**Fixes landed:** + +- [x] On `enter_tip_hold`, burst `RB_POST_BURST` (8) copies of local POST + + RESOLVED *before* `clear_post_handshake`, so the lagging peer can still + complete Verify after the winner leaves it. +- [x] On `take_rb_resolved` while stuck in Verify with a matching + `g_post_target`, accept RESOLVED as the missing handshake half and + `enter_tip_hold` (log: `rb verify accept peer RESOLVED`). +- [x] On peer follow-NACK: demote `agreed_through` below the refused load, + drop a pin at/above that load, `schedule_live_realign` to a snap at + the demoted tip, and `arm_rewind_cooldown_ticks` so tip-extend cannot + immediately reopen the same unilateral tip. + +**Re-soak checks:** no `verify timeout (peer POST missing)` on LAN; if one +still appears, the lagging peer should show `verify accept peer RESOLVED` +instead of a 4s hang; NACK lines should be followed by realign + cooldown, +not `begin epoch=N+1` with the same refused `load=` within milliseconds. + +--- + +## 14. TipHold coalesce was pump-spin, not sim/wall time (2026-08-01) + +**Symptoms (full-match soak after §13):** zero verify timeouts / NACKs / +aborts, avg ~57 fps — but **351** rewind-requests / **350** fresh light +episodes (~33 per 1k frames), avg `replay%` ~25%, and **0 tip-extends**. +Every press/release edge (gaps of 3–7 sim ticks) opened a new episode +immediately after tip-hold committed. + +**Root cause:** MotK uses `tip_seal_slack = 0`, so Live invent stalls at the +sealed tip during TipHold. Finalize counted **admit-pump iterations** while +`sim >= tip`. The admit loop spins with `wait_recv(1)`, so "24 quiet frames" +elapsed in a few milliseconds of wall time — long before the paired release +edge arrived — then tip-hold committed and the next edge opened epoch N+1. + +**Fixes landed:** + +- [x] Replace pump-frame quiet counter with wall-clock + `g_tip_hold_quiet_t0_ms`: finalize after `runway` frames at 60 Hz + (`runway * 1000 / 60` ms, clamped 80–500). Tip-extend resets the timer + so an active coalesce keeps the episode open. +- [x] TipHold admit: still never *invent* past tip+slack, but **do** advance + when every remote wire row is already present — so tip-hold is not a + hard freeze for the whole coalesce window, and sim-time finalize + (`sim > tip_hold_until`) can fire when confirmed inputs walk forward. + +**Re-soak checks:** `rb tip-extend epoch=` / `tip-extend FOLLOW` should +appear for press→release pairs; rewind-request count and avg `replay%` +should drop vs the 351 / 25% baseline; tip-hold→commit gaps should be +tens–hundreds of ms, not adjacent log lines. + +--- + +## 15. Light-tip depth ceiling vs TipHold coalesce runway (2026-08-01) + +**Post-§14 soak (LAN, menu nav, `delay=4`):** genuine improvement — +avg 59.1 fps, avg `replay%` 17% (down from ~25%), 91 rewinds / 6799 frames +(~13.4/1k, down from ~32.7/1k). But `tip-extend` fired only **once** in the +whole soak: the presses in this soak were spaced ~800 ms apart on average +(`avg gap=47` ticks @ 60 Hz), wider than the 24-tick (~400 ms) TipHold +coalesce window, so §14 rarely got a second edge to merge. The count/cost +drop here is mostly attributable to Tier‑1 CRC + the RTT-sized invent-grace +fix (§12), not coalescing — worth re-checking during dense-input gameplay +(combo strings), not just menu nav, where coalescing should matter more. + +Every remaining rewind-request carries a real `pub` vs `wire` bit diff (a +genuine opponent input change), confirming these are not spurious — under a +hold-last/idle-invent predictor a real remote button transition necessarily +mispredicts. ICE also confirmed `typ host` on both sides (not relayed +through the coturn TURN server despite `force_turn=1` in the startup line, +which only affects candidate *gathering*, not which candidate wins), with a +genuine ~10–13 ms measured latency — this is real direct-LAN behavior, not a +transport misconfiguration. + +**New finding — the two remaining depth knobs had drifted apart:** +`RNET_RB_LIGHT_TIP_MAX_DEPTH` (`recomp-net/include/recomp_net/rollback.h`) +is a library-wide constant of **16** ticks: episodes at or under that depth +skip the pre-Replay ready-ACK round trip (`rnet_rb_is_light_tip_candidate`). +§14 widened MotK's TipHold coalesce runway (`RB_MOTK_TIP_RUNWAY`) to **24** +so it could keep absorbing edges longer — but nothing widened the light-tip +ceiling to match. Depth-bucketing every `light=` episode in the soak showed +a perfectly clean split: + +``` +light=1 depths: 1,2,3 ... 16 (73 episodes, all <= 16) +light=0 depths: 17 (x5), 24 (x12) (17 episodes, all > 16) +``` + +**17 of 90 episodes (19%) in this soak** grew past 16 ticks purely from +coalescing/span-cap headroom and lost light-tip eligibility, paying an +extra ready-ACK round trip on top of the POST-verify round trip both paths +already pay — and these are also the *deepest* rewinds, i.e. already the +most visually disruptive ones, now also getting the most expensive +handshake. + +**Fix landed:** made the light-tip depth ceiling a per-session config field +instead of a single hardcoded constant, and set MotK's session to match its +TipHold runway: + +- [x] `recomp-net`: added `RNetRbConfig.light_tip_max_depth` (0 → library + default `RNET_RB_LIGHT_TIP_MAX_DEPTH`, clamped to 32 like + `tip_runway`). Added `rnet_rb_get_light_tip_max_depth()` and + `rnet_rb_is_light_tip_candidate_ex(load, target, resolved_through, + max_depth)` (explicit-ceiling sibling of the existing + `rnet_rb_is_light_tip_candidate`, which now just calls `_ex` with the + library default so existing callers/tests are unaffected). + `rnet_rb_recommend_light_tip` and `rnet_rb_begin_episode` now read + `cfg.light_tip_max_depth` from the session instead of the bare + constant. New unit tests in `rollback_episode_test.c` cover: default + resolves to the constant, `_ex` with an explicit wider ceiling accepts + a depth the plain wrapper rejects, and a session created with + `light_tip_max_depth=24` correctly flags a depth-20 `begin_episode` as + light (would not with the untouched default). +- [x] `psx_netplay_rb.c`: `psx_netplay_rb_start()` sets + `cfg.light_tip_max_depth = RB_MOTK_TIP_RUNWAY` (24) so the two + thresholds can't drift apart again. Both direct pre-flight call sites + (initiator + follower `begin_episode` paths) switched from + `rnet_rb_is_light_tip_candidate` to `rnet_rb_is_light_tip_candidate_ex` + with `rnet_rb_get_light_tip_max_depth(g_rb)` so the precomputed flag + matches what `rnet_rb_begin_episode` will compute internally. +- [x] `recomp-net` is a submodule (`psxrecomp/lib/recomp-net`, separate + clone from the standalone `recomp-net` workspace repo on the same + `feat/rollback` branch) — kept both copies of the touched files + (`include/recomp_net/rollback.h`, `src/rollback/rnet_rollback.c`, + `tests/rollback_episode_test.c`, `docs/rollback.md`) byte-identical. + +**Not changed (deliberately):** did not just bump the library constant +`RNET_RB_LIGHT_TIP_MAX_DEPTH` itself — it's shared across every recomp-net +consumer, not just MotK, so widening it globally would be a cross-project +policy change, not a MotK tuning knob. The per-session `cfg` field lets +MotK opt in without affecting other hosts that still want the conservative +16-tick default. + +**Re-soak checks:** `light=0` should now only appear for episodes that hit +`SPAN CAP` (depth > `RB_MOTK_TIP_RUNWAY`), not for every coalesced episode +past depth 16; the `light=1`/`light=0` split should track `RB_MOTK_TIP_RUNWAY` +(24) instead of the old library default (16). Re-run a soak with denser +input (active gameplay, not just menu nav) to see whether §14's coalescing +actually engages more (`tip-extend` count > 1) once presses land closer +together than the ~800 ms average seen here — that's the next real lever if +rewind volume is still the complaint, since the per-episode cost items +(Tier 1 CRC, RTT-sized invent grace, this light-tip alignment) are now +largely exhausted. After that, Tier 2 (dirty-region digest, documented +above) is the next compute-cost lever, and it disproportionately helps +exactly the deep/full-verify episodes since they still walk the full +2 MiB RAM + 1 MiB VRAM CRC at Verify regardless of light/full status. + +--- + +## 16. Follow-NACK realign fork + HC re-poison + asymmetric light-tip (2026-08-01) + +**Post-§15 soak (LAN, menu nav, `delay=4`):** §15 worked — **44/44** episodes +`light=1` including depths 17 and 24; avg `replay%` 13% (down from 17%); +~8.9 rewinds/1k frames. Then a late cliff: host `4.0 fps admit=235 ms`, +peer `5.9 fps`, `ready timeout (initiator never sent GO)`, `resim core +diverge sim=3841`, and peer `follow NACK load=3856` / `promote-no-resim +reason=cooldown` spam. + +**Root cause chain (matched-pair at tip=3840):** + +1. Both peers commit through **3840** (matched POST). +2. Peer HC-advances `agreed 3840→3856` and opens `epoch=59 load=3856`. +3. Host still has frontier **3840** → `follow REFUSED … past frontier=3840` + + NACK. (The follow-up `snap missing` line from `send_follow_nack` is + misleading — refuse reason was the frontier check.) +4. Peer's NACK handler demoted the watermark **and always + `schedule_live_realign(demote)`**, even though the refused load snap was + never applied (still in SealInputs). That rewound ~25 ticks of good + matched Live on only the peer → silent core fork at demote+1, while + host kept running Live ahead. +5. Cooldown after that realign made late wire `promote-no-resim` — peer + could not correct invents during catch-up. +6. Peer then immediately re-ran `agreed ADVANCE 3840→3856` from **stale + HC** (demote did not `netplay_hc_prime_after` / demote library + `resolved_through`) — same poison, next episode. +7. Host's agreed stayed stuck at 3840 for ~229 ticks (peer digests no + longer confirmed). Host finally opened + `SPAN CAP mismatch=4069 load=3840 target→3864` as **light=1** and + skipped ready-ACK without emitting GO. +8. Peer followed `load=3840` but its `resolved_through` was 3856 → + **not** light → waited for initiator GO → `ready timeout` after 4s + (the admit cliff). Host aborted with `resim core diverge sim=3841` + while in Verify — leftover of the earlier Live fork. + +**Fixes landed:** + +- [x] **NACK keep-live when snap never applied:** capture + `g_episode_snap_applied` before `abort_episode`; only + `schedule_live_realign(demote)` when the refused load was actually + applied. Otherwise log `NACK keep-live` and stay on the pre-episode + Live tip (symmetric with the NACK-sender, who never left Live). +- [x] **Demote + prime HC:** on NACK, demote `g_agreed_through` / + `g_agreed_span_lo`, call new `rnet_rb_demote_resolved_through(g_rb, + demote)` (library watermark survives `session_reset` and + `set_peer_convergence` only advances), and + `netplay_hc_prime_after(hc, demote)` so live hash_confirm cannot + immediately re-ADVANCE past the refused tip. +- [x] **Light-tip still emits ready/GO:** skip *waiting* for the RTT (the + light-tip win) but still `send_baseline_burst(1, …)` so a peer that + classified the same episode as non-light is not stranded until + `RB_READY_TIMEOUT_MS`. Log line: + `light-tip skip wait … (still emitted ready/GO for asymmetric peer)`. +- [x] Unit test for `rnet_rb_demote_resolved_through` (advance / demote / + no-op / survives `session_reset`). Synced into both the MotK + submodule and the standalone `recomp-net` checkout. + +**Re-soak checks:** after a `follow REFUSED … past frontier` / NACK, look +for `NACK keep-live` (not an immediate realign to demote) when the +initiator never applied the load snap; no immediate +`agreed ADVANCE demote→refused_load` on the next pump; no +`ready timeout (initiator never sent GO)` when one side light-skips; host +and peer live digests at the same sim should stay matched after a NACK +instead of forking for hundreds of ticks then SPAN-CAP from a stale +frontier. + +--- + +## 17. Tip-hold Live-walk held dpad → double menu inputs (2026-08-01) + +**Post-§16 soak (LAN, menu nav, `delay=4`):** §16 healthy — 0 aborts / +NACKs / ready-timeouts / promote-no-resim; ~61 fps; `replay%` ~17%. UX +still broken: **dpad press/release felt like double inputs**, `tip-extend` +stayed at **0**, and every tip-hold→commit pair was still adjacent +(runway consumed as Live, not as wait-for-release). + +**Root cause (not fake mismatches — every rewind was a real edge):** + +1. **Tip-hold Live-walked held digital.** After a press episode POST-matched, + tip-hold entered with `invent_slack=0` and `until = tip + 24`. Admit + advanced past invent-cap whenever remote wire was present — on a held + dpad that walked ~24 Live frames with the button still down, then + finalized via `sim > tip_hold_until`. MotK menus key-repeat on hold → + ~24 extra navigations from one physical press. The release arrived + *after* tip-hold already committed → second episode; coalesce never + saw it (`tip-extend=0`). +2. **Release episodes re-simulated several held frames** (often 6–10) + before the release tip — another held run after the tip-hold walk. +3. **Ghost second release:** after a real release commit, predicted hist + ahead still showed the button held (`pub=ffdf wire=ffff` again a few + dozen ticks later). Tip-hold enter/finalize also did not + `netplay_hc_prime_after(tip)`, so HC could re-`agreed ADVANCE` over + invent-poisoned ticks (soak: release tip=1607 then + `agreed ADVANCE 1607→1632`). + +**Fixes landed:** + +- [x] **Tip-hold invent-cap stall while digital held:** past `tip+slack`, + admit advances only when every remote wire row is present **and** + all pads (local + remote) are idle (`buttons == 0xFFFF`). Held → + stall so wall-clock quiet / peek-ahead coalesce owns the runway. +- [x] **HC prime on tip-hold enter + finalize:** + `netplay_hc_prime_after(hc, tip)` (+ `set_peer_convergence`) so + invent-hold FRAME_COMMITs past the tip cannot immediately re-ADVANCE. +- [x] **Scrub-ahead on digital release promote:** rewrite predicted hist + `release_tick+1..sim` to the released pad (`rb scrub-ahead release`). +- [x] **Tip-hold coalesce-ahead:** while tip-holding, peek wire + `tip+1..tip+runway` (no predicted hist required); on first pad delta + vs tip hist, promote the span and `tip_extend` (`rb tip-hold + coalesce-ahead`). TipHold tip-extend now rereplays when + `mismatch_tick > old_target` even if Live never left the tip + (`sim == old_target`). +- [x] Getter `psx_netplay_rb_tip_runway()` for the host scan. + +**Re-soak checks:** dpad tap should move the cursor once; look for +`tip-hold coalesce-ahead` / `tip-extend` on press→release within the +runway; `scrub-ahead release` after release promotes; no back-to-back +ghost `pub=held wire=ffff` release episodes; no +`agreed ADVANCE tip→tip+N` immediately after tip-hold enter/commit from +stale HC; tip-hold→commit should often be non-adjacent when a release +extends the tip. diff --git a/docs/ci/README.md b/docs/ci/README.md new file mode 100644 index 00000000..4cc4603c --- /dev/null +++ b/docs/ci/README.md @@ -0,0 +1,64 @@ +# CI helpers for setup-host releases + +Shared scripts and GitHub Actions so every PSXRecomp title uses one +standardized release flow. + +**Start here:** [`../GAME_PROJECT_SETUP.md`](../GAME_PROJECT_SETUP.md) + +## Workflow template + +```bash +mkdir -p .github/workflows +cp psxrecomp/docs/ci/templates/setup-release.yml .github/workflows/release.yml +# edit YOUR_* placeholders +``` + +Template: [`templates/setup-release.yml`](templates/setup-release.yml) + +## Tools under `psxrecomp/tools/` + +| Script | Role | +|--------|------| +| `ci/normalize_version.sh` | Normalize / write `VERSION` + `TAG` | +| `ci/clear_generated.sh` | Clear `generated/` for setup-host CI | +| `ci/record_pins.sh` | Log `psxrecomp` / `recomp-ui` / `recomp-net` SHAs | +| `ci/build_emitters.sh` | Build `psxrecomp-game` + `psxrecomp-bios` | +| `fetch_toolchain.sh` | Download/unpack cmake-clang-v1 (Windows emitter builds; optional embed) | +| `stage_setup_sdk.sh` | Emitters, OpenBIOS, optional `toolchain/`, MinGW DLLs | +| `bundle_mingw_dlls.sh` | Copy imported non-system DLLs next to Windows PEs | +| `package_setup_host.sh` | Lean setup-host zip (optional `--embed-toolchain`) | +| `../cmake/toolchain-mingw-w64.cmake` | Linux→Windows MinGW cross toolchain | +| `../host/psxrecomp_codegen_host.*` | Portable Generate & rebuild host (via CMake helper) | +| `templates/game.gitignore` | Suggested gitignore for title repos | + +Also use `psxrecomp_add_game_runtime(...)` in `runtime/runtime.cmake` for +setup-host / full-game wiring. The CLI (`psxrecomp_cli.py`) lives in this +submodule — there is no separate `psxrecomp-sdk/` overlay. + +## Composite actions + +```yaml +- uses: ./psxrecomp/.github/actions/build-emitters + +# Prefer package_setup_host.sh (allow-no-toolchain by default). +# fetch-toolchain + --embed-toolchain only for offline-first packs. +- uses: ./psxrecomp/.github/actions/stage-setup-sdk + with: + stage: dist/stage-setup-${{ matrix.artifact }} + recompiler-build: build-recompiler + allow-no-toolchain: 'true' + runtime-bin: /mingw64/bin # Windows / MSYS2 +``` + +## Title responsibilities + +Keep only this in the game repo: + +- Setup-host CMake flags and exe basename +- Thin `scripts/package_setup_release.sh` wrapping `package_setup_host.sh` +- Zip prefix / display name / disc hint in that wrapper +- Release notes / GitHub Release job naming + +## Release checklist + +See [`../GAME_PROJECT_SETUP.md`](../GAME_PROJECT_SETUP.md#bundled-release-checklist). diff --git a/docs/ci/setup-release.yml.example b/docs/ci/setup-release.yml.example new file mode 100644 index 00000000..169e5121 --- /dev/null +++ b/docs/ci/setup-release.yml.example @@ -0,0 +1,23 @@ +# Deprecated stub — use the full template: +# +# docs/ci/templates/setup-release.yml +# +# Install: +# cp psxrecomp/docs/ci/templates/setup-release.yml .github/workflows/release.yml +# +# Guide: docs/GAME_PROJECT_SETUP.md +# Actions: docs/ci/README.md + +name: Release builds (see docs/ci/templates/setup-release.yml) + +on: + workflow_dispatch: + +jobs: + redirect: + runs-on: ubuntu-24.04 + steps: + - run: | + echo "Copy psxrecomp/docs/ci/templates/setup-release.yml to .github/workflows/release.yml" + echo "See psxrecomp/docs/GAME_PROJECT_SETUP.md" + exit 1 diff --git a/docs/ci/templates/game.gitignore b/docs/ci/templates/game.gitignore new file mode 100644 index 00000000..17663300 --- /dev/null +++ b/docs/ci/templates/game.gitignore @@ -0,0 +1,63 @@ +# Suggested .gitignore for a PSXRecomp game repository. +# Copy to the game repo root and adjust title-specific disc dirs. + +# Build outputs +/build/ +/build-*/ +/out/ +/.happypath/ + +# Release packaging +/release-stage/ +/dist/ +*.zip + +# Local PlayStation assets (never commit) +/bios/ +/isos/ +/saves/ +/prepared_disc/ +# Title disc working trees (rename to match your layout, e.g. /bpe/ /motk/) +/disc/ + +# Disc / image / memcard formats +*.bin +*.cue +*.iso +*.img +*.sub +*.ccd +*.mcd +*.mcr +*.chd + +# Ghidra / local analysis +/ghidra/projects/ +*.gpr +*.rep/ +*.lock +*.lock~ + +# Editor / cache +.vs/ +.vscode/ +*.user +*.log +__pycache__/ + +# Runtime artifacts from psx-runtime launches +psx_crash.txt +psx_last_run_report.json +psx_screenshot.bmp +psx_freeze_heartbeat.json +starvation_dump.jsonl +disc.cfg + +# Diagnostic dumps (root-level) +/*.json +/*.bmp +/*.png + +# Setup-host titles: leave generated/ gitignored (users generate locally). +# Titles that still CI-build from committed game C may track generated/ instead. +/generated/ diff --git a/docs/ci/templates/setup-release.yml b/docs/ci/templates/setup-release.yml new file mode 100644 index 00000000..fc3e7ae4 --- /dev/null +++ b/docs/ci/templates/setup-release.yml @@ -0,0 +1,252 @@ +# Setup-host multi-platform release template for a PSXRecomp *game* repository. +# +# Install: +# mkdir -p .github/workflows +# cp psxrecomp/docs/ci/templates/setup-release.yml .github/workflows/release.yml +# Then replace every YOUR_* token. See docs/GAME_PROJECT_SETUP.md. +# +# Shared (do not fork these into the title): +# ./psxrecomp/.github/actions/{build-emitters,fetch-toolchain (optional embed),stage-setup-sdk} +# psxrecomp/tools/ci/*.sh +# psxrecomp/tools/{fetch_toolchain,stage_setup_sdk,bundle_mingw_dlls,package_setup_host}.sh +# psxrecomp/psxrecomp_cli.py (CLI lives in the submodule — no psxrecomp-sdk/) +# +# Title-owned: +# CMake setup-host flags, thin scripts/package_setup_release.sh wrapper + +name: Release builds + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g. 0.1.0 or v0.1.0)' + required: true + type: string + publish: + description: 'Create git tag vX.Y.Z and GitHub Release' + required: true + type: boolean + default: true + push: + tags: + - 'v*' + +concurrency: + group: yourgame-release-${{ github.event_name }}-${{ github.ref }}-${{ github.run_id }} + cancel-in-progress: true + +permissions: + contents: write + +env: + CMAKE_BUILD_TYPE: Release + +jobs: + prepare: + name: Resolve version + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.meta.outputs.version }} + tag: ${{ steps.meta.outputs.tag }} + publish: ${{ steps.meta.outputs.publish }} + steps: + - name: Normalize version / tag + id: meta + shell: bash + run: | + set -euo pipefail + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + RAW="${{ github.event.inputs.version }}" + PUBLISH="${{ github.event.inputs.publish }}" + else + RAW="${{ github.ref_name }}" + PUBLISH="true" + fi + RAW="$(printf '%s' "${RAW}" | tr -d '[:space:]')" + VER="${RAW#v}" + if [[ ! "${VER}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.+-][A-Za-z0-9.+-]*)?$ ]]; then + echo "::error::invalid version '${RAW}' (want X.Y.Z or vX.Y.Z)" + exit 1 + fi + echo "version=${VER}" >>"$GITHUB_OUTPUT" + echo "tag=v${VER}" >>"$GITHUB_OUTPUT" + echo "publish=${PUBLISH}" >>"$GITHUB_OUTPUT" + + build: + name: ${{ matrix.artifact }} + needs: prepare + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + artifact: linux-x64 + shell: bash + - os: windows-2022 + artifact: windows-x64 + shell: 'msys2 {0}' + - os: macos-15 + artifact: macos-arm64 + shell: bash + deployment_target: '11.0' + - os: macos-15-intel + artifact: macos-x64 + shell: bash + deployment_target: '11.0' + defaults: + run: + shell: ${{ matrix.shell }} + env: + # Rename to match your packager (e.g. BPE_RELEASE_VERSION). + RELEASE_VERSION: ${{ needs.prepare.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 1 + + - name: Pin VERSION + shell: bash + run: | + set -euo pipefail + printf '%s\n' "${RELEASE_VERSION}" >VERSION + echo "VERSION=$(cat VERSION)" + + - name: Record pinned framework revisions + shell: bash + run: bash psxrecomp/tools/ci/record_pins.sh + + - name: Setup host mode (no game C / BIOS backends in CI) + shell: bash + run: bash psxrecomp/tools/ci/clear_generated.sh + + - name: Setup MSYS2 (Windows) + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + mingw-w64-x86_64-toolchain + mingw-w64-x86_64-cmake + mingw-w64-x86_64-ninja + mingw-w64-x86_64-SDL2 + mingw-w64-x86_64-pkgconf + curl + zip + + - name: Install deps (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build pkg-config \ + libsdl2-dev libgl1-mesa-dev zip + + - name: Install deps (macOS) + if: runner.os == 'macOS' + run: | + brew install cmake ninja pkg-config sdl3 || brew install cmake ninja pkg-config sdl2 + + # Windows: build emitters with llvm-mingw from cmake-clang-v1 (static CLI). + # Zip still omits toolchain/ — fetch is for the build machine only. + - name: Fetch portable toolchain (Windows emitters) + if: runner.os == 'Windows' + uses: ./psxrecomp/.github/actions/fetch-toolchain + with: + artifact: ${{ matrix.artifact }} + + - name: Build recompiler tools (emitters) + uses: ./psxrecomp/.github/actions/build-emitters + + - name: Configure & build setup host + env: + CMAKE_OSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '' }} + run: | + set -euo pipefail + EXTRA=() + if [[ -n "${CMAKE_OSX_DEPLOYMENT_TARGET:-}" ]]; then + EXTRA+=("-DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") + fi + JOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" + # Prefer -DPSXRECOMP_FORCE_SETUP_HOST=ON (psxrecomp_add_game_runtime). + # Legacy -DYOURGAME_FORCE_SETUP_HOST=ON still works if the title wires it. + cmake -S . -B build-ci -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DPSXRECOMP_FORCE_SETUP_HOST=ON \ + -DPSX_GAME_VERSION="${RELEASE_VERSION}" \ + "${EXTRA[@]}" + cmake --build build-ci --target psx-runtime -j"${JOBS}" + + - name: Package setup zip + run: | + set -euo pipefail + chmod +x scripts/package_setup_release.sh + if [[ "${{ runner.os }}" == "Windows" ]]; then + export PSXRECOMP_RUNTIME_BIN_DIR=/mingw64/bin + export BPE_RUNTIME_BIN_DIR=/mingw64/bin + fi + export RELEASE_VERSION + # Lean zip (no embedded toolchain/). RetComM / wizard download + # cmake-clang-v1; pass --embed-toolchain only for offline-first packs. + scripts/package_setup_release.sh build-ci "${{ matrix.artifact }}" build-recompiler + ls -la dist/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + # YOUR_ZIP_PREFIX e.g. bpe, motk, yourgame + name: YOUR_ZIP_PREFIX-${{ matrix.artifact }} + path: dist/YOUR_ZIP_PREFIX-*-${{ matrix.artifact }}.zip + if-no-files-found: error + retention-days: 30 + + release: + name: GitHub Release + needs: [prepare, build] + if: needs.prepare.outputs.publish == 'true' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Flatten zips + run: | + set -euo pipefail + mkdir -p dist + find artifacts -type f -name 'YOUR_ZIP_PREFIX-*.zip' -exec cp -v {} dist/ \; + ls -la dist/ + + - name: Ensure release tag exists + if: github.event_name == 'workflow_dispatch' + env: + TAG: ${{ needs.prepare.outputs.tag }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if ! git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + git tag -a "${TAG}" -m "YOUR_GAME_TITLE ${TAG}" + fi + if ! git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + git push origin "refs/tags/${TAG}" + fi + + - name: Publish release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.prepare.outputs.tag }} + name: YOUR_GAME_TITLE ${{ needs.prepare.outputs.tag }} + files: | + dist/YOUR_ZIP_PREFIX-*.zip + generate_release_notes: true + fail_on_unmatched_files: true + target_commitish: ${{ github.sha }} diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c new file mode 100644 index 00000000..a9741e78 --- /dev/null +++ b/host/psxrecomp_codegen_host.c @@ -0,0 +1,1969 @@ +/* Portable generate → rebuild (--no-pgo from setup) → relaunch host. */ + +#include "psxrecomp_codegen_host.h" + +#include +#include +#include + +#if defined(_WIN32) +# include +#else +# include +# include +# include +# include +# include +# include +# include +extern char** environ; +#endif + +static const PsxrecompCodegenHostConfig* g_cfg; +static char g_project_root[1024]; +static char g_cli_path[1100]; +static char g_game_toml[1100]; +static char g_python[512]; +static char g_cmake[512]; +static char g_build_dir[1100]; +static char g_exe_path[1100]; +static char g_helper_path[1100]; +static char g_cmake_target[256]; +static char g_exe_basename[256]; +static char g_display[128]; +static char g_toolchain_bin[1400]; +/* Last ensure-toolchain JSONL result path (bin/); shared-cache fallback. */ +static char g_cli_toolchain_bin[1400]; +static int g_ready; +static int g_relaunch_is_helper; + +static const char* cfg_or(const char* v, const char* d) { + return (v && v[0]) ? v : d; +} + +static int path_is_file(const char* path) { +#if defined(_WIN32) + DWORD attr = GetFileAttributesA(path); + return attr != INVALID_FILE_ATTRIBUTES && + (attr & FILE_ATTRIBUTE_DIRECTORY) == 0; +#else + struct stat st; + return stat(path, &st) == 0 && S_ISREG(st.st_mode); +#endif +} + +static int path_is_dir(const char* path) { +#if defined(_WIN32) + DWORD attr = GetFileAttributesA(path); + return attr != INVALID_FILE_ATTRIBUTES && + (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; +#else + struct stat st; + return stat(path, &st) == 0 && S_ISDIR(st.st_mode); +#endif +} + +static int join_path(char* out, size_t cap, const char* a, const char* b) { + size_t na = strlen(a); + int need_slash = na > 0 && a[na - 1] != '/' && a[na - 1] != '\\'; + int n = snprintf(out, cap, "%s%s%s", a, need_slash ? "/" : "", b); + return n > 0 && (size_t)n < cap; +} + +#if defined(_WIN32) +/* Microsoft Store Python redirects %LOCALAPPDATA% writes into + * Packages\PythonSoftwareFoundation.Python.*\LocalCache\Local\... + * Map a virtual LocalAppData path to those on-disk mirrors. */ +static int store_python_localcache_mirror(const char* virtual_path, char* out, + size_t cap) { + const char* local = getenv("LOCALAPPDATA"); + if (!local || !local[0] || !virtual_path || !virtual_path[0]) + return 0; + size_t llen = strlen(local); + if (_strnicmp(virtual_path, local, (int)llen) != 0) + return 0; + const char* suffix = virtual_path + llen; + if (*suffix != '\\' && *suffix != '/') + return 0; + while (*suffix == '\\' || *suffix == '/') + ++suffix; + if (!suffix[0]) + return 0; + + char packages[1100]; + if (!join_path(packages, sizeof(packages), local, "Packages")) + return 0; + char pattern[1200]; + snprintf(pattern, sizeof(pattern), + "%s\\PythonSoftwareFoundation.Python.*", packages); + WIN32_FIND_DATAA fd; + HANDLE h = FindFirstFileA(pattern, &fd); + if (h == INVALID_HANDLE_VALUE) + return 0; + int found = 0; + do { + if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + char mirror[1400]; + int n = snprintf(mirror, sizeof(mirror), + "%s\\%s\\LocalCache\\Local\\%s", packages, + fd.cFileName, suffix); + if (n <= 0 || (size_t)n >= sizeof(mirror)) + continue; + DWORD attr = GetFileAttributesA(mirror); + if (attr != INVALID_FILE_ATTRIBUTES) { + snprintf(out, cap, "%s", mirror); + found = 1; + break; + } + } while (FindNextFileA(h, &fd)); + FindClose(h); + return found; +} + +/* Prefer the real on-disk directory (handles Store Python redirection). */ +static int resolve_existing_dir(const char* path, char* out, size_t cap) { + if (path && path[0] && path_is_dir(path)) { + snprintf(out, cap, "%s", path); + return 1; + } + char alt[1400]; + if (path && store_python_localcache_mirror(path, alt, sizeof(alt)) && + path_is_dir(alt)) { + snprintf(out, cap, "%s", alt); + return 1; + } + return 0; +} + +static int python_path_is_store(const char* path) { + if (!path || !path[0]) + return 0; + char lower[1100]; + size_t n = strlen(path); + if (n >= sizeof(lower)) + n = sizeof(lower) - 1; + for (size_t i = 0; i < n; ++i) { + char c = path[i]; + lower[i] = (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; + } + lower[n] = '\0'; + return strstr(lower, "windowsapps") != NULL || + strstr(lower, "pythonsoftwarefoundation") != NULL; +} + +/* Run a short cmdline and capture the first stdout line (trimmed). */ +static int capture_cmd_first_line(const char* cmdline, char* out, size_t cap) { + SECURITY_ATTRIBUTES sa; + memset(&sa, 0, sizeof(sa)); + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE rd = NULL, wr = NULL; + if (!CreatePipe(&rd, &wr, &sa, 0)) + return 0; + SetHandleInformation(rd, HANDLE_FLAG_INHERIT, 0); + + STARTUPINFOA si; + PROCESS_INFORMATION pi; + memset(&si, 0, sizeof(si)); + memset(&pi, 0, sizeof(pi)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdOutput = wr; + si.hStdError = GetStdHandle(STD_ERROR_HANDLE); + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + + char mutable_cmd[1024]; + snprintf(mutable_cmd, sizeof(mutable_cmd), "%s", cmdline); + if (!CreateProcessA(NULL, mutable_cmd, NULL, NULL, TRUE, CREATE_NO_WINDOW, + NULL, NULL, &si, &pi)) { + CloseHandle(rd); + CloseHandle(wr); + return 0; + } + CloseHandle(wr); + + char buf[512]; + size_t got = 0; + DWORD n = 0; + out[0] = '\0'; + while (ReadFile(rd, buf, sizeof(buf), &n, NULL) && n > 0) { + for (DWORD i = 0; i < n; ++i) { + char c = buf[i]; + if (c == '\r') + continue; + if (c == '\n') { + out[got] = '\0'; + got = cap; /* mark complete */ + break; + } + if (got + 1 < cap) + out[got++] = c; + } + if (got >= cap) + break; + } + if (got > 0 && got < cap) + out[got] = '\0'; + CloseHandle(rd); + WaitForSingleObject(pi.hProcess, 8000); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return out[0] != '\0'; +} +#else +static int resolve_existing_dir(const char* path, char* out, size_t cap) { + if (path && path[0] && path_is_dir(path)) { + snprintf(out, cap, "%s", path); + return 1; + } + return 0; +} + +#endif + +static int dirname_copy(char* out, size_t cap, const char* path) { + size_t n = strlen(path); + while (n > 0 && (path[n - 1] == '/' || path[n - 1] == '\\')) + --n; + while (n > 0 && path[n - 1] != '/' && path[n - 1] != '\\') + --n; + while (n > 0 && (path[n - 1] == '/' || path[n - 1] == '\\')) + --n; + if (n == 0) { + if (cap < 2) return 0; + out[0] = '.'; + out[1] = '\0'; + return 1; + } + if (n >= cap) return 0; + memcpy(out, path, n); + out[n] = '\0'; + return 1; +} + +static int resolve_cli_path(const char* root, char* out, size_t cap) { + const char* candidates[] = { + cfg_or(g_cfg->psxrecomp_cli_relpath, "psxrecomp/psxrecomp_cli.py"), + "psxrecomp/psxrecomp_cli.py", + }; + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { + if (!candidates[i] || !candidates[i][0]) + continue; + if (!join_path(out, cap, root, candidates[i])) + continue; + if (path_is_file(out)) + return 1; + } + return 0; +} + +static int looks_like_project_root(const char* root) { + char cli[1100], toml[1100]; + if (!join_path(toml, sizeof(toml), root, + cfg_or(g_cfg->seed_cfg_relpath, "game.toml"))) + return 0; + if (!path_is_file(toml)) + return 0; + return resolve_cli_path(root, cli, sizeof(cli)); +} + +static int find_on_path(const char* name, char* out, size_t cap) { +#if defined(_WIN32) + char cmd[640]; + snprintf(cmd, sizeof(cmd), "where %s >nul 2>nul", name); + if (system(cmd) == 0) { + snprintf(out, cap, "%s", name); + return 1; + } +#else + char cmd[640]; + snprintf(cmd, sizeof(cmd), "command -v %s >/dev/null 2>&1", name); + if (system(cmd) == 0) { + snprintf(out, cap, "%s", name); + return 1; + } +#endif + return 0; +} + +static int find_python(char* out, size_t cap) { + const char* env = getenv("PYTHON"); + if (env && env[0] && path_is_file(env)) { + snprintf(out, cap, "%s", env); + return 1; + } +#if defined(_WIN32) + /* Prefer python.org / py-launcher installs over the Microsoft Store + * stub: Store Python redirects LocalAppData writes into LocalCache. */ + char resolved[1100]; + if (capture_cmd_first_line( + "py -3 -c \"import sys; print(sys.executable)\"", resolved, + sizeof(resolved)) && + path_is_file(resolved) && !python_path_is_store(resolved)) { + snprintf(out, cap, "%s", resolved); + return 1; + } + if (capture_cmd_first_line( + "python -c \"import sys; print(sys.executable)\"", resolved, + sizeof(resolved)) && + path_is_file(resolved)) { + snprintf(out, cap, "%s", resolved); + return 1; + } + const char* candidates[] = {"python.exe", "python3.exe", "py.exe"}; +#else + const char* candidates[] = {"python3", "python"}; +#endif + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { + if (find_on_path(candidates[i], out, cap)) + return 1; + } + return 0; +} + +static int toolchain_bin_has_cmake(const char* bin, char* out, size_t cap) { + char cmake[1200]; +#if defined(_WIN32) + if (join_path(cmake, sizeof(cmake), bin, "cmake.exe") && path_is_file(cmake)) { + snprintf(out, cap, "%s", bin); + return 1; + } +#else + if (join_path(cmake, sizeof(cmake), bin, "cmake") && path_is_file(cmake)) { + snprintf(out, cap, "%s", bin); + return 1; + } +#endif + return 0; +} + +static int resolve_toolchain_bin_under(const char* wrap, char* out, size_t cap) { + char cand[1100], cmake[1200], root[1400]; + if (!wrap || !wrap[0]) + return 0; + if (!resolve_existing_dir(wrap, root, sizeof(root))) + return 0; + wrap = root; + if (join_path(cand, sizeof(cand), wrap, "bin") && + toolchain_bin_has_cmake(cand, out, cap)) + return 1; +#if defined(_WIN32) + WIN32_FIND_DATAA fd; + char pattern[1200]; + snprintf(pattern, sizeof(pattern), "%s\\*", wrap); + HANDLE h = FindFirstFileA(pattern, &fd); + if (h == INVALID_HANDLE_VALUE) + return 0; + int found = 0; + do { + if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + if (fd.cFileName[0] == '.') + continue; + char nested[1100], nbin[1100]; + if (!join_path(nested, sizeof(nested), wrap, fd.cFileName)) + continue; + if (!join_path(nbin, sizeof(nbin), nested, "bin")) + continue; + if (join_path(cmake, sizeof(cmake), nbin, "cmake.exe") && path_is_file(cmake)) { + snprintf(out, cap, "%s", nbin); + found = 1; + break; + } + } while (FindNextFileA(h, &fd)); + FindClose(h); + return found; +#else + DIR* dir = opendir(wrap); + if (!dir) + return 0; + int found = 0; + struct dirent* ent; + while ((ent = readdir(dir)) != NULL) { + if (ent->d_name[0] == '.') + continue; + char nested[1100], nbin[1100]; + if (!join_path(nested, sizeof(nested), wrap, ent->d_name)) + continue; + if (!path_is_dir(nested)) + continue; + if (!join_path(nbin, sizeof(nbin), nested, "bin")) + continue; + if (join_path(cmake, sizeof(cmake), nbin, "cmake") && path_is_file(cmake)) { + snprintf(out, cap, "%s", nbin); + found = 1; + break; + } + } + closedir(dir); + return found; +#endif +} + +/* Prefer tagged installs (CLI writes latest/ / offline/) then any child. */ +static int resolve_toolchain_cache_base(const char* base, char* out, size_t cap) { + static const char* prefer[] = {"latest", "offline", NULL}; + char cand[1100]; + if (!base || !base[0]) + return 0; + for (int i = 0; prefer[i]; ++i) { + if (join_path(cand, sizeof(cand), base, prefer[i]) && + resolve_toolchain_bin_under(cand, out, cap)) + return 1; + } + return resolve_toolchain_bin_under(base, out, cap); +} + +/* Same layout as psxrecomp/tools/toolchain_pack.py shared_cache_roots(). */ +static int resolve_shared_toolchain_cache(char* out, size_t cap) { + char bases[8][1400]; + int n = 0; +#if defined(_WIN32) + const char* local = getenv("LOCALAPPDATA"); + if (local && local[0]) { + if (join_path(bases[n], sizeof(bases[n]), local, + "psxrecomp/toolchains/cmake-clang-v1")) + ++n; + if (n < 8 && + join_path(bases[n], sizeof(bases[n]), local, + "retcomm/toolchains/cmake-clang-v1")) + ++n; + /* Store Python may have written only into LocalCache mirrors. */ + for (int i = 0, lim = n; i < lim && n < 8; ++i) { + char mirror[1400]; + if (store_python_localcache_mirror(bases[i], mirror, + sizeof(mirror))) + snprintf(bases[n++], sizeof(bases[0]), "%s", mirror); + } + } +#else + const char* xdg = getenv("XDG_DATA_HOME"); + const char* home = getenv("HOME"); + if (xdg && xdg[0]) { + if (join_path(bases[n], sizeof(bases[n]), xdg, + "psxrecomp/toolchains/cmake-clang-v1")) + ++n; + if (n < 8 && + join_path(bases[n], sizeof(bases[n]), xdg, + "retcomm/toolchains/cmake-clang-v1")) + ++n; + } else if (home && home[0]) { + if (join_path(bases[n], sizeof(bases[n]), home, + ".local/share/psxrecomp/toolchains/cmake-clang-v1")) + ++n; + if (n < 8 && + join_path(bases[n], sizeof(bases[n]), home, + ".local/share/retcomm/toolchains/cmake-clang-v1")) + ++n; + } +#endif + for (int i = 0; i < n; ++i) { + if (resolve_toolchain_cache_base(bases[i], out, cap)) + return 1; + } + return 0; +} + +/* CLI writes project_root/toolchain/.psxrecomp-bin with the bin path. */ +static int resolve_toolchain_stamp(char* out, size_t cap) { + char stamp[1200], line[1400], resolved[1400]; + FILE* f; + if (!g_project_root[0]) + return 0; + if (!join_path(stamp, sizeof(stamp), g_project_root, + "toolchain/.psxrecomp-bin")) + return 0; +#if defined(_WIN32) + { + char alt[1400]; + if (!path_is_file(stamp) && + store_python_localcache_mirror(stamp, alt, sizeof(alt)) && + path_is_file(alt)) + snprintf(stamp, sizeof(stamp), "%s", alt); + } +#endif + if (!path_is_file(stamp)) + return 0; + f = fopen(stamp, "r"); + if (!f) + return 0; + if (!fgets(line, sizeof(line), f)) { + fclose(f); + return 0; + } + fclose(f); + size_t n = strlen(line); + while (n && (line[n - 1] == '\n' || line[n - 1] == '\r' || + line[n - 1] == ' ')) + line[--n] = '\0'; + if (!line[0]) + return 0; + if (resolve_existing_dir(line, resolved, sizeof(resolved)) && + toolchain_bin_has_cmake(resolved, out, cap)) + return 1; + return 0; +} + +static int resolve_toolchain_bin(char* out, size_t cap) { + /* Fresh ensure-toolchain result (bin directory). */ + if (g_cli_toolchain_bin[0]) { + char resolved[1400]; + if (resolve_existing_dir(g_cli_toolchain_bin, resolved, + sizeof(resolved)) && + toolchain_bin_has_cmake(resolved, out, cap)) + return 1; + } + + const char* env_keys[] = { + "PSXRECOMP_TOOLCHAIN_DIR", "RETCOMM_TOOLCHAIN_DIR", "TOOLCHAIN_DIR", + "BPE_TOOLCHAIN_DIR", NULL}; + for (int i = 0; env_keys[i]; ++i) { + const char* e = getenv(env_keys[i]); + if (e && e[0] && resolve_toolchain_bin_under(e, out, cap)) + return 1; + } + if (resolve_toolchain_stamp(out, cap)) + return 1; + if (g_project_root[0]) { + char wrap[1100]; + if (join_path(wrap, sizeof(wrap), g_project_root, "toolchain") && + resolve_toolchain_bin_under(wrap, out, cap)) + return 1; + } + /* CLI downloads land here — must match toolchain_pack.py. */ + if (resolve_shared_toolchain_cache(out, cap)) + return 1; + return 0; +} + +static void activate_toolchain_path(void) { + char pack_root[1400]; + g_toolchain_bin[0] = '\0'; + if (!resolve_toolchain_bin(g_toolchain_bin, sizeof(g_toolchain_bin))) + return; + const char* old = getenv("PATH"); +#if defined(_WIN32) + char neu[8192]; + snprintf(neu, sizeof(neu), "%s%s%s", g_toolchain_bin, old ? ";" : "", + old ? old : ""); + _putenv_s("PATH", neu); +#else + char neu[8192]; + snprintf(neu, sizeof(neu), "%s%s%s", g_toolchain_bin, old ? ":" : "", + old ? old : ""); + setenv("PATH", neu, 1); +#endif + /* Pack root (parent of bin/) — Windows cmake-clang-v1 ships zlib here. */ + if (dirname_copy(pack_root, sizeof(pack_root), g_toolchain_bin) && + pack_root[0]) { +#if defined(_WIN32) + _putenv_s("ZLIB_ROOT", pack_root); + { + const char* prev = getenv("CMAKE_PREFIX_PATH"); + char pref[8192]; + if (prev && prev[0] && !strstr(prev, pack_root)) + snprintf(pref, sizeof(pref), "%s;%s", pack_root, prev); + else if (!prev || !prev[0]) + snprintf(pref, sizeof(pref), "%s", pack_root); + else + pref[0] = '\0'; + if (pref[0]) + _putenv_s("CMAKE_PREFIX_PATH", pref); + } +#else + setenv("ZLIB_ROOT", pack_root, 1); + { + const char* prev = getenv("CMAKE_PREFIX_PATH"); + char pref[8192]; + if (prev && prev[0] && !strstr(prev, pack_root)) + snprintf(pref, sizeof(pref), "%s:%s", pack_root, prev); + else if (!prev || !prev[0]) + snprintf(pref, sizeof(pref), "%s", pack_root); + else + pref[0] = '\0'; + if (pref[0]) + setenv("CMAKE_PREFIX_PATH", pref, 1); + } +#endif + } +} + +static int find_cmake(char* out, size_t cap) { + const char* env = getenv("CMAKE"); + if (env && env[0] && path_is_file(env)) { + snprintf(out, cap, "%s", env); + return 1; + } + char tc[1100], cand[1200]; + if (resolve_toolchain_bin(tc, sizeof(tc))) { +#if defined(_WIN32) + if (join_path(cand, sizeof(cand), tc, "cmake.exe") && path_is_file(cand)) { + snprintf(out, cap, "%s", cand); + return 1; + } +#else + if (join_path(cand, sizeof(cand), tc, "cmake") && path_is_file(cand)) { + snprintf(out, cap, "%s", cand); + return 1; + } +#endif + } +#if defined(_WIN32) + return find_on_path("cmake.exe", out, cap); +#else + return find_on_path("cmake", out, cap); +#endif +} + +static int discover_project_root(char* out, size_t cap) { + const char* env_name = + cfg_or(g_cfg->project_root_env, "PSXRECOMP_PROJECT_ROOT"); + const char* env = getenv(env_name); + if (env && env[0] && looks_like_project_root(env)) { + snprintf(out, cap, "%s", env); + return 1; + } + + char start[1024]; +#if defined(_WIN32) + if (!GetCurrentDirectoryA((DWORD)sizeof(start), start)) + start[0] = '\0'; +#else + if (!getcwd(start, sizeof(start))) + start[0] = '\0'; +#endif + + char cur[1024]; + snprintf(cur, sizeof(cur), "%s", start[0] ? start : "."); + for (int i = 0; i < 10; ++i) { + if (looks_like_project_root(cur)) { + snprintf(out, cap, "%s", cur); + return 1; + } + char parent[1024]; + if (!dirname_copy(parent, sizeof(parent), cur)) + break; + if (strcmp(parent, cur) == 0) + break; + snprintf(cur, sizeof(cur), "%s", parent); + } + return 0; +} + +static int resolve_build_paths(void) { + const char* env_name = + cfg_or(g_cfg->build_dir_env, "PSXRECOMP_BUILD_DIR"); + const char* env = getenv(env_name); + if (env && env[0]) { + snprintf(g_build_dir, sizeof(g_build_dir), "%s", env); + } else { + const char* names[] = { + cfg_or(g_cfg->build_dir_name, "build"), + "build-release", + "build", + "build-ci", + }; + int found = 0; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { + if (!names[i] || !names[i][0]) + continue; + char cand[1100]; + if (!join_path(cand, sizeof(cand), g_project_root, names[i])) + continue; + if (path_is_dir(cand)) { + snprintf(g_build_dir, sizeof(g_build_dir), "%s", cand); + found = 1; + break; + } + } + /* Setup zips ship without a pre-made build tree. Prefer the configured + * name so rebuild can cmake -B it on first Generate & rebuild. */ + if (!found) { + const char* prefer = cfg_or(g_cfg->build_dir_name, "build-release"); + if (!join_path(g_build_dir, sizeof(g_build_dir), g_project_root, + prefer)) + return 0; + } + } + + char exe_name[300]; +#if defined(_WIN32) + snprintf(exe_name, sizeof(exe_name), "%s.exe", g_exe_basename); +#else + snprintf(exe_name, sizeof(exe_name), "%s", g_exe_basename); +#endif + return join_path(g_exe_path, sizeof(g_exe_path), g_build_dir, exe_name); +} + +static int bios_backends_missing(void) { + char openbios[1100], scph[1100]; + if (!join_path(openbios, sizeof(openbios), g_project_root, + "psxrecomp/generated/OpenBIOS_dispatch.c")) + return 1; + if (!join_path(scph, sizeof(scph), g_project_root, + "psxrecomp/generated/SCPH1001_dispatch.c")) + return 1; + return !(path_is_file(openbios) || path_is_file(scph)); +} + +int psxrecomp_codegen_host_sources_missing( + const PsxrecompCodegenHostConfig* cfg) { + if (!cfg || !cfg->cmake_target || !cfg->exe_basename) + return 0; + g_cfg = cfg; + if (!g_project_root[0] && + !discover_project_root(g_project_root, sizeof(g_project_root))) + return 0; + char marker[1100]; + if (!join_path(marker, sizeof(marker), g_project_root, + cfg_or(cfg->gen_marker_relpath, + "generated/SLUS_011.89_dispatch.c"))) + return 1; + if (!path_is_file(marker)) + return 1; + return bios_backends_missing(); +} + +static int read_line_file(const char* path, char* out, size_t cap) { + FILE* f = fopen(path, "r"); + if (!f) return 0; + if (!fgets(out, (int)cap, f)) { + fclose(f); + return 0; + } + fclose(f); + size_t n = strlen(out); + while (n && (out[n - 1] == '\n' || out[n - 1] == '\r')) + out[--n] = '\0'; + return n > 0; +} + +static int resolve_bios_arg(char* out, size_t cap) { + char cand[1100]; + if (join_path(cand, sizeof(cand), g_project_root, "bios.cfg") && + read_line_file(cand, out, cap) && path_is_file(out)) + return 1; + if (read_line_file("bios.cfg", out, cap) && path_is_file(out)) + return 1; + out[0] = '\0'; + return 0; +} + +static int json_get_string(const char* line, const char* key, char* out, + size_t out_cap) { + char pattern[96]; + snprintf(pattern, sizeof(pattern), "\"%s\":\"", key); + const char* p = strstr(line, pattern); + if (!p) return 0; + p += strlen(pattern); + size_t i = 0; + while (*p && *p != '"' && i + 1 < out_cap) { + if (*p == '\\' && p[1]) { + ++p; + out[i++] = *p++; + continue; + } + out[i++] = *p++; + } + out[i] = '\0'; + return i > 0; +} + +static int json_get_number(const char* line, const char* key, double* out) { + char pattern[96]; + snprintf(pattern, sizeof(pattern), "\"%s\":", key); + const char* p = strstr(line, pattern); + if (!p) return 0; + p += strlen(pattern); + while (*p == ' ') ++p; + char* end = NULL; + double v = strtod(p, &end); + if (end == p) return 0; + *out = v; + return 1; +} + +static void handle_progress_line(const char* line, + RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx) { + if (!line || line[0] != '{') + return; + char event[64] = ""; + json_get_string(line, "event", event, sizeof(event)); + /* Capture ensure-toolchain result even when UI progress is absent. */ + if (strcmp(event, "result") == 0) { + char tb[1400], resolved[1400]; + if (json_get_string(line, "toolchain_bin", tb, sizeof(tb)) && tb[0] && + resolve_existing_dir(tb, resolved, sizeof(resolved))) { + snprintf(g_cli_toolchain_bin, sizeof(g_cli_toolchain_bin), "%s", + resolved); + /* Env expects pack root (parent of bin/), not bin/ itself. */ + char pack_root[1400]; + if (dirname_copy(pack_root, sizeof(pack_root), resolved) && + pack_root[0]) { +#if defined(_WIN32) + _putenv_s("PSXRECOMP_TOOLCHAIN_DIR", pack_root); +#else + setenv("PSXRECOMP_TOOLCHAIN_DIR", pack_root, 1); +#endif + } + } + } + if (!on_progress) + return; + if (strcmp(event, "phase") == 0) { + char message[240] = ""; + char phase[64] = ""; + double pct = -1.0; + json_get_string(line, "message", message, sizeof(message)); + json_get_string(line, "phase", phase, sizeof(phase)); + if (!json_get_number(line, "pct", &pct)) + pct = -1.0; + if (!message[0] && phase[0]) + snprintf(message, sizeof(message), "%s", phase); + on_progress(progress_ctx, (float)pct, message[0] ? message : NULL); + } else if (strcmp(event, "log") == 0 || strcmp(event, "error") == 0) { + char message[240] = ""; + if (json_get_string(line, "message", message, sizeof(message))) + on_progress(progress_ctx, -1.0f, message); + } +} + +#if defined(_WIN32) +static int run_cli_win(const char* cmdline, + RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx, char* err_msg, size_t err_cap, + const char* fail_label) { + SECURITY_ATTRIBUTES sa; + memset(&sa, 0, sizeof(sa)); + sa.nLength = sizeof(sa); + sa.bInheritHandle = TRUE; + HANDLE rd = NULL, wr = NULL; + if (!CreatePipe(&rd, &wr, &sa, 0)) { + snprintf(err_msg, err_cap, "CreatePipe failed."); + return 0; + } + SetHandleInformation(rd, HANDLE_FLAG_INHERIT, 0); + + STARTUPINFOA si; + PROCESS_INFORMATION pi; + memset(&si, 0, sizeof(si)); + memset(&pi, 0, sizeof(pi)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdOutput = wr; + si.hStdError = GetStdHandle(STD_ERROR_HANDLE); + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + + char mutable_cmd[4096]; + snprintf(mutable_cmd, sizeof(mutable_cmd), "%s", cmdline); + if (!CreateProcessA(NULL, mutable_cmd, NULL, NULL, TRUE, 0, NULL, + g_project_root, &si, &pi)) { + CloseHandle(rd); + CloseHandle(wr); + snprintf(err_msg, err_cap, "Failed to spawn %s.", fail_label); + return 0; + } + CloseHandle(wr); + + char buf[512]; + /* Long JSONL rows (toolchain paths under LocalAppData) need headroom. */ + char line[16384]; + size_t line_len = 0; + DWORD n = 0; + while (ReadFile(rd, buf, sizeof(buf), &n, NULL) && n > 0) { + for (DWORD i = 0; i < n; ++i) { + char c = buf[i]; + if (c == '\r') continue; + if (c == '\n') { + line[line_len] = '\0'; + handle_progress_line(line, on_progress, progress_ctx); + line_len = 0; + continue; + } + if (line_len + 1 < sizeof(line)) + line[line_len++] = c; + } + } + if (line_len) { + line[line_len] = '\0'; + handle_progress_line(line, on_progress, progress_ctx); + } + CloseHandle(rd); + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + if (code == 0) return 1; + if (code == 3) + snprintf(err_msg, err_cap, "Disc verification failed (wrong dump)."); + else + snprintf(err_msg, err_cap, "%s failed (exit %lu).", fail_label, + (unsigned long)code); + return 0; +} +#else +static int run_cli_posix(char* const argv[], + RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx, char* err_msg, size_t err_cap, + const char* fail_label) { + int pipefd[2]; + if (pipe(pipefd) != 0) { + snprintf(err_msg, err_cap, "pipe() failed: %s", strerror(errno)); + return 0; + } + + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_addclose(&actions, pipefd[0]); + posix_spawn_file_actions_adddup2(&actions, pipefd[1], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&actions, pipefd[1]); + + pid_t pid = 0; + int rc = posix_spawnp(&pid, argv[0], &actions, NULL, argv, environ); + posix_spawn_file_actions_destroy(&actions); + close(pipefd[1]); + if (rc != 0) { + close(pipefd[0]); + snprintf(err_msg, err_cap, "Failed to spawn %s: %s", fail_label, + strerror(rc)); + return 0; + } + + FILE* out = fdopen(pipefd[0], "r"); + if (!out) { + close(pipefd[0]); + waitpid(pid, NULL, 0); + snprintf(err_msg, err_cap, "fdopen failed."); + return 0; + } + char line[16384]; + while (fgets(line, sizeof(line), out)) { + size_t n = strlen(line); + while (n && (line[n - 1] == '\n' || line[n - 1] == '\r')) + line[--n] = '\0'; + handle_progress_line(line, on_progress, progress_ctx); + } + fclose(out); + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + snprintf(err_msg, err_cap, "waitpid failed: %s", strerror(errno)); + return 0; + } + int code = WIFEXITED(status) ? WEXITSTATUS(status) : 1; + if (code == 0) return 1; + if (code == 3) + snprintf(err_msg, err_cap, "Disc verification failed (wrong dump)."); + else + snprintf(err_msg, err_cap, "%s failed (exit %d).", fail_label, code); + return 0; +} +#endif + +/* ---- Host-native toolchain install (no Store Python AppData redirect) ---- */ + +static const char* k_tc_repo = "TechnicallyComputers/retcomm-toolchains"; + +static const char* toolchain_zip_asset_name(void) { +#if defined(_WIN32) + return "cmake-clang-v1-windows-x64.zip"; +#elif defined(__APPLE__) + return "cmake-clang-v1-macos-universal.zip"; +#else + return "cmake-clang-v1-linux-x64.zip"; +#endif +} + +static int mkdir_p(const char* path) { + char tmp[1400]; + size_t n; + if (!path || !path[0]) + return 0; + n = strlen(path); + if (n >= sizeof(tmp)) + return 0; + memcpy(tmp, path, n + 1); +#if defined(_WIN32) + { + char* p = tmp; + if (n >= 2 && tmp[1] == ':') + p = tmp + 2; + while (*p == '\\' || *p == '/') + ++p; + for (; *p; ++p) { + if (*p == '\\' || *p == '/') { + char save = *p; + *p = '\0'; + if (tmp[0]) + CreateDirectoryA(tmp, NULL); + *p = save; + } + } + return CreateDirectoryA(tmp, NULL) || + GetLastError() == ERROR_ALREADY_EXISTS || path_is_dir(tmp); + } +#else + { + char* p = tmp; + if (*p == '/') + ++p; + for (; *p; ++p) { + if (*p == '/') { + *p = '\0'; + if (mkdir(tmp, 0755) != 0 && errno != EEXIST && !path_is_dir(tmp)) + return 0; + *p = '/'; + } + } + return mkdir(tmp, 0755) == 0 || errno == EEXIST || path_is_dir(tmp); + } +#endif +} + +static int rmtree_path(const char* path) { + char cmd[1600]; + if (!path || !path[0] || !path_is_dir(path)) + return 1; +#if defined(_WIN32) + snprintf(cmd, sizeof(cmd), "cmd.exe /c rmdir /s /q \"%s\"", path); +#else + snprintf(cmd, sizeof(cmd), "rm -rf \"%s\"", path); +#endif + return system(cmd) == 0 || !path_is_dir(path); +} + +#if defined(_WIN32) +static int run_cmdline_wait(const char* cmdline, DWORD* out_code) { + STARTUPINFOA si; + PROCESS_INFORMATION pi; + char mutable_cmd[8192]; + DWORD code = 1; + memset(&si, 0, sizeof(si)); + memset(&pi, 0, sizeof(pi)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + si.hStdError = GetStdHandle(STD_ERROR_HANDLE); + snprintf(mutable_cmd, sizeof(mutable_cmd), "%s", cmdline); + if (!CreateProcessA(NULL, mutable_cmd, NULL, NULL, TRUE, CREATE_NO_WINDOW, + NULL, NULL, &si, &pi)) + return 0; + WaitForSingleObject(pi.hProcess, INFINITE); + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + if (out_code) + *out_code = code; + return 1; +} +#endif + +static int shared_toolchain_latest_dir(char* out, size_t cap) { +#if defined(_WIN32) + const char* local = getenv("LOCALAPPDATA"); + if (!local || !local[0]) + return 0; + return join_path(out, cap, local, "psxrecomp/toolchains/cmake-clang-v1/latest"); +#else + const char* xdg = getenv("XDG_DATA_HOME"); + const char* home = getenv("HOME"); + char base[1100]; + if (xdg && xdg[0]) { + if (!join_path(base, sizeof(base), xdg, + "psxrecomp/toolchains/cmake-clang-v1/latest")) + return 0; + } else if (home && home[0]) { + if (!join_path(base, sizeof(base), home, + ".local/share/psxrecomp/toolchains/cmake-clang-v1/latest")) + return 0; + } else { + return 0; + } + snprintf(out, cap, "%s", base); + return 1; +#endif +} + +static int pack_root_has_cmake(const char* root) { + char bin[1400], cmake[1400]; + if (!root || !root[0]) + return 0; + if (!join_path(bin, sizeof(bin), root, "bin")) + return 0; +#if defined(_WIN32) + return join_path(cmake, sizeof(cmake), bin, "cmake.exe") && path_is_file(cmake); +#else + return join_path(cmake, sizeof(cmake), bin, "cmake") && path_is_file(cmake); +#endif +} + +static int write_project_toolchain_stamp(const char* bin_dir) { + char tc_dir[1200], stamp[1200]; + FILE* f; + if (!g_project_root[0] || !bin_dir || !bin_dir[0]) + return 0; + if (!join_path(tc_dir, sizeof(tc_dir), g_project_root, "toolchain")) + return 0; + mkdir_p(tc_dir); + if (!join_path(stamp, sizeof(stamp), tc_dir, ".psxrecomp-bin")) + return 0; + f = fopen(stamp, "w"); + if (!f) + return 0; + fprintf(f, "%s\n", bin_dir); + fclose(f); + return 1; +} + +static int activate_installed_pack_root(const char* pack_root) { + char bin[1400]; + if (!pack_root_has_cmake(pack_root)) + return 0; + if (!join_path(bin, sizeof(bin), pack_root, "bin")) + return 0; + snprintf(g_cli_toolchain_bin, sizeof(g_cli_toolchain_bin), "%s", bin); +#if defined(_WIN32) + _putenv_s("PSXRECOMP_TOOLCHAIN_DIR", pack_root); +#else + setenv("PSXRECOMP_TOOLCHAIN_DIR", pack_root, 1); +#endif + write_project_toolchain_stamp(bin); + activate_toolchain_path(); + return find_cmake(g_cmake, sizeof(g_cmake)) ? 1 : 0; +} + +#if defined(_WIN32) +/* Point a real LocalAppData path at a Store-Python LocalCache pack (no copy). */ +static int junction_dir(const char* link_path, const char* target_path) { + char cmd[3200]; + DWORD code = 1; + char parent[1400]; + if (!link_path || !target_path || !path_is_dir(target_path)) + return 0; + if (path_is_dir(link_path) || path_is_file(link_path)) { + /* Replace broken/empty dir; keep a usable pack. */ + if (pack_root_has_cmake(link_path)) + return 1; + rmtree_path(link_path); + } + if (!dirname_copy(parent, sizeof(parent), link_path)) + return 0; + mkdir_p(parent); + snprintf(cmd, sizeof(cmd), "cmd.exe /c mklink /J \"%s\" \"%s\"", link_path, + target_path); + if (!run_cmdline_wait(cmd, &code)) + return 0; + return code == 0 && path_is_dir(link_path); +} + +static int find_store_localcache_pack_root(char* out, size_t cap) { + const char* local = getenv("LOCALAPPDATA"); + char packages[1100], pattern[1200]; + WIN32_FIND_DATAA fd; + HANDLE h; + if (!local || !local[0]) + return 0; + if (!join_path(packages, sizeof(packages), local, "Packages")) + return 0; + snprintf(pattern, sizeof(pattern), + "%s\\PythonSoftwareFoundation.Python.*", packages); + h = FindFirstFileA(pattern, &fd); + if (h == INVALID_HANDLE_VALUE) + return 0; + do { + const char* suffixes[] = { + "LocalCache\\Local\\psxrecomp\\toolchains\\cmake-clang-v1\\latest", + "LocalCache\\Local\\psxrecomp\\toolchains\\cmake-clang-v1\\offline", + "LocalCache\\Local\\retcomm\\toolchains\\cmake-clang-v1\\latest", + "LocalCache\\Local\\retcomm\\toolchains\\cmake-clang-v1\\offline", + NULL}; + int i; + if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + for (i = 0; suffixes[i]; ++i) { + char cand[1400], nested[1400]; + snprintf(cand, sizeof(cand), "%s\\%s\\%s", packages, fd.cFileName, + suffixes[i]); + if (pack_root_has_cmake(cand)) { + snprintf(out, cap, "%s", cand); + FindClose(h); + return 1; + } + /* Nested single child with bin/cmake.exe */ + if (path_is_dir(cand)) { + WIN32_FIND_DATAA kd; + char kp[1400]; + HANDLE hk; + snprintf(kp, sizeof(kp), "%s\\*", cand); + hk = FindFirstFileA(kp, &kd); + if (hk == INVALID_HANDLE_VALUE) + continue; + do { + if (!(kd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + if (kd.cFileName[0] == '.') + continue; + if (!join_path(nested, sizeof(nested), cand, kd.cFileName)) + continue; + if (pack_root_has_cmake(nested)) { + snprintf(out, cap, "%s", nested); + FindClose(hk); + FindClose(h); + return 1; + } + } while (FindNextFileA(hk, &kd)); + FindClose(hk); + } + } + } while (FindNextFileA(h, &fd)); + FindClose(h); + return 0; +} + +/* allow_copy=0: point this process at LocalCache (cheap). + * allow_copy=1: also junction/robocopy into real %LOCALAPPDATA% (ensure). */ +static int harvest_store_python_toolchain(int allow_copy) { + char cache_root[1400], real_latest[1400], proj_tc[1200]; + if (!find_store_localcache_pack_root(cache_root, sizeof(cache_root))) + return 0; + if (!allow_copy) + return activate_installed_pack_root(cache_root); + if (!shared_toolchain_latest_dir(real_latest, sizeof(real_latest))) + return activate_installed_pack_root(cache_root); + if (!junction_dir(real_latest, cache_root) && + !pack_root_has_cmake(real_latest)) { + /* Junction failed — robocopy into the real tree. */ + char parent[1400], cmd[3200]; + DWORD code = 1; + if (!dirname_copy(parent, sizeof(parent), real_latest)) + return activate_installed_pack_root(cache_root); + mkdir_p(parent); + rmtree_path(real_latest); + mkdir_p(real_latest); + snprintf(cmd, sizeof(cmd), + "cmd.exe /c robocopy \"%s\" \"%s\" /E /NFL /NDL /NJH /NJS /nc " + "/ns /np", + cache_root, real_latest); + if (!run_cmdline_wait(cmd, &code) || code > 7 || + !pack_root_has_cmake(real_latest)) + return activate_installed_pack_root(cache_root); + } + if (g_project_root[0] && + join_path(proj_tc, sizeof(proj_tc), g_project_root, "toolchain")) { + if (!pack_root_has_cmake(proj_tc)) + junction_dir(proj_tc, pack_root_has_cmake(real_latest) + ? real_latest + : cache_root); + } + return activate_installed_pack_root( + pack_root_has_cmake(real_latest) ? real_latest : cache_root); +} + +static int host_download_url_to_file(const char* url, const char* dest, + char* err_msg, size_t err_cap) { + char cmd[4096]; + DWORD code = 1; + char parent[1400]; + if (!dirname_copy(parent, sizeof(parent), dest)) { + snprintf(err_msg, err_cap, "Bad download destination."); + return 0; + } + mkdir_p(parent); + DeleteFileA(dest); + /* Windows 10+ ships curl.exe. -L follows GitHub release redirects. */ + snprintf(cmd, sizeof(cmd), + "curl.exe -fsSL --retry 3 --retry-delay 2 -o \"%s\" \"%s\"", dest, + url); + if (!run_cmdline_wait(cmd, &code) || code != 0) { + snprintf(err_msg, err_cap, + "Toolchain download failed (curl exit %lu). Check network / " + "curl.exe.", + (unsigned long)code); + return 0; + } + return path_is_file(dest); +} + +static int host_extract_zip(const char* zip_path, const char* dest_dir, + char* err_msg, size_t err_cap) { + char cmd[3200]; + DWORD code = 1; + char parent[1400]; + if (!path_is_file(zip_path)) { + snprintf(err_msg, err_cap, "Toolchain zip not found: %s", zip_path); + return 0; + } + if (!dirname_copy(parent, sizeof(parent), dest_dir)) { + snprintf(err_msg, err_cap, "Bad extract destination."); + return 0; + } + mkdir_p(parent); + rmtree_path(dest_dir); + mkdir_p(dest_dir); + /* tar.exe on Windows 10+ extracts .zip */ + snprintf(cmd, sizeof(cmd), "tar.exe -xf \"%s\" -C \"%s\"", zip_path, + dest_dir); + if (!run_cmdline_wait(cmd, &code) || code != 0) { + snprintf(err_msg, err_cap, "Failed to extract toolchain zip (tar exit %lu).", + (unsigned long)code); + return 0; + } + if (pack_root_has_cmake(dest_dir)) + return 1; + /* Single nested directory layout — resolve_toolchain_bin_under handles it. */ + { + WIN32_FIND_DATAA fd; + char pattern[1400], child[1400]; + HANDLE h; + snprintf(pattern, sizeof(pattern), "%s\\*", dest_dir); + h = FindFirstFileA(pattern, &fd); + if (h != INVALID_HANDLE_VALUE) { + do { + if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + if (fd.cFileName[0] == '.') + continue; + if (!join_path(child, sizeof(child), dest_dir, fd.cFileName)) + continue; + if (pack_root_has_cmake(child)) { + FindClose(h); + return 1; + } + } while (FindNextFileA(h, &fd)); + FindClose(h); + } + } + snprintf(err_msg, err_cap, "Toolchain zip missing bin/cmake.exe."); + return 0; +} +#else +static int host_download_url_to_file(const char* url, const char* dest, + char* err_msg, size_t err_cap) { + char cmd[4096]; + char parent[1400]; + if (!dirname_copy(parent, sizeof(parent), dest)) { + snprintf(err_msg, err_cap, "Bad download destination."); + return 0; + } + mkdir_p(parent); + unlink(dest); + snprintf(cmd, sizeof(cmd), + "curl -fsSL --retry 3 --retry-delay 2 -o \"%s\" \"%s\"", dest, url); + if (system(cmd) != 0) { + snprintf(err_msg, err_cap, "Toolchain download failed (curl)."); + return 0; + } + return path_is_file(dest); +} + +static int host_extract_zip(const char* zip_path, const char* dest_dir, + char* err_msg, size_t err_cap) { + char cmd[3200]; + char parent[1400]; + if (!path_is_file(zip_path)) { + snprintf(err_msg, err_cap, "Toolchain zip not found: %s", zip_path); + return 0; + } + if (!dirname_copy(parent, sizeof(parent), dest_dir)) { + snprintf(err_msg, err_cap, "Bad extract destination."); + return 0; + } + mkdir_p(parent); + rmtree_path(dest_dir); + mkdir_p(dest_dir); + snprintf(cmd, sizeof(cmd), "unzip -q \"%s\" -d \"%s\"", zip_path, dest_dir); + if (system(cmd) != 0) { + snprintf(cmd, sizeof(cmd), "tar -xf \"%s\" -C \"%s\"", zip_path, + dest_dir); + if (system(cmd) != 0) { + snprintf(err_msg, err_cap, "Failed to extract toolchain zip."); + return 0; + } + } + if (pack_root_has_cmake(dest_dir)) + return 1; + /* Nested child with bin/ is fine — resolve_toolchain_bin_under handles it. */ + { + DIR* dir = opendir(dest_dir); + struct dirent* ent; + if (!dir) { + snprintf(err_msg, err_cap, "Toolchain zip missing bin/cmake."); + return 0; + } + while ((ent = readdir(dir)) != NULL) { + char child[1400]; + if (ent->d_name[0] == '.') + continue; + if (!join_path(child, sizeof(child), dest_dir, ent->d_name)) + continue; + if (path_is_dir(child) && pack_root_has_cmake(child)) { + closedir(dir); + return 1; + } + } + closedir(dir); + } + snprintf(err_msg, err_cap, "Toolchain zip missing bin/cmake."); + return 0; +} +#endif + +static int link_or_stamp_project_toolchain(const char* pack_root) { + char proj_tc[1200], bin[1400]; + if (!pack_root_has_cmake(pack_root)) + return 0; + if (!join_path(bin, sizeof(bin), pack_root, "bin")) + return 0; +#if defined(_WIN32) + if (g_project_root[0] && + join_path(proj_tc, sizeof(proj_tc), g_project_root, "toolchain")) { + if (!pack_root_has_cmake(proj_tc)) + junction_dir(proj_tc, pack_root); + } +#else + if (g_project_root[0] && + join_path(proj_tc, sizeof(proj_tc), g_project_root, "toolchain")) { + if (!pack_root_has_cmake(proj_tc) && !path_is_dir(proj_tc)) { + char cmd[2800]; + snprintf(cmd, sizeof(cmd), "ln -s \"%s\" \"%s\"", pack_root, + proj_tc); + (void)system(cmd); + } + } +#endif + write_project_toolchain_stamp(bin); + return 1; +} + +static int host_install_toolchain_from_zip( + const char* zip_path, RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx, char* err_msg, size_t err_cap) { + char dest[1400]; + if (!shared_toolchain_latest_dir(dest, sizeof(dest))) { + snprintf(err_msg, err_cap, "Cannot resolve shared toolchain directory."); + return 0; + } + if (on_progress) + on_progress(progress_ctx, 0.45f, "Extracting portable cmake/clang…"); + if (!host_extract_zip(zip_path, dest, err_msg, err_cap)) + return 0; + if (on_progress) + on_progress(progress_ctx, 0.85f, "Activating toolchain…"); + link_or_stamp_project_toolchain(dest); + if (activate_installed_pack_root(dest)) + return 1; + /* Nested layout under dest/ */ + if (resolve_toolchain_bin_under(dest, g_cli_toolchain_bin, + sizeof(g_cli_toolchain_bin))) { + char pack[1400]; + if (dirname_copy(pack, sizeof(pack), g_cli_toolchain_bin)) + return activate_installed_pack_root(pack); + } + snprintf(err_msg, err_cap, "Extracted toolchain but cmake was not found."); + return 0; +} + +static int host_download_and_install_toolchain( + RecompLauncherCPrepareProgressFn on_progress, void* progress_ctx, + char* err_msg, size_t err_cap) { + char url[512], zip_path[1400], tmp_dir[1100]; + const char* asset = toolchain_zip_asset_name(); + snprintf(url, sizeof(url), + "https://github.com/%s/releases/latest/download/%s", k_tc_repo, + asset); +#if defined(_WIN32) + { + char tmp[512]; + DWORD n = GetTempPathA(sizeof(tmp), tmp); + if (n == 0 || n >= sizeof(tmp)) { + snprintf(err_msg, err_cap, "GetTempPath failed."); + return 0; + } + snprintf(tmp_dir, sizeof(tmp_dir), "%spsxrecomp-tc-%lu", tmp, + (unsigned long)GetCurrentProcessId()); + } +#else + snprintf(tmp_dir, sizeof(tmp_dir), "/tmp/psxrecomp-tc-%d", (int)getpid()); +#endif + mkdir_p(tmp_dir); + if (!join_path(zip_path, sizeof(zip_path), tmp_dir, asset)) { + snprintf(err_msg, err_cap, "Temp path too long."); + return 0; + } + if (on_progress) + on_progress(progress_ctx, 0.1f, "Downloading portable cmake/clang…"); + if (!host_download_url_to_file(url, zip_path, err_msg, err_cap)) { + rmtree_path(tmp_dir); + return 0; + } + if (!host_install_toolchain_from_zip(zip_path, on_progress, progress_ctx, + err_msg, err_cap)) { + rmtree_path(tmp_dir); + return 0; + } + rmtree_path(tmp_dir); + return 1; +} + +static int host_toolchain_is_ready(void) { + if (!g_project_root[0]) + return 0; + activate_toolchain_path(); + if (find_cmake(g_cmake, sizeof(g_cmake))) + return 1; +#if defined(_WIN32) + /* Reuse a Store-Python LocalCache install without copying. */ + if (harvest_store_python_toolchain(0)) + return 1; +#endif + return 0; +} + +/* Download or offline-install cmake-clang-v1 (wizard page 0 / rebuild fallback). + * Prefer host-native curl/tar so Microsoft Store Python cannot redirect the + * unpack into Packages\\...\\LocalCache. */ +static int host_ensure_toolchain_with_progress( + int download, const char* zip_path, char* err_msg, size_t err_cap, + RecompLauncherCPrepareProgressFn on_progress, void* progress_ctx) { + if (!g_project_root[0]) { + snprintf(err_msg, err_cap, "Project root is not available."); + return 0; + } + activate_toolchain_path(); + if (find_cmake(g_cmake, sizeof(g_cmake))) + return 1; + +#if defined(_WIN32) + if (on_progress) + on_progress(progress_ctx, 0.02f, + "Checking for an existing portable toolchain…"); + if (harvest_store_python_toolchain(1)) + return 1; +#endif + + if (zip_path && zip_path[0]) { + if (on_progress) + on_progress(progress_ctx, 0.05f, "Installing toolchain from zip…"); + if (host_install_toolchain_from_zip(zip_path, on_progress, progress_ctx, + err_msg, err_cap)) + return 1; + return 0; + } + + if (download) { + if (host_download_and_install_toolchain(on_progress, progress_ctx, + err_msg, err_cap)) + return 1; + return 0; + } + + snprintf(err_msg, err_cap, + "No portable toolchain found. Enable automatic download, pick a " + "cmake-clang-v1 zip, or set PSXRECOMP_TOOLCHAIN_DIR."); + return 0; +} + +/* Rebuild-time fallback if the wizard step was skipped / cache pruned. */ +static int host_ensure_toolchain(RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx, char* err_msg, + size_t err_cap) { + return host_ensure_toolchain_with_progress(1, NULL, err_msg, err_cap, + on_progress, progress_ctx); +} + +static int host_prepare_generate(const char* source_path, char* out_path, + size_t out_cap, char* err_msg, size_t err_cap, + RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx) { + if (!g_ready) { + snprintf(err_msg, err_cap, "Local codegen tools are not available."); + return 0; + } + if (!source_path || !source_path[0]) { + snprintf(err_msg, err_cap, "No disc selected."); + return 0; + } + activate_toolchain_path(); + if (on_progress) + on_progress(progress_ctx, 0.02f, "Starting psxrecomp generate…"); + + char bios_path[1100]; + const int have_bios = resolve_bios_arg(bios_path, sizeof(bios_path)); + +#if defined(_WIN32) + char cmdline[4096]; + if (have_bios) { + snprintf(cmdline, sizeof(cmdline), + "\"%s\" \"%s\" generate --project-root \"%s\" --config \"%s\" " + "--disc \"%s\" --bios \"%s\" --json-progress", + g_python, g_cli_path, g_project_root, g_game_toml, source_path, + bios_path); + } else { + snprintf(cmdline, sizeof(cmdline), + "\"%s\" \"%s\" generate --project-root \"%s\" --config \"%s\" " + "--disc \"%s\" --json-progress", + g_python, g_cli_path, g_project_root, g_game_toml, source_path); + } + if (!run_cli_win(cmdline, on_progress, progress_ctx, err_msg, err_cap, + "psxrecomp generate")) + return 0; +#else + char* argv[16]; + int argc = 0; + argv[argc++] = g_python; + argv[argc++] = g_cli_path; + argv[argc++] = "generate"; + argv[argc++] = "--project-root"; + argv[argc++] = g_project_root; + argv[argc++] = "--config"; + argv[argc++] = g_game_toml; + argv[argc++] = "--disc"; + argv[argc++] = (char*)source_path; + if (have_bios) { + argv[argc++] = "--bios"; + argv[argc++] = bios_path; + } + argv[argc++] = "--json-progress"; + argv[argc] = NULL; + if (!run_cli_posix(argv, on_progress, progress_ctx, err_msg, err_cap, + "psxrecomp generate")) + return 0; +#endif + + snprintf(out_path, out_cap, "%s", source_path); + if (on_progress) + on_progress(progress_ctx, 1.0f, "Generate complete"); + return 1; +} + +#if defined(_WIN32) +static void bat_write_set(FILE* f, const char* name, const char* value) { + fprintf(f, "set \"%s=", name); + for (const char* p = value; *p; ++p) { + if (*p == '%') + fputc('%', f); + fputc(*p, f); + } + fprintf(f, "\"\r\n"); +} + +static int write_windows_deferred_rebuild_helper(char* err_msg, size_t err_cap) { + if (!join_path(g_helper_path, sizeof(g_helper_path), g_build_dir, + "recomp_deferred_rebuild.cmd")) { + snprintf(err_msg, err_cap, "Failed to form helper path."); + return 0; + } + /* Setup zips omit build-release/; create it before writing the .cmd. */ + if (!mkdir_p(g_build_dir)) { + snprintf(err_msg, err_cap, "Failed to create build dir: %s", + g_build_dir); + return 0; + } + FILE* f = fopen(g_helper_path, "wb"); + if (!f) { + snprintf(err_msg, err_cap, "Failed to write rebuild helper: %s", + g_helper_path); + return 0; + } + char pid_buf[32]; + snprintf(pid_buf, sizeof(pid_buf), "%lu", + (unsigned long)GetCurrentProcessId()); + fprintf(f, "@echo off\r\n"); + fprintf(f, "setlocal EnableExtensions\r\n"); + fprintf(f, "title %s - rebuilding\r\n", g_display); + bat_write_set(f, "PARENT_PID", pid_buf); + bat_write_set(f, "PYTHON", g_python); + bat_write_set(f, "CLI", g_cli_path); + bat_write_set(f, "ROOT", g_project_root); + bat_write_set(f, "CONFIG", g_game_toml); + bat_write_set(f, "BUILD_DIR", g_build_dir); + bat_write_set(f, "TARGET", g_cmake_target); + bat_write_set(f, "EXE_BASE", g_exe_basename); + bat_write_set(f, "EXE", g_exe_path); + bat_write_set(f, "DISPLAY", g_display); + if (g_toolchain_bin[0]) + bat_write_set(f, "TC_BIN", g_toolchain_bin); + fprintf(f, + "echo Waiting for %%DISPLAY%% to exit...\r\n" + ":waitloop\r\n" + "tasklist /FI \"PID eq %%PARENT_PID%%\" 2>NUL | " + "findstr /I \"%%PARENT_PID%%\" >NUL\r\n" + "if not errorlevel 1 (\r\n" + " ping -n 2 127.0.0.1 >NUL\r\n" + " goto waitloop\r\n" + ")\r\n" + "echo Ensuring toolchain...\r\n" + "cd /d \"%%ROOT%%\"\r\n" + "if defined TC_BIN set \"PATH=%%TC_BIN%%;%%PATH%%\"\r\n" + "\"%%PYTHON%%\" \"%%CLI%%\" ensure-toolchain --project-root \"%%ROOT%%\"\r\n" + "if errorlevel 1 (\r\n" + " echo.\r\n" + " echo Toolchain missing. Download cmake-clang-v1 or set\r\n" + " echo PSXRECOMP_TOOLCHAIN_DIR / pass --toolchain-zip on rebuild.\r\n" + " pause\r\n" + " exit /b 1\r\n" + ")\r\n" + "echo Building...\r\n" + "\"%%PYTHON%%\" \"%%CLI%%\" rebuild --project-root \"%%ROOT%%\" " + "--config \"%%CONFIG%%\" --build-dir \"%%BUILD_DIR%%\" " + "--target \"%%TARGET%%\" --exe-basename \"%%EXE_BASE%%\" " + "--no-pgo --prune-after build-intermediates\r\n" + "if errorlevel 1 (\r\n" + " echo.\r\n" + " echo Build failed. Fix the errors above, then rebuild manually.\r\n" + " pause\r\n" + " exit /b 1\r\n" + ")\r\n" + "echo Starting %%DISPLAY%%...\r\n" + "start \"\" /D \"%%ROOT%%\" \"%%EXE%%\" --launcher\r\n" + "endlocal\r\n"); + fclose(f); + return 1; +} +#endif + +static int host_rebuild_game(const char* disc_path, char* out_exe_path, + size_t out_cap, char* err_msg, size_t err_cap, + RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx) { + g_relaunch_is_helper = 0; + if (!g_ready || !g_build_dir[0]) { + snprintf(err_msg, err_cap, "CMake build environment is not available."); + return 0; + } + + if (!host_ensure_toolchain(on_progress, progress_ctx, err_msg, err_cap)) + return 0; + +#if defined(_WIN32) + (void)disc_path; + activate_toolchain_path(); + if (on_progress) + on_progress(progress_ctx, 0.4f, + "Scheduling Windows rebuild after exit…"); + if (!write_windows_deferred_rebuild_helper(err_msg, err_cap)) + return 0; + g_relaunch_is_helper = 1; + snprintf(out_exe_path, out_cap, "%s", g_helper_path); + if (on_progress) + on_progress(progress_ctx, 1.0f, + "Exiting so Windows can rebuild safely…"); + return 1; +#else + if (on_progress) + on_progress(progress_ctx, 0.05f, "Starting rebuild (cmake)…"); + + activate_toolchain_path(); + + char disc_arg_storage[1100]; + char* argv[36]; + int argc = 0; + argv[argc++] = g_python; + argv[argc++] = g_cli_path; + argv[argc++] = "rebuild"; + argv[argc++] = "--project-root"; + argv[argc++] = g_project_root; + argv[argc++] = "--config"; + argv[argc++] = g_game_toml; + argv[argc++] = "--build-dir"; + argv[argc++] = g_build_dir; + argv[argc++] = "--target"; + argv[argc++] = g_cmake_target; + argv[argc++] = "--exe-basename"; + argv[argc++] = g_exe_basename; + if (disc_path && disc_path[0]) { + snprintf(disc_arg_storage, sizeof(disc_arg_storage), "%s", disc_path); + argv[argc++] = "--disc"; + argv[argc++] = disc_arg_storage; + } + argv[argc++] = "--no-pgo"; + argv[argc++] = "--prune-after"; + argv[argc++] = "build-intermediates"; + argv[argc++] = "--json-progress"; + argv[argc] = NULL; + + if (!run_cli_posix(argv, on_progress, progress_ctx, err_msg, err_cap, + "psxrecomp rebuild")) + return 0; + if (!path_is_file(g_exe_path)) { + snprintf(err_msg, err_cap, "Build succeeded but binary missing: %s", + g_exe_path); + return 0; + } + snprintf(out_exe_path, out_cap, "%s", g_exe_path); + if (on_progress) + on_progress(progress_ctx, 1.0f, "Build complete"); + return 1; +#endif +} + +void psxrecomp_codegen_host_relaunch_or_exit(const char* disc_path) { + char exe[512]; + if (!recomp_launcher_relaunch_exe(exe, sizeof(exe)) || !exe[0]) { + fprintf(stderr, "psxrecomp-codegen: relaunch requested but no path\n"); + exit(1); + } + if (disc_path && disc_path[0]) { + FILE* rc = fopen("disc.cfg", "w"); + if (rc) { + fprintf(rc, "%s\n", disc_path); + fclose(rc); + } + } + +#if defined(_WIN32) + { + STARTUPINFOA si; + PROCESS_INFORMATION pi; + char cmd[1536]; + DWORD flags = 0; + memset(&si, 0, sizeof(si)); + memset(&pi, 0, sizeof(pi)); + si.cb = sizeof(si); + if (g_relaunch_is_helper) { + fprintf(stderr, + "psxrecomp-codegen: starting deferred rebuild helper\n"); + snprintf(cmd, sizeof(cmd), "cmd.exe /C \"%s\"", exe); + flags = CREATE_NEW_CONSOLE; + } else { + fprintf(stderr, "psxrecomp-codegen: relaunching %s\n", exe); + snprintf(cmd, sizeof(cmd), "\"%s\" --launcher", exe); + } + if (!CreateProcessA(NULL, cmd, NULL, NULL, FALSE, flags, NULL, + g_project_root, &si, &pi)) { + fprintf(stderr, "psxrecomp-codegen: CreateProcess failed\n"); + exit(1); + } + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + ExitProcess(0); + } +#else + { + fprintf(stderr, "psxrecomp-codegen: relaunching %s\n", exe); + char* args[] = {exe, "--launcher", NULL}; + execv(exe, args); + perror("psxrecomp-codegen: execv failed"); + exit(1); + } +#endif +} + +void psxrecomp_codegen_host_apply(RecompLauncherCGameInfo* gi, + const PsxrecompCodegenHostConfig* cfg) { + if (!gi || !cfg || !cfg->cmake_target || !cfg->exe_basename) + return; + + g_cfg = cfg; + g_ready = 0; + g_relaunch_is_helper = 0; + g_project_root[0] = '\0'; + g_cli_path[0] = '\0'; + g_game_toml[0] = '\0'; + g_python[0] = '\0'; + g_cmake[0] = '\0'; + g_build_dir[0] = '\0'; + g_exe_path[0] = '\0'; + g_helper_path[0] = '\0'; + g_toolchain_bin[0] = '\0'; + g_cli_toolchain_bin[0] = '\0'; + + snprintf(g_display, sizeof(g_display), "%s", + cfg_or(cfg->display_name, "Game")); + snprintf(g_cmake_target, sizeof(g_cmake_target), "%s", cfg->cmake_target); + snprintf(g_exe_basename, sizeof(g_exe_basename), "%s", cfg->exe_basename); + + const char* force_env = + cfg_or(cfg->force_setup_env, "PSXRECOMP_FORCE_SETUP"); + const char* force = getenv(force_env); + const int force_setup = force && force[0] && force[0] != '0'; + + if (!discover_project_root(g_project_root, sizeof(g_project_root))) { + /* Still force the wizard when generated/ is missing — discover may + * fail if the process cwd is unrelated to the project tree. */ + if (force_setup) { + gi->needs_setup = 1; + gi->prepare_required_before_continue = 1; + } + return; + } + if (!resolve_cli_path(g_project_root, g_cli_path, sizeof(g_cli_path))) { + if (psxrecomp_codegen_host_sources_missing(cfg) || force_setup) { + gi->needs_setup = 1; + gi->prepare_required_before_continue = 1; + } + return; + } + if (!join_path(g_game_toml, sizeof(g_game_toml), g_project_root, + cfg_or(cfg->game_toml_relpath, "game.toml"))) + return; + if (!path_is_file(g_game_toml)) + return; + if (!find_python(g_python, sizeof(g_python))) { + if (psxrecomp_codegen_host_sources_missing(cfg) || force_setup) { + gi->needs_setup = 1; + gi->prepare_required_before_continue = 1; + } + return; + } + + g_ready = 1; + activate_toolchain_path(); + gi->prepare_with_progress = host_prepare_generate; + gi->prepare_use_selected_rom = 1; + /* Number prefix is applied in the setup UI (BIOS adds a step). */ + gi->prepare_section_title = "Generate BIOS + game C & rebuild"; + gi->prepare_busy_status = "Generating BIOS + game sources…"; + gi->prepare_success_status = "Sources ready — building…"; + + /* Rebuild is offered whenever the build tree can be formed; the wizard + * installs cmake-clang-v1 on page 0 before Generate & rebuild. */ + const int can_rebuild = resolve_build_paths(); + if (can_rebuild) { + gi->prepare_disc_label = "Generate & rebuild…"; +#if defined(_WIN32) + gi->prepare_disc_note = + cfg->prepare_note_windows + ? cfg->prepare_note_windows + : "Uses your disc with the local psxrecomp SDK to regenerate " + "generated/, then quits and rebuilds via a helper so the " + "running .exe is not locked."; + gi->rebuild_busy_status = "Scheduling rebuild…"; + gi->rebuild_success_status = + "Exiting for Windows rebuild — a console will finish the build…"; +#else + gi->prepare_disc_note = + cfg->prepare_note + ? cfg->prepare_note + : "Uses your disc with the local psxrecomp SDK to regenerate " + "generated/, then runs cmake --build and restarts."; + gi->rebuild_busy_status = "Building…"; + gi->rebuild_success_status = "Build complete — restarting…"; +#endif + gi->rebuild_with_progress = host_rebuild_game; + gi->rebuild_after_prepare = 1; + gi->relaunch_after_rebuild = 1; + gi->setup_needs_toolchain = 1; + gi->toolchain_is_ready = host_toolchain_is_ready; + gi->ensure_toolchain_with_progress = host_ensure_toolchain_with_progress; + } else { + gi->prepare_disc_label = "Generate sources…"; + gi->prepare_disc_note = + cfg->prepare_note_no_cmake + ? cfg->prepare_note_no_cmake + : "Regenerates generated/ with the local psxrecomp SDK. " + "Build dir could not be resolved — rebuild manually, then " + "relaunch."; + gi->prepare_success_status = + "Sources generated. Rebuild manually, then relaunch."; + } + + if (psxrecomp_codegen_host_sources_missing(cfg) || force_setup) { + gi->needs_setup = 1; + gi->prepare_required_before_continue = 1; + } +} diff --git a/host/psxrecomp_codegen_host.h b/host/psxrecomp_codegen_host.h new file mode 100644 index 00000000..3068f042 --- /dev/null +++ b/host/psxrecomp_codegen_host.h @@ -0,0 +1,55 @@ +/* Portable recomp-ui setup host: disc → generate → cmake/PGO rebuild → relaunch. + * + * Games compile psxrecomp_codegen_host.c, fill PsxrecompCodegenHostConfig, + * and call psxrecomp_codegen_host_apply() when building RecompLauncherCGameInfo. + * + * Requires recomp_launcher.h on the include path (recomp-ui submodule). + */ +#ifndef PSXRECOMP_CODEGEN_HOST_H +#define PSXRECOMP_CODEGEN_HOST_H + +#include "recomp_launcher.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct PsxrecompCodegenHostConfig { + const char* display_name; + + /* Env vars (optional). Defaults: PSXRECOMP_PROJECT_ROOT / PSXRECOMP_BUILD_DIR / + * PSXRECOMP_FORCE_SETUP when NULL. */ + const char* project_root_env; + const char* build_dir_env; + const char* force_setup_env; + + /* Paths relative to project root. NULL → defaults below. */ + const char* psxrecomp_cli_relpath; /* default: psxrecomp/psxrecomp_cli.py */ + const char* seed_cfg_relpath; /* default: game.toml (root probe) */ + const char* game_toml_relpath; /* default: game.toml */ + const char* gen_marker_relpath; /* default: generated/SLUS_011.89_dispatch.c */ + const char* build_dir_name; /* default: build */ + + /* CMake / binary identity (required for auto-rebuild). */ + const char* cmake_target; /* e.g. psx-runtime */ + const char* exe_basename; /* no .exe */ + + /* Optional UI copy overrides (NULL → generic defaults). */ + const char* prepare_note; + const char* prepare_note_windows; + const char* prepare_note_no_cmake; +} PsxrecompCodegenHostConfig; + +void psxrecomp_codegen_host_apply(RecompLauncherCGameInfo* gi, + const PsxrecompCodegenHostConfig* cfg); + +int psxrecomp_codegen_host_sources_missing( + const PsxrecompCodegenHostConfig* cfg); + +void psxrecomp_codegen_host_relaunch_or_exit(const char* disc_path); + +#ifdef __cplusplus +} +#endif + +#endif /* PSXRECOMP_CODEGEN_HOST_H */ diff --git a/lib/recomp-net b/lib/recomp-net index 23eec611..8b31aad3 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit 23eec611eb151709b9a6a60d913027eb7f5b4346 +Subproject commit 8b31aad3a782e1294352e5618757b340974186fc diff --git a/psxrecomp_cli.py b/psxrecomp_cli.py new file mode 100755 index 00000000..6c2d9f2e --- /dev/null +++ b/psxrecomp_cli.py @@ -0,0 +1,1342 @@ +#!/usr/bin/env python3 +"""Headless disc → generate / rebuild / optional PGO for psxrecomp games. + +Commands: + verify-disc Hash-check a dump against game.toml [prepare_disc] + generate Prepare disc (if needed) + run psxrecomp-game → generated/ + rebuild cmake --build; if [pgo] enabled, instrument → train → use + pgo-train Standalone PGO train (same as rebuild's PGO phase) + +Exit codes: 0 ok · 1 runtime · 2 usage · 3 disc verify fail +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Optional + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT / "tools")) +from sdk_progress import ProgressReporter # noqa: E402 +from toolchain_pack import ( # noqa: E402 + ensure_toolchain as _ensure_toolchain_pack, + resolve_toolchain_bin, +) + +EXIT_OK = 0 +EXIT_ERROR = 1 +EXIT_USAGE = 2 +EXIT_VERIFY = 3 + + +def resolve_embedded_toolchain_bin(project_root: Path) -> Optional[Path]: + """Return a usable toolchain bin/ (project, env, or shared cache).""" + return resolve_toolchain_bin(project_root) + + +def activate_embedded_toolchain( + project_root: Path, progress: Optional[ProgressReporter] = None +) -> bool: + """Prepend resolved toolchain bin/ to PATH for cmake/ninja/clang.""" + bin_dir = resolve_toolchain_bin(project_root) + if not bin_dir: + return False + from toolchain_pack import activate_toolchain_bin + + activate_toolchain_bin( + bin_dir, log=(progress.log if progress else None) + ) + return True + + +def ensure_toolchain_for_rebuild( + project_root: Path, + progress: ProgressReporter, + *, + from_zip: str = "", + download: bool = True, +) -> bool: + """Ensure cmake is available via cache / download / offline zip.""" + try: + _ensure_toolchain_pack( + project_root, + from_zip=Path(from_zip) if from_zip else None, + download=download and not from_zip, + log=progress.log, + ) + return True + except Exception as exc: # noqa: BLE001 — surface to progress UI + progress.log(f"Toolchain ensure: {exc}") + return False + + +def prune_after_rebuild( + project_root: Path, + build_dir: Path, + modes: set[str], + progress: ProgressReporter, +) -> None: + """Free disk after a successful rebuild (wizard / one-shot setup).""" + if not modes: + return + if "toolchain" in modes or "all" in modes: + tc = project_root / "toolchain" + if tc.is_dir(): + shutil.rmtree(tc, ignore_errors=True) + progress.log(f"Pruned {tc}") + if "build-intermediates" in modes or "all" in modes: + if build_dir.is_dir(): + for name in ("CMakeFiles", ".ninja_deps", ".ninja_log", "CMakeCache.txt", + "cmake_install.cmake", "build.ninja", "compile_commands.json"): + p = build_dir / name + if p.is_dir(): + shutil.rmtree(p, ignore_errors=True) + elif p.is_file(): + try: + p.unlink() + except OSError: + pass + # Drop object/lib digests but keep the launch binary + assets/. + for p in build_dir.rglob("*"): + if not p.is_file(): + continue + if p.suffix in {".o", ".obj", ".a", ".lib", ".pdb", ".ilk", ".exp"}: + try: + p.unlink() + except OSError: + pass + progress.log(f"Pruned build intermediates under {build_dir}") + if "build-tree" in modes: + # Keep only the executable + assets next to it, then wipe the rest of + # the build dir by moving keepers aside — used by aggressive cleanup. + if build_dir.is_dir(): + keep_names = set() + for p in build_dir.iterdir(): + if p.is_file() and os.access(p, os.X_OK): + keep_names.add(p.name) + if p.name.lower().endswith(".exe"): + keep_names.add(p.name) + assets = build_dir / "assets" + staging = build_dir.parent / f".prune-keep-{build_dir.name}" + if staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + staging.mkdir(parents=True, exist_ok=True) + for name in keep_names: + src = build_dir / name + if src.is_file(): + shutil.move(str(src), str(staging / name)) + if assets.is_dir(): + shutil.move(str(assets), str(staging / "assets")) + shutil.rmtree(build_dir, ignore_errors=True) + staging.rename(build_dir) + progress.log(f"Pruned build tree to binary+assets under {build_dir}") + + +def clamp_future_mtimes( + root: Path, + *, + skip: Optional[Path] = None, + now: Optional[float] = None, +) -> int: + """Clamp mtimes ahead of *now* so Ninja does not infinite-reconfigure. + + Release zips often preserve CI clocks that are slightly ahead of a user's + clock (timezone / skew). Ninja then treats every source as newer than + ``build.ninja``, re-runs CMake forever, and fails with + ``manifest 'build.ninja' still dirty after 100 tries``. + """ + if not root.is_dir(): + return 0 + stamp = time.time() if now is None else now + skip_res: Optional[Path] = None + if skip is not None: + try: + skip_res = skip.resolve() + except OSError: + skip_res = skip + n = 0 + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + dpath = Path(dirpath) + try: + d_res = dpath.resolve() + except OSError: + d_res = dpath + # Skip the active build tree entirely (outputs are rewritten anyway). + if skip_res is not None and ( + d_res == skip_res or skip_res in d_res.parents + ): + dirnames[:] = [] + continue + # Never descend into VCS metadata; prune the build dir at the parent. + pruned: list[str] = [] + for x in dirnames: + if x == ".git": + continue + if skip_res is not None: + try: + if (dpath / x).resolve() == skip_res: + continue + except OSError: + pass + pruned.append(x) + dirnames[:] = pruned + for name in filenames: + p = dpath / name + try: + mtime = p.stat().st_mtime + except OSError: + continue + if mtime > stamp: + try: + os.utime(p, (stamp, stamp), follow_symlinks=False) + n += 1 + except OSError: + pass + return n + + +def _parse_array_items(inner: str) -> list[Any]: + items: list[Any] = [] + if not inner.strip(): + return items + for part in re.split(r",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", inner): + part = part.strip().rstrip(",").strip() + if not part: + continue + if part.startswith('"') and part.endswith('"'): + items.append(part[1:-1]) + else: + try: + items.append(int(part, 0)) + except ValueError: + items.append(part) + return items + + +def parse_toml_simple(text: str) -> dict[str, dict[str, Any]]: + sections: dict[str, dict[str, Any]] = {"": {}} + cur = "" + array_key: Optional[str] = None + array_buf: list[str] = [] + for raw in text.splitlines(): + line = raw.split("#", 1)[0].strip() + if array_key is not None: + array_buf.append(line) + joined = " ".join(array_buf) + if "]" in joined: + inner = joined.split("]", 1)[0] + if inner.startswith("["): + inner = inner[1:] + sections[cur][array_key] = _parse_array_items(inner) + array_key = None + array_buf = [] + continue + if not line: + continue + if line.startswith("[") and line.endswith("]"): + cur = line[1:-1].strip() + sections.setdefault(cur, {}) + continue + if "=" not in line: + continue + key, val = line.split("=", 1) + key, val = key.strip(), val.strip() + if val.startswith("["): + if val.endswith("]") and val.count("[") == val.count("]"): + sections[cur][key] = _parse_array_items(val[1:-1]) + else: + array_key = key + array_buf = [val[1:]] + continue + if val.startswith('"') and val.endswith('"'): + sections[cur][key] = val[1:-1] + elif val.lower() in ("true", "false"): + sections[cur][key] = val.lower() == "true" + else: + try: + sections[cur][key] = int(val, 0) + except ValueError: + sections[cur][key] = val + return sections + + +def file_hashes(path: Path) -> tuple[str, str, int]: + h_md5, h_sha1 = hashlib.md5(), hashlib.sha1() + size = 0 + with open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + h_md5.update(chunk) + h_sha1.update(chunk) + return h_md5.hexdigest(), h_sha1.hexdigest(), size + + +def resolve_cue_bin(cue_path: Path) -> Path: + text = cue_path.read_text(encoding="utf-8", errors="replace") + m = re.search(r'FILE\s+"([^"]+)"\s+BINARY', text, re.I) + if not m: + m = re.search(r"FILE\s+(\S+)\s+BINARY", text, re.I) + if not m: + raise ValueError(f"no BINARY FILE in cue: {cue_path}") + cand = Path(m.group(1)) + if not cand.is_absolute(): + cand = cue_path.parent / cand + if not cand.is_file(): + raise ValueError(f"cue references missing bin: {cand}") + return cand + + +def _find_recompiler_tool(project_root: Path, basename: str, env_name: str) -> Path: + env = os.environ.get(env_name, "").strip() + if env: + p = Path(env).expanduser() + if p.is_file(): + return p.resolve() + names = [basename, f"{basename}.exe"] + search_dirs = [ + project_root / "psxrecomp" / "recompiler" / "build", + project_root / "psxrecomp" / "recompiler" / "build" / "Release", + project_root / "build-recompiler", + ROOT / "recompiler" / "build", + ROOT / "recompiler" / "build" / "Release", + ] + for d in search_dirs: + for name in names: + c = d / name + if c.is_file(): + return c.resolve() + which = shutil.which(basename) + if which: + return Path(which).resolve() + raise FileNotFoundError( + f"{basename} not found. Build it:\n" + " cmake -S psxrecomp/recompiler -B psxrecomp/recompiler/build -G Ninja\n" + f" cmake --build psxrecomp/recompiler/build --target {basename}\n" + f"Or set {env_name}=/path/to/{basename}" + ) + + +def find_psxrecomp_game(project_root: Path) -> Path: + return _find_recompiler_tool(project_root, "psxrecomp-game", "PSXRECOMP_GAME") + + +def find_psxrecomp_bios(project_root: Path) -> Path: + return _find_recompiler_tool(project_root, "psxrecomp-bios", "PSXRECOMP_BIOS") + + +def framework_root(project_root: Path) -> Path: + """Directory containing bios/*.toml and recompiler/ (usually …/psxrecomp).""" + cand = project_root / "psxrecomp" + if (cand / "bios" / "OpenBIOS.toml").is_file() or ( + (cand / "bios").is_dir() and (cand / "recompiler").is_dir() + ): + if (cand / "bios").is_dir(): + return cand + if (ROOT / "bios").is_dir() and (ROOT / "recompiler").is_dir(): + return ROOT + return cand + + +def _copy_missing(src: Path, dest: Path) -> None: + if src.is_dir(): + dest.mkdir(parents=True, exist_ok=True) + for child in src.iterdir(): + _copy_missing(child, dest / child.name) + return + if not src.is_file(): + return + if dest.is_file(): + return + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + + +def ensure_framework( + project_root: Path, *, progress: Optional[ProgressReporter] = None +) -> Path: + """Ensure project_root/psxrecomp has BIOS profiles + seeds for local generate. + + GitHub zipballs omit git submodules, so RetComM source trees often lack + psxrecomp/bios. Seed from the SDK pack that ships this CLI (ROOT). + """ + fw = project_root / "psxrecomp" + need_seed = not (fw / "bios" / "OpenBIOS.toml").is_file() + if need_seed and (ROOT / "bios").is_dir(): + if progress: + progress.log( + f"Seeding psxrecomp BIOS profiles from SDK -> {fw}" + ) + fw.mkdir(parents=True, exist_ok=True) + _copy_missing(ROOT / "bios", fw / "bios") + seeds_src = ROOT / "recompiler" / "seeds" + if seeds_src.is_dir(): + _copy_missing(seeds_src, fw / "recompiler" / "seeds") + (fw / "recompiler" / "build").mkdir(parents=True, exist_ok=True) + # psxrecomp-bios walks up from bios/*.toml looking for .gitignore/.git/ + # CMakeLists.txt to find the framework root. Zipball trees lack the + # submodule .git — plant a marker so rom = "bios/openbios.bin" resolves. + if (fw / "bios").is_dir() and not any( + (fw / m).exists() for m in (".gitignore", ".git", "CMakeLists.txt") + ): + marker = fw / ".gitignore" + if not marker.is_file(): + marker.write_text( + "# RetComM SDK seed marker (project-root for psxrecomp-bios)\n", + encoding="utf-8", + ) + return framework_root(project_root) + + +def bios_backend_present(fw: Path, stem: str) -> bool: + dispatch = fw / "generated" / f"{stem}_dispatch.c" + full = fw / "generated" / f"{stem}_full.c" + if not dispatch.is_file() or not full.is_file(): + return False + try: + text = dispatch.read_text(encoding="utf-8", errors="ignore") + except OSError: + return False + return f"{stem}_psx_bios_backend" in text + + +def regen_bios_profile( + project_root: Path, + profile_rel: str, + *, + progress: ProgressReporter, +) -> None: + fw = framework_root(project_root) + profile = fw / profile_rel + if not profile.is_file(): + raise FileNotFoundError(f"BIOS profile not found: {profile}") + bios_tool = find_psxrecomp_bios(project_root) + progress.log(f"regen BIOS via {bios_tool.name} --config {profile_rel}") + (fw / "generated").mkdir(parents=True, exist_ok=True) + # Pass a framework-relative config path and cwd=fw so toml paths like + # rom = "bios/openbios.bin" resolve under the framework root (not bios/bios/). + proc = subprocess.run( + [str(bios_tool), "--config", profile_rel], + cwd=str(fw), + capture_output=True, + text=True, + ) + for stream in (proc.stdout, proc.stderr): + if not stream: + continue + for line in stream.splitlines(): + if line.strip(): + progress.log(line) + if proc.returncode != 0: + raise RuntimeError( + f"psxrecomp-bios failed for {profile_rel} (exit {proc.returncode})" + ) + + +def load_sections(config: Path) -> dict[str, dict[str, Any]]: + return parse_toml_simple(config.read_text(encoding="utf-8")) + + +def verify_disc_path( + disc: Path, + prep: dict[str, Any], + *, + skip_hash: bool, + progress: ProgressReporter, +) -> dict[str, Any]: + path = disc.resolve() + if path.suffix.lower() == ".cue": + path = resolve_cue_bin(path) + md5, sha1, size = file_hashes(path) + identity = { + "path": str(path), + "md5": md5, + "sha1": sha1, + "size": size, + "verified": False, + } + progress.event("disc", **identity) + sizes = [int(s) for s in (prep.get("known_sizes") or [])] + md5s = [str(x).lower() for x in (prep.get("known_md5") or [])] + sha1s = [str(x).lower() for x in (prep.get("known_sha1") or [])] + if not md5s and not sha1s and not sizes: + identity["verified"] = True + return identity + if skip_hash: + return identity + ok = (md5 in md5s) or (sha1 in sha1s) + if not ok and sizes and size in sizes and not md5s and not sha1s: + ok = True + if not ok: + raise DiscVerifyError( + f"disc digests not in prepare_disc.known_* " + f"(size={size} md5={md5} sha1={sha1})" + ) + identity["verified"] = True + return identity + + +class DiscVerifyError(Exception): + pass + + +def cmd_verify_disc(args: argparse.Namespace, progress: ProgressReporter) -> int: + config = Path(args.config).expanduser().resolve() + if not config.is_file(): + progress.error(f"config not found: {config}", code=EXIT_USAGE) + return EXIT_USAGE + project_root = ( + Path(args.project_root).expanduser().resolve() + if args.project_root + else config.parent + ) + activate_embedded_toolchain(project_root, progress) + disc = Path(args.disc).expanduser() + if not disc.is_absolute(): + disc = (project_root / disc).resolve() + else: + disc = disc.resolve() + if not disc.is_file(): + progress.error(f"disc not found: {disc}", code=EXIT_USAGE) + return EXIT_USAGE + secs = load_sections(config) + prep = secs.get("prepare_disc") or {} + progress.phase("verify", pct=0.1, message=f"Verifying {disc.name}") + try: + identity = verify_disc_path( + disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress + ) + except DiscVerifyError as exc: + progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True) + return EXIT_VERIFY + progress.phase("done", pct=1.0, message="Disc OK") + progress.result(ok=True, **identity) + return EXIT_OK + + +def run_prepare_disc( + project_root: Path, + config: Path, + source: Path, + progress: ProgressReporter, +) -> Path: + script = ROOT / "tools" / "prepare_disc.py" + if not script.is_file(): + raise RuntimeError(f"missing {script}") + progress.phase("prepare_disc", pct=0.15, message="Normalizing disc image...") + cmd = [ + sys.executable, + str(script), + "--config", + str(config), + "--project-root", + str(project_root), + str(source), + ] + proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True) + out = (proc.stdout or "") + (proc.stderr or "") + for line in out.splitlines(): + if line.strip(): + progress.log(line) + if proc.returncode != 0: + raise RuntimeError(f"prepare_disc failed (exit {proc.returncode})") + marker = "RESULT_CUE=" + cue = None + for line in out.splitlines(): + if line.startswith(marker): + cue = Path(line[len(marker) :].strip()) + if not cue or not cue.is_file(): + raise RuntimeError("prepare_disc did not print RESULT_CUE=") + return cue.resolve() + + +def cmd_generate(args: argparse.Namespace, progress: ProgressReporter) -> int: + config = Path(args.config).expanduser().resolve() + if not config.is_file(): + progress.error(f"config not found: {config}", code=EXIT_USAGE) + return EXIT_USAGE + project_root = ( + Path(args.project_root).expanduser().resolve() + if args.project_root + else config.parent + ) + activate_embedded_toolchain(project_root, progress) + secs = load_sections(config) + game = secs.get("game") or {} + prep = secs.get("prepare_disc") or {} + recomp = secs.get("recompiler") or {} + runtime = secs.get("runtime") or {} + openbios_allowed = bool(runtime.get("openbios", True)) + + disc_arg = args.disc + if not disc_arg: + disc_arg = game.get("disc") or "" + if not disc_arg: + progress.error("no --disc and game.disc empty", code=EXIT_USAGE) + return EXIT_USAGE + disc = Path(str(disc_arg)).expanduser() + if not disc.is_absolute(): + disc = (project_root / disc).resolve() + else: + disc = disc.resolve() + + progress.phase("verify", pct=0.05, message=f"Checking disc {disc.name}") + try: + if disc.is_file(): + verify_disc_path( + disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress + ) + except DiscVerifyError as exc: + progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True) + return EXIT_VERIFY + + boot = str(prep.get("boot_exe") or Path(str(game.get("exe") or "")).name) + out_rel = str(prep.get("out_dir") or "prepared_disc") + boot_path = project_root / out_rel / boot + working_disc = disc + # Normalize library dumps when boot EXE missing or source looks like ISO/raw. + need_prep = (not boot_path.is_file()) or disc.suffix.lower() in ( + ".iso", + ".ISO", + ) + if need_prep or args.force_prepare: + try: + working_disc = run_prepare_disc(project_root, config, disc, progress) + except Exception as exc: # noqa: BLE001 + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + if not boot_path.is_file(): + progress.error(f"boot EXE missing after prepare: {boot_path}", code=EXIT_ERROR) + return EXIT_ERROR + + # ---- BIOS backends (local only; CI ships none) ---- + fw = ensure_framework(project_root, progress=progress) + bios_arg = (getattr(args, "bios", None) or "").strip() + staged_retail = False + if bios_arg: + bios_path = Path(bios_arg).expanduser() + if not bios_path.is_absolute(): + bios_path = (project_root / bios_path).resolve() + else: + bios_path = bios_path.resolve() + if not bios_path.is_file(): + progress.error(f"BIOS not found: {bios_path}", code=EXIT_USAGE) + return EXIT_USAGE + dest = fw / "bios" / "SCPH1001.BIN" + progress.phase("bios", pct=0.15, message="Staging retail BIOS dump...") + try: + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.resolve() != bios_path.resolve(): + shutil.copy2(bios_path, dest) + except OSError as exc: + progress.error(f"failed to stage BIOS: {exc}", code=EXIT_ERROR) + return EXIT_ERROR + try: + progress.phase("bios", pct=0.2, message="Generating SCPH1001 BIOS C...") + regen_bios_profile(project_root, "bios/SCPH1001.toml", progress=progress) + staged_retail = True + except Exception as exc: # noqa: BLE001 + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + elif not openbios_allowed and not bios_backend_present(fw, "SCPH1001"): + progress.error( + "This title requires a retail BIOS dump. Pass --bios SCPH1001.BIN " + "(or pick one in the setup wizard).", + code=EXIT_USAGE, + ) + return EXIT_USAGE + + if openbios_allowed and ( + args.force_bios or not bios_backend_present(fw, "OpenBIOS") + ): + try: + progress.phase( + "bios", pct=0.25, message="Generating OpenBIOS backend C..." + ) + regen_bios_profile(project_root, "bios/OpenBIOS.toml", progress=progress) + except Exception as exc: # noqa: BLE001 + # Prefer retail when --bios was given; OpenBIOS is then best-effort. + if staged_retail: + progress.log(f"OpenBIOS regen skipped (retail BIOS ok): {exc}") + else: + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + + try: + game_tool = find_psxrecomp_game(project_root) + except FileNotFoundError as exc: + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + + out_dir = project_root / str(recomp.get("out_dir") or "generated") + progress.phase( + "emit", + pct=0.35, + message=f"Running psxrecomp-game -> {out_dir}", + ) + cmd = [str(game_tool), "--config", str(config)] + proc = subprocess.run( + cmd, + cwd=str(project_root), + capture_output=True, + text=True, + ) + for stream in (proc.stdout, proc.stderr): + if not stream: + continue + for line in stream.splitlines(): + if line.strip(): + progress.log(line) + if proc.returncode != 0: + progress.error( + f"psxrecomp-game failed (exit {proc.returncode})", code=EXIT_ERROR + ) + return EXIT_ERROR + + marker_name = args.gen_marker or f"{boot}_dispatch.c" + marker = out_dir / marker_name + if not marker.is_file(): + # Accept any *_dispatch.c + hits = list(out_dir.glob("*_dispatch.c")) + if not hits: + progress.error(f"generate produced no dispatch under {out_dir}", code=EXIT_ERROR) + return EXIT_ERROR + marker = hits[0] + + progress.phase("done", pct=1.0, message="Generate complete") + progress.result( + ok=True, + out_dir=str(out_dir), + marker=str(marker), + disc=str(working_disc), + boot_exe=str(boot_path), + ) + return EXIT_OK + + +def _cmake_configure( + project_root: Path, + build_dir: Path, + *, + pgo: str, + extra: list[str], + progress: ProgressReporter, +) -> None: + build_dir.mkdir(parents=True, exist_ok=True) + # Prefer Ninja when available; fall back so hosts without ninja still build. + if not (build_dir / "CMakeCache.txt").is_file() and shutil.which("ninja"): + gen = ["-G", "Ninja"] + else: + gen = [] + cmd = [ + "cmake", + "-S", + str(project_root), + "-B", + str(build_dir), + *gen, + "-DCMAKE_BUILD_TYPE=Release", + f"-DPSX_PGO={pgo}", + *extra, + ] + progress.log(" ".join(cmd)) + proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True) + for stream in (proc.stdout, proc.stderr): + if stream: + for line in stream.splitlines(): + if line.strip(): + progress.log(line) + if proc.returncode != 0: + raise RuntimeError(f"cmake configure failed (exit {proc.returncode})") + + +def _cmake_build( + build_dir: Path, + target: str, + progress: ProgressReporter, +) -> None: + jobs = os.environ.get("CMAKE_BUILD_PARALLEL_LEVEL") or str( + os.cpu_count() or 4 + ) + cmd = [ + "cmake", + "--build", + str(build_dir), + "--parallel", + jobs, + "--target", + target, + ] + progress.log(" ".join(cmd)) + proc = subprocess.run(cmd, capture_output=True, text=True) + for stream in (proc.stdout, proc.stderr): + if stream: + for line in stream.splitlines(): + if line.strip(): + progress.log(line) + if proc.returncode != 0: + err = f"cmake --build failed (exit {proc.returncode})" + combined = "\n".join( + s for s in (proc.stdout or "", proc.stderr or "") if s + ) + if "still dirty after" in combined: + err += ( + " — Ninja reconfigure loop (often release-zip mtimes ahead of " + "the system clock). Re-run after the CLI clamps future mtimes, " + "or touch the project tree / fix the clock." + ) + raise RuntimeError(err) + + +def _soft_stop(pid: int, timeout: int = 30) -> None: + try: + os.kill(pid, 15) # SIGTERM + except ProcessLookupError: + return + for _ in range(timeout): + try: + os.kill(pid, 0) + except ProcessLookupError: + return + time.sleep(1) + try: + os.kill(pid, 9) + except ProcessLookupError: + pass + + +def _pgo_train_warning(*, hide_video: bool) -> str: + if hide_video: + return ( + "WARNING: PGO training is running (no video window). Do not cancel " + "or kill this process until training finishes — interrupting may " + "corrupt profiles and force a restart of this step." + ) + return ( + "WARNING: PGO training is running. Do not close the game window or " + "interact with the application until training finishes — closing or " + "clicking may corrupt profiles and force a restart of this step." + ) + + +def run_pgo_train( + project_root: Path, + build_dir: Path, + *, + exe_basename: str, + disc: Path, + config: Path, + train_secs: int, + train_runs: int, + mute_host_audio: bool, + hide_video: bool, + progress: ProgressReporter, +) -> None: + exe = build_dir / exe_basename + if not exe.is_file(): + exe = build_dir / f"{exe_basename}.exe" + if not exe.is_file(): + raise RuntimeError(f"train binary missing: {exe_basename} under {build_dir}") + + pgo_dir = build_dir / "pgo" + if pgo_dir.exists(): + shutil.rmtree(pgo_dir) + pgo_dir.mkdir(parents=True, exist_ok=True) + + env = os.environ.copy() + env.setdefault("PSX_BIOS_HLE", "0") + env["LLVM_PROFILE_FILE"] = str(pgo_dir / "bpe-%p.profraw") + # Discard SDL output; SPU/CD still advance (profiles stay valid). + if mute_host_audio: + env["PSX_HOST_MUTE"] = "1" + else: + env.pop("PSX_HOST_MUTE", None) + # No on-screen FMV (photosensitive / abrasive content). Guest MDEC/GPU + # still run; host present is skipped. Matches prior CI train shape. + if hide_video: + env["PSX_HEADLESS"] = "1" + env.setdefault("SDL_VIDEODRIVER", "dummy") + else: + env.pop("PSX_HEADLESS", None) + + warning = _pgo_train_warning(hide_video=hide_video) + progress.phase("pgo_train", pct=0.35, message=warning) + # Always surface on the console even when JSONL owns stdout. + print(f"\n*** {warning} ***\n", file=sys.stderr, flush=True) + if mute_host_audio: + progress.log("Host audio muted for train (PSX_HOST_MUTE=1; SPU still runs).") + else: + progress.log("Host audio unmuted for train (speakers will play).") + if hide_video: + progress.log( + "Video hidden for train (--headless / PSX_HEADLESS=1; " + "guest FMV decode still runs, nothing shown on screen)." + ) + else: + progress.log("Video window visible for train (host present paths included).") + + for run in range(1, train_runs + 1): + progress.phase( + "pgo_train", + pct=0.4 + 0.4 * (run - 1) / max(train_runs, 1), + message=( + f"WARNING: PGO train run {run}/{train_runs} ({train_secs}s) — " + + ( + "Do not cancel this process." + if hide_video + else "Do not close or interact with the application." + ) + ), + ) + cmd = [ + str(exe), + "--no-launcher", + "--game", + str(config), + "--disc", + str(disc), + ] + if hide_video: + cmd.append("--headless") + proc = subprocess.Popen( + cmd, + cwd=str(project_root), + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + time.sleep(train_secs) + if proc.poll() is None: + _soft_stop(proc.pid) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + + n_gcda = len(list(build_dir.rglob("*.gcda"))) + n_raw = len(list(pgo_dir.glob("*.profraw"))) + progress.log(f"PGO profiles: {n_gcda} .gcda, {n_raw} .profraw") + if n_raw >= 1: + merge = shutil.which("llvm-profdata") + if not merge: + # macOS xcrun + try: + r = subprocess.run( + ["xcrun", "--find", "llvm-profdata"], + capture_output=True, + text=True, + check=False, + ) + if r.returncode == 0 and r.stdout.strip(): + merge = "xcrun" + except OSError: + pass + raws = list(pgo_dir.glob("*.profraw")) + out = pgo_dir / "default.profdata" + if merge == "xcrun": + cmd = ["xcrun", "llvm-profdata", "merge", "-sparse", *[str(p) for p in raws], "-o", str(out)] + elif merge: + cmd = [merge, "merge", "-sparse", *[str(p) for p in raws], "-o", str(out)] + else: + raise RuntimeError("llvm-profdata not found (Clang PGO merge)") + subprocess.run(cmd, check=True) + if n_gcda < 1 and not (pgo_dir / "default.profdata").is_file(): + raise RuntimeError("no PGO profiles written — train did not flush") + + +def cmd_rebuild(args: argparse.Namespace, progress: ProgressReporter) -> int: + config = Path(args.config).expanduser().resolve() + if not config.is_file(): + progress.error(f"config not found: {config}", code=EXIT_USAGE) + return EXIT_USAGE + project_root = ( + Path(args.project_root).expanduser().resolve() + if args.project_root + else config.parent + ) + build_dir = Path(args.build_dir).expanduser() + if not build_dir.is_absolute(): + build_dir = (project_root / build_dir).resolve() + else: + build_dir = build_dir.resolve() + target = args.target or "psx-runtime" + exe_basename = args.exe_basename or "psx-runtime" + + secs = load_sections(config) + pgo = secs.get("pgo") or {} + # Opt-in only: requires game.toml [pgo] enabled = true (or --force-pgo). + pgo_enabled = bool(pgo.get("enabled", False)) and not args.no_pgo + if args.force_pgo: + pgo_enabled = True + + game = secs.get("game") or {} + prep = secs.get("prepare_disc") or {} + if args.disc: + disc = Path(args.disc).expanduser() + if not disc.is_absolute(): + disc = (project_root / disc).resolve() + else: + disc = disc.resolve() + else: + cue_name = prep.get("cue_name") + if cue_name: + disc = ( + project_root / str(prep.get("out_dir") or "bpe") / str(cue_name) + ).resolve() + elif game.get("disc"): + disc = (project_root / str(game["disc"])).resolve() + if disc.suffix.lower() == ".bin": + cue = disc.with_suffix(".cue") + if cue.is_file(): + disc = cue + else: + disc = None + + zip_arg = getattr(args, "toolchain_zip", "") or "" + no_dl = bool(getattr(args, "no_toolchain_download", False)) + if zip_arg: + if not ensure_toolchain_for_rebuild( + project_root, progress, from_zip=zip_arg, download=False + ): + progress.error( + f"Failed to install toolchain from zip: {zip_arg}", + code=EXIT_ERROR, + ) + return EXIT_ERROR + elif not activate_embedded_toolchain(project_root, progress): + if no_dl or not ensure_toolchain_for_rebuild( + project_root, progress, download=True + ): + if not (shutil.which("cmake") or shutil.which("cmake.exe")): + progress.error( + "No portable toolchain and no cmake on PATH. " + "Pass --toolchain-zip PATH, allow download, " + "or set PSXRECOMP_TOOLCHAIN_DIR / RETCOMM_TOOLCHAIN_DIR.", + code=EXIT_ERROR, + ) + return EXIT_ERROR + progress.log("Using system cmake on PATH") + + cmake_extra = [] + if args.cmake_extra: + cmake_extra.extend(args.cmake_extra) + # Full playable link after local generate (not the CI setup-host shape). + cmake_extra.append("-DBPE_FORCE_SETUP_HOST=OFF") + cmake_extra.append("-DPSXRECOMP_ALLOW_NO_BIOS=OFF") + + clamped = clamp_future_mtimes(project_root, skip=build_dir) + if clamped: + progress.log( + f"Clamped {clamped} future mtime(s) under {project_root} " + "(avoids Ninja dirty-manifest loop from release-zip clocks)." + ) + + # mute_host_audio / hide_video: default ON; game.toml / CLI can disable. + mute_host = True + if "mute_host_audio" in pgo: + mute_host = bool(pgo.get("mute_host_audio")) + if getattr(args, "pgo_audio", False): + mute_host = False + if getattr(args, "pgo_mute", False): + mute_host = True + + hide_video = True + if "hide_video" in pgo: + hide_video = bool(pgo.get("hide_video")) + if getattr(args, "pgo_video", False): + hide_video = False + if getattr(args, "pgo_hide_video", False): + hide_video = True + + try: + if not pgo_enabled: + progress.phase("build", pct=0.2, message="cmake Release build...") + _cmake_configure( + project_root, build_dir, pgo="", extra=cmake_extra, progress=progress + ) + _cmake_build(build_dir, target, progress) + else: + # Framework defaults (60×2); game.toml / CLI may lengthen for hard titles. + train_secs = int(getattr(args, "train_secs", 0) or 0) + if train_secs <= 0: + train_secs = int(pgo.get("train_secs") or 60) + train_runs = int(getattr(args, "train_runs", 0) or 0) + if train_runs <= 0: + train_runs = int(pgo.get("train_runs") or 2) + progress.phase( + "pgo_instrument", + pct=0.1, + message="PGO instrumented build...", + ) + _cmake_configure( + project_root, + build_dir, + pgo="generate", + extra=cmake_extra, + progress=progress, + ) + _cmake_build(build_dir, target, progress) + if not disc or not Path(disc).is_file(): + raise RuntimeError( + f"PGO train needs a playable disc (got {disc!s}). " + "Pass --disc or set game.disc / prepare_disc.cue_name." + ) + run_pgo_train( + project_root, + build_dir, + exe_basename=exe_basename, + disc=Path(disc), + config=config, + train_secs=train_secs, + train_runs=train_runs, + mute_host_audio=mute_host, + hide_video=hide_video, + progress=progress, + ) + progress.phase("pgo_use", pct=0.85, message="PGO optimized rebuild...") + _cmake_configure( + project_root, + build_dir, + pgo="use", + extra=cmake_extra, + progress=progress, + ) + _cmake_build(build_dir, target, progress) + except Exception as exc: # noqa: BLE001 + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + + exe = build_dir / exe_basename + if not exe.is_file(): + exe = build_dir / f"{exe_basename}.exe" + if not exe.is_file(): + progress.error(f"build succeeded but binary missing: {exe}", code=EXIT_ERROR) + return EXIT_ERROR + + prune_raw = (getattr(args, "prune_after", None) or "").strip() + if prune_raw: + modes = {m.strip() for m in prune_raw.split(",") if m.strip()} + progress.phase("prune", pct=0.97, message="Pruning toolchain / build bulk...") + prune_after_rebuild(project_root, build_dir, modes, progress) + + progress.phase("done", pct=1.0, message="Rebuild complete") + progress.result(ok=True, exe=str(exe), pgo=pgo_enabled) + return EXIT_OK + + +def cmd_pgo_train(args: argparse.Namespace, progress: ProgressReporter) -> int: + args.force_pgo = True + args.no_pgo = False + return cmd_rebuild(args, progress) + + +def cmd_ensure_toolchain(args: argparse.Namespace, progress: ProgressReporter) -> int: + """Resolve / download / unpack cmake-clang-v1 into project toolchain/ + cache.""" + project_root = ( + Path(args.project_root).expanduser().resolve() + if args.project_root + else Path.cwd().resolve() + ) + zip_arg = (args.from_zip or "").strip() + # Default: reuse cache/env, else download unless --no-download. + want_download = (not zip_arg) and not bool(getattr(args, "no_download", False)) + if getattr(args, "download", False): + want_download = not zip_arg + try: + bin_dir = _ensure_toolchain_pack( + project_root, + from_zip=Path(zip_arg) if zip_arg else None, + download=want_download, + log=progress.log, + ) + except Exception as exc: # noqa: BLE001 + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + progress.phase("done", pct=1.0, message=f"Toolchain ready: {bin_dir}") + progress.result(ok=True, toolchain_bin=str(bin_dir)) + return EXIT_OK + + +def build_parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser(prog="psxrecomp_cli", description=__doc__) + sub = ap.add_subparsers(dest="command", required=True) + + def add_common(p: argparse.ArgumentParser) -> None: + p.add_argument("--config", default="game.toml", help="game.toml path") + p.add_argument("--project-root", default="", help="game project root") + p.add_argument("--json-progress", action="store_true") + + v = sub.add_parser("verify-disc", help="verify disc digests") + add_common(v) + v.add_argument("--disc", required=True) + v.add_argument("--skip-hash-check", action="store_true") + v.set_defaults(handler=cmd_verify_disc) + + g = sub.add_parser( + "generate", help="regen BIOS backends + prepare disc + generate game C" + ) + add_common(g) + g.add_argument("--disc", default="", help="source dump or working cue/bin") + g.add_argument( + "--bios", + default="", + help="optional retail BIOS dump (staged as bios/SCPH1001.BIN + regen)", + ) + g.add_argument( + "--force-bios", + action="store_true", + help="regenerate OpenBIOS even if generated/ backends already exist", + ) + g.add_argument("--skip-hash-check", action="store_true") + g.add_argument("--force-prepare", action="store_true") + g.add_argument("--gen-marker", default="", help="expected marker under out_dir") + g.set_defaults(handler=cmd_generate) + + r = sub.add_parser("rebuild", help="cmake build (+ optional local PGO)") + add_common(r) + r.add_argument("--build-dir", required=True) + r.add_argument("--target", default="psx-runtime") + r.add_argument("--exe-basename", default="") + r.add_argument("--disc", default="", help="disc for PGO train") + r.add_argument( + "--no-pgo", + action="store_true", + help="skip PGO even if game.toml [pgo] enabled=true", + ) + r.add_argument( + "--force-pgo", + action="store_true", + help="run PGO even without [pgo] enabled=true", + ) + r.add_argument( + "--train-secs", + type=int, + default=0, + help="override [pgo] train_secs (default framework 60)", + ) + r.add_argument( + "--train-runs", + type=int, + default=0, + help="override [pgo] train_runs (default framework 2)", + ) + r.add_argument( + "--pgo-mute", + action="store_true", + help="force mute host speakers during train (default)", + ) + r.add_argument( + "--pgo-audio", + action="store_true", + help="allow host speakers during train (overrides mute default)", + ) + r.add_argument( + "--pgo-hide-video", + action="store_true", + help="force headless train with no on-screen video (default)", + ) + r.add_argument( + "--pgo-video", + action="store_true", + help="show a game window during train (includes host present paths)", + ) + r.add_argument( + "--prune-after", + default="", + help=( + "comma list after success: toolchain, build-intermediates, " + "build-tree, all (frees disk for one-shot wizard installs)" + ), + ) + r.add_argument( + "--toolchain-zip", + default="", + help="offline cmake-clang-v1-*.zip to install into the shared cache", + ) + r.add_argument( + "--no-toolchain-download", + action="store_true", + help="do not fetch cmake-clang-v1 when no local pack is found", + ) + r.add_argument("--cmake-extra", action="append", default=[]) + r.set_defaults(handler=cmd_rebuild) + + e = sub.add_parser( + "ensure-toolchain", + help="download or unpack cmake-clang-v1 into the shared cache", + ) + e.add_argument("--project-root", default="", help="game project root (optional)") + e.add_argument("--json-progress", action="store_true") + e.add_argument( + "--from-zip", + default="", + help="install from a local cmake-clang-v1-*.zip (offline)", + ) + e.add_argument( + "--download", + action="store_true", + help="force download even if a pack is already cached", + ) + e.add_argument( + "--no-download", + action="store_true", + help="only resolve env/cache/project toolchain/ (no network)", + ) + e.set_defaults(handler=cmd_ensure_toolchain) + + p = sub.add_parser("pgo-train", help="force PGO rebuild+train+use") + add_common(p) + p.add_argument("--build-dir", required=True) + p.add_argument("--target", default="psx-runtime") + p.add_argument("--exe-basename", default="") + p.add_argument("--disc", default="") + p.add_argument("--train-secs", type=int, default=0) + p.add_argument("--train-runs", type=int, default=0) + p.add_argument("--pgo-mute", action="store_true") + p.add_argument("--pgo-audio", action="store_true") + p.add_argument("--pgo-hide-video", action="store_true") + p.add_argument("--pgo-video", action="store_true") + p.add_argument("--cmake-extra", action="append", default=[]) + p.add_argument("--no-pgo", action="store_false", default=False) + p.add_argument("--force-pgo", action="store_true", default=True) + p.set_defaults(handler=cmd_pgo_train) + + return ap + + +def _configure_stdio() -> None: + """Best-effort: don't crash on Windows cp1252 when logging Unicode.""" + for stream in (sys.stdout, sys.stderr): + reconf = getattr(stream, "reconfigure", None) + if callable(reconf): + try: + reconf(errors="replace") + except Exception: # noqa: BLE001 + pass + + +def main() -> int: + _configure_stdio() + args = build_parser().parse_args() + progress = ProgressReporter(json_progress=bool(getattr(args, "json_progress", False))) + try: + return int(args.handler(args, progress)) + except BrokenPipeError: + return EXIT_ERROR + except KeyboardInterrupt: + progress.error("interrupted", code=EXIT_ERROR) + return EXIT_ERROR + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index b33681c5..97015d0e 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -203,13 +203,19 @@ elseif(MSVC) target_link_options(psxrecomp-bios PRIVATE /STACK:16777216) endif() -# Released Windows CLI archives must run without a separate MinGW install. -# Keep this opt-in so ordinary developer/debug builds retain their normal -# linker behavior on every platform. -option(PSXRECOMP_STATIC_CLI "Statically link MinGW runtime libraries into CLI tools" OFF) -if(PSXRECOMP_STATIC_CLI AND MINGW) +# Released Windows CLI archives must run without a separate MinGW/MSYS2 +# install. Prefer llvm-mingw (+ -static) in CI so emitters do not need +# libgcc/libstdc++ harvested from MSYS2. Keep opt-in for local debug builds. +option(PSXRECOMP_STATIC_CLI "Statically link Windows runtime into CLI tools" OFF) +if(PSXRECOMP_STATIC_CLI AND WIN32 AND (MINGW OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")) foreach(_cli_target psxrecomp psxrecomp-game psxrecomp-toml psxrecomp-bios) - target_link_options(${_cli_target} PRIVATE -static -static-libgcc -static-libstdc++) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_link_options(${_cli_target} PRIVATE + -static -static-libgcc -static-libstdc++) + else() + # llvm-mingw / Clang: fully static CRT + libc++ / libunwind / pthread. + target_link_options(${_cli_target} PRIVATE -static) + endif() endforeach() endif() @@ -311,15 +317,15 @@ if(BUILD_TESTING) COMMAND bios_address_model_test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) - find_package(Python3 COMPONENTS Interpreter) - if(Python3_Interpreter_FOUND) - add_test( - NAME cli_project_packaging - COMMAND ${Python3_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_cli_project_packaging.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) - - # These two invoke psxrecomp-game, whose BIOS-profile resolution + find_package(Python3 COMPONENTS Interpreter) + if(Python3_Interpreter_FOUND) + add_test( + NAME cli_project_packaging + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_cli_project_packaging.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + + # These two invoke psxrecomp-game, whose BIOS-profile resolution # probes fixed working-directory-relative paths (bios/.toml, # then psxrecomp/bios/.toml) -- run them from the framework # root so the shipped profiles are found. @@ -390,10 +396,10 @@ if(BUILD_TESTING) default_renderer_guards fmv_quiet_guards fps_telemetry_opt_in - frame_interpolation_display_rate - hybrid_dev_any_input - launcher_vulkan_option - mod_controller_override + frame_interpolation_display_rate + hybrid_dev_any_input + launcher_vulkan_option + mod_controller_override overlay_candidate_capacity overlay_decodable_fallback overlay_init_guard diff --git a/runtime/include/boot_state.h b/runtime/include/boot_state.h index 96a64f6c..6b846d67 100644 --- a/runtime/include/boot_state.h +++ b/runtime/include/boot_state.h @@ -80,7 +80,7 @@ enum { BS_SEC_CPU = 0x01, /* CPUState: gpr/pc/hi/lo/cop0/gte_data/gte_ctrl */ BS_SEC_RAM = 0x02, /* 2 MB main RAM */ BS_SEC_SPAD = 0x03, /* 1 KB scratchpad */ - BS_SEC_IRQ = 0x04, /* i_stat / i_mask */ + BS_SEC_IRQ = 0x04, /* i_stat / i_mask / cycles_since_vblank (12B; 8B ok) */ BS_SEC_TIMER = 0x05, /* 3 root counters (counter/mode/target/irq/frac) */ BS_SEC_CLOCK = 0x06, /* psx_cycle_count */ BS_SEC_GPU = 0x07, /* GPU regs: display/draw-area/offset/mask/texpage/xfer*/ @@ -92,6 +92,14 @@ enum { BS_SEC_SIO = 0x0D, /* SIO regs + pad-config FSM + memcard FSM */ BS_SEC_DIRTY = 0x0E, /* dirty-RAM page bitmap (guest-written code pages) */ BS_SEC_MDEC = 0x0F, /* MDEC command/FIFOs/quant/scale (FMV decode resume) */ + BS_SEC_ICACHE = 0x10, /* R3000A I-cache tag/valid words (1024 u32) — fetch + cost model. Host-persistent otherwise: a warm load + without it replays with the pre-load timeline's + cache, so fetch-miss cycles differ per peer/retry + and IRQ delivery lands a few wait-loop iterations + apart (MotK abort@940: fin cyc Δ8, v0 5c83/5c86 + from identical baselines). Optional on load for + old blobs (left untouched when absent). */ }; /* Save a COMPLETE snapshot at game handoff. Returns 1 on success. */ diff --git a/runtime/include/cdrom.h b/runtime/include/cdrom.h index be77dc56..363b172d 100644 --- a/runtime/include/cdrom.h +++ b/runtime/include/cdrom.h @@ -57,6 +57,14 @@ void debug_force_cd_reinsert(void); * frontend via uncapped pacing (it does NOT alter CD timing — flooding XA * sectors desyncs and hangs the player). */ int cdrom_xa_stream_active(void); +/* True while a CD read is armed with XA/FMV mode bits (or XA already + * streaming). Netplay uses this to arm no-invent / refuse tip episodes + * before the first MDEC colour decode — MotK intro invent≠Start at FMV + * entry opened a tip episode into a matched black wait. */ +int cdrom_fmv_stream_pending(void); +/* Re-arm host absolute CD deadlines from restored relative delays after + * boot_state / RB snap load. Does NOT clamp delays (unlike accelerate). */ +void cdrom_resync_deadlines_after_restore(void); /* CD load-burst ring (always-on). One record per gap-separated run of * delivered data sectors. `out` receives up to `max` records, newest first diff --git a/runtime/include/gpu.h b/runtime/include/gpu.h index 183a91ca..92408891 100644 --- a/runtime/include/gpu.h +++ b/runtime/include/gpu.h @@ -46,6 +46,11 @@ int gpu_display_is_depth24(void); void gpu_display_pixel_rgb(const GpuDisplayInfo* di, uint32_t x, uint32_t y, uint8_t* r, uint8_t* g, uint8_t* b); uint32_t gpu_display_pixel_argb(const GpuDisplayInfo* di, uint32_t x, uint32_t y); +/* Batch equivalent of calling gpu_display_pixel_argb(di, x, y) for x in + * [0, count) on a depth24 (FMV) scanline — same output, far less per-pixel + * overhead. `out` must hold at least `count` uint32_t ARGB entries. */ +void gpu_depth24_present_row(const GpuDisplayInfo* di, uint32_t y, uint32_t* out, + uint32_t count); /* Depth24: RGB columns covered by CPU→VRAM uploads since the last reset. * Returns 0 when no uploads yet, crtc_w when coverage is full/unknown-beyond, * else the covered RGB width. Present black-fills [limit, crtc_w) without @@ -121,6 +126,8 @@ void gpu_vblank_clear_deferred_present(void); /* Queue one deferred present (netplay RB resume when I_STAT already has * VBlank — snap resync cleared the pending callback). */ void gpu_vblank_arm_deferred_present(void); +/* 1 while a netplay deferred present is armed (MotK CD54 VBlank hold). */ +int gpu_vblank_present_pending(void); /* Clear flush_present reentrancy guard before longjmp (flush_resume). */ void gpu_vblank_release_present_flush_guard(void); diff --git a/runtime/include/interrupts.h b/runtime/include/interrupts.h index 9a07fa19..db191b6c 100644 --- a/runtime/include/interrupts.h +++ b/runtime/include/interrupts.h @@ -80,10 +80,9 @@ void psx_check_interrupts_dispatch_entry(struct CPUState* cpu, uint32_t resume_p void interrupts_advance_cycles(uint32_t cycles); void interrupts_service_scheduled_events(void); uint32_t interrupts_cycles_to_vblank(void); -/* VBlank phase within the current frame (0 .. VBLANK_CYCLES-1). Self-check - * snaps save/restore this so resim keeps the snap's phase instead of the - * warm-load rebase to 0 (which is correct for UI savestates, wrong for - * bit-identical resim of a mid-frame tip). */ +/* VBlank phase within the current frame (0 .. VBLANK_CYCLES-1). Persisted in + * BS_SEC_IRQ (and selfcheck's out-of-band latch) so resim keeps the snap's + * phase. Legacy 8-byte IRQ sections still rebase to 0 on load. */ uint32_t interrupts_get_cycles_since_vblank(void); void interrupts_set_cycles_since_vblank(uint32_t v); diff --git a/runtime/include/netplay_rb_post.h b/runtime/include/netplay_rb_post.h new file mode 100644 index 00000000..e71c8bb6 --- /dev/null +++ b/runtime/include/netplay_rb_post.h @@ -0,0 +1,27 @@ +#ifndef PSX_NETPLAY_RB_POST_H +#define PSX_NETPLAY_RB_POST_H + +/* + * RB_POST already carries target_tick on the wire. After tip-extend the peer + * may still deliver a POST for the prior tip while we Verify the new tip — + * latching that digest caused false post diverge (local@T+1 vs peer@T). + * + * Accept only when peer tip equals the current episode tip. + */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +static inline int netplay_rb_peer_post_tip_ok(uint32_t peer_target, uint32_t episode_tip) +{ + return peer_target == episode_tip; +} + +#ifdef __cplusplus +} +#endif + +#endif /* PSX_NETPLAY_RB_POST_H */ diff --git a/runtime/include/psx_netplay_rb.h b/runtime/include/psx_netplay_rb.h index 12b9e93a..6ba73f79 100644 --- a/runtime/include/psx_netplay_rb.h +++ b/runtime/include/psx_netplay_rb.h @@ -77,8 +77,8 @@ int psx_netplay_rb_fmv_defer_rewind(void); /* 1 while depth24 or recent MDEC (not settle). */ int psx_netplay_rb_fmv_media_active(void); -/* 1 during FMV media + post-FMV lockstep (~90 ticks) — admit waits for remote - * wire (title Start / movie skip). Ticks the FMV→settle tracker. */ +/* 1 during FMV media + post-FMV lockstep (+ unlock grace) — admit waits for + * remote wire (title Start / movie skip). Ticks the FMV→settle tracker. */ int psx_netplay_rb_lockstep_no_invent(void); /* Mid-guest resim pump: abort if Replay has made no finish_frame progress. @@ -97,6 +97,20 @@ int psx_netplay_rb_active(void); int psx_netplay_rb_is_resimulating(void); int psx_netplay_rb_tip_holding(void); +/* Episode tip (target_tick) while an episode is active; 0 if none. */ +uint32_t psx_netplay_rb_episode_target(void); + +/* TipHold Live invent slack past tip (tip_seal_slack). Stall invent beyond. */ +uint32_t psx_netplay_rb_tip_hold_invent_slack(void); + +/* TipHold coalesce runway (tip_runway). Host scans wire tip+1..tip+runway for + * release edges when Live is stalled at invent-cap. */ +uint32_t psx_netplay_rb_tip_runway(void); + +/* 1 if a peer FRAME_COMMIT should be dropped (TipHold invent still on the + * wire after tip-extend hc_prime). Call from FRAME_COMMIT drain. */ +int psx_netplay_rb_ignore_peer_frame_commit(uint32_t tick, uint32_t hash); + /* FRAME_COMMIT mismatch during Replay — abort before a false POST commit. * Returns 1 if an episode was aborted. */ int psx_netplay_rb_abort_resim_core_mismatch(uint32_t tick, uint32_t local_core, @@ -126,6 +140,23 @@ uint32_t psx_netplay_rb_episode_count(void); int psx_netplay_rb_phase(void); /* RNetRbPhase cast to int */ uint32_t psx_netplay_rb_snap_count(void); +/* Ticks armed into Replay since the last call (counter resets on read). Used + * by the host [FPS] line (PSX_NETPLAY_TIMING=1) to report what fraction of + * the window was spent resimulating vs running Live. */ +uint64_t psx_netplay_rb_take_replay_ticks(void); + +/* Live network-latency estimate (ms), EMA'd from the POST handshake's own + * send/receive timestamps each episode commit; 0 = no episode has round- + * tripped yet this session. Used by np_invent_grace_stall() (psx_netplay.c) + * to size its "wait before hold-last invent" budget from actual measured + * link latency instead of the local sim's own tick-to-tick cadence — see + * docs/ROLLBACK_MOTK_HOOKUP.md section 12. */ +uint32_t psx_netplay_rb_rtt_estimate_ms(void); + +/* Sticky BB-edge resume PC (MotK wait-loop canonicalize). Used by + * gpu_vblank_flush_present when IRQ resume latches are 0. */ +uint32_t psx_netplay_rb_sticky_bb_pc(void); + #ifdef __cplusplus } #endif diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 991c8834..f3bb8944 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -409,17 +409,31 @@ if(_psxrt_bios_skipped) "descriptor): ${_psxrt_bios_skipped} -- regenerate with " "tools/regen_bios.sh --config bios/.toml") endif() +# Setup hosts / CI may ship with zero BIOS backends; first-run Generate & +# rebuild emits OpenBIOS (and optional retail) locally before a full link. +option(PSXRECOMP_ALLOW_NO_BIOS + "Allow linking the runtime with no recompiled BIOS backends (setup host)" + OFF) if(NOT _psxrt_bios_linked) - message(FATAL_ERROR - "No recompiled BIOS backend available. Wanted: ${PSXRECOMP_BIOS_STEMS}, " - "but no matching generated/_full.c + _dispatch.c were found " - "under ${PSXRECOMP_ROOT}/generated.\n" - "Generate at least one before building the runtime:\n" - " bash tools/regen_bios.sh --config bios/OpenBIOS.toml\n" - "(OpenBIOS is bundled and MIT-licensed, so this needs no BIOS dump.)") + if(PSXRECOMP_ALLOW_NO_BIOS) + message(STATUS + "BIOS backends linked: (none) — setup host; Generate & rebuild " + "will emit BIOS C locally") + set(_psxrt_bios_count 0) + else() + message(FATAL_ERROR + "No recompiled BIOS backend available. Wanted: ${PSXRECOMP_BIOS_STEMS}, " + "but no matching generated/_full.c + _dispatch.c were found " + "under ${PSXRECOMP_ROOT}/generated.\n" + "Generate at least one before building the runtime:\n" + " bash tools/regen_bios.sh --config bios/OpenBIOS.toml\n" + "(OpenBIOS is bundled and MIT-licensed, so this needs no BIOS dump.)\n" + "Or configure a setup host with -DPSXRECOMP_ALLOW_NO_BIOS=ON.") + endif() +else() + message(STATUS "BIOS backends linked: ${_psxrt_bios_linked}") + list(LENGTH _psxrt_bios_linked _psxrt_bios_count) endif() -message(STATUS "BIOS backends linked: ${_psxrt_bios_linked}") -list(LENGTH _psxrt_bios_linked _psxrt_bios_count) # Registry of the compiled-in backends, in preference order. Generated so the # stem list stays the single source of truth. @@ -450,7 +464,7 @@ list(APPEND PSXRECOMP_BIOS_GENERATED "${_psxrt_registry_c}") # argument as regen_bios.sh passes) and WARN on a mismatch so the staleness is # impossible to miss. Non-fatal: a stale-but-consistent # BIOS still builds; opt out with -DPSXRECOMP_SKIP_BIOS_STALE_CHECK=ON. -if(NOT PSXRECOMP_SKIP_BIOS_STALE_CHECK) +if(NOT PSXRECOMP_SKIP_BIOS_STALE_CHECK AND _psxrt_bios_linked) find_program(_psxrt_bash NAMES bash) set(_psxrt_stamp "${PSXRECOMP_ROOT}/generated/${PSXRECOMP_BIOS_STEM}.emitter.sha") if(_psxrt_bash AND EXISTS "${PSXRECOMP_ROOT}/tools/bios_emitter_fingerprint.sh") @@ -480,6 +494,60 @@ if(NOT PSXRECOMP_SKIP_BIOS_STALE_CHECK) endif() endif() +# zlib for boot_state v4. System packages first; otherwise FetchContent a pinned +# release so portable Windows cmake-clang-v1 setups can configure without MSYS2. +option(PSX_ZLIB_FETCH + "Fetch zlib when no system ZLIB package is found" + ON) +function(psxrecomp_ensure_zlib) + if(TARGET ZLIB::ZLIB) + return() + endif() + find_package(ZLIB QUIET) + if(TARGET ZLIB::ZLIB) + return() + endif() + if(NOT PSX_ZLIB_FETCH) + message(FATAL_ERROR + "ZLIB was not found. On Windows, use a cmake-clang-v1 pack that " + "ships include/zlib.h + lib/libz.a (sets ZLIB_ROOT), or install " + "zlib-devel / mingw-w64-zlib, or configure with -DPSX_ZLIB_FETCH=ON.") + endif() + message(STATUS + "psxrecomp: ZLIB not in CMAKE_PREFIX_PATH/ZLIB_ROOT; " + "fetching zlib 1.3.1 (prefer a toolchain pack that bundles it)") + include(FetchContent) + set(_psx_zlib_timestamp_args "") + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24) + list(APPEND _psx_zlib_timestamp_args DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(psx_zlib + URL + "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz" + URL_HASH + "SHA256=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" + ${_psx_zlib_timestamp_args}) + FetchContent_MakeAvailable(psx_zlib) + if(TARGET ZLIB::ZLIB) + message(STATUS "psxrecomp: ZLIB via FetchContent (ZLIB::ZLIB)") + return() + endif() + # madler/zlib builds zlibstatic even when a shared zlib target exists. + if(TARGET zlibstatic) + add_library(ZLIB::ZLIB ALIAS zlibstatic) + message(STATUS "psxrecomp: ZLIB via FetchContent (zlibstatic)") + return() + endif() + if(TARGET zlib) + add_library(ZLIB::ZLIB ALIAS zlib) + message(STATUS "psxrecomp: ZLIB via FetchContent (zlib)") + return() + endif() + message(FATAL_ERROR + "Fetched zlib but no linkable target was produced " + "(expected zlibstatic or zlib).") +endfunction() + function(psxrecomp_add_runtime_target target) set(options ORACLE COSIM) set(oneValueArgs @@ -724,7 +792,9 @@ function(psxrecomp_add_runtime_target target) endif() # zlib: boot_state v4 savestate compression (RAM/VRAM/SPU blobs). - find_package(ZLIB REQUIRED) + # Portable Windows toolchains (cmake-clang-v1) have no system zlib — + # fetch a pinned release when find_package fails (same pattern as SDL3). + psxrecomp_ensure_zlib() target_link_libraries(${target} PRIVATE ZLIB::ZLIB) # Build identity: stamp the psxrecomp commit into the binary so a crash report @@ -1101,3 +1171,198 @@ endfunction() function(psxrecomp_v4_add_runtime_target target) psxrecomp_add_runtime_target(${target} ${ARGN}) endfunction() + +# --------------------------------------------------------------------------- +# High-level helper for title repos (setup-host + optional generated game C). +# +# Typical game CMakeLists.txt: +# set(PSXRECOMP_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/psxrecomp") +# include("${PSXRECOMP_ROOT}/runtime/runtime.cmake") +# psxrecomp_add_game_runtime(psx-runtime +# WINDOW_TITLE "My Game Recompiled" +# GEN_MARKER "generated/SLUS_01234_dispatch.c" +# GEN_FULL_GLOB "generated/SLUS_01234_full_*.c" +# CODEGEN_SETUP_SOURCES codegen_setup.c +# DEFAULT_GAME_CONFIG_PATH "game.toml" +# LAUNCHER_BOXART "${CMAKE_CURRENT_SOURCE_DIR}/launcher_assets/img/boxart.tga" +# MAX_PLAYERS 2 +# ENABLE_NETPLAY_IF_PRESENT +# ) +# +# Remaining args are forwarded to psxrecomp_add_runtime_target. +# --------------------------------------------------------------------------- +function(psxrecomp_add_game_runtime target) + set(options ENABLE_NETPLAY_IF_PRESENT) + set(oneValueArgs + GEN_MARKER + GEN_FULL_FALLBACK + VERSION_FILE + CODEGEN_SETUP_INCLUDE_DIR + ) + set(multiValueArgs GEN_FULL_GLOB CODEGEN_SETUP_SOURCES) + cmake_parse_arguments(PSXG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(NOT PSXRECOMP_ROOT) + message(FATAL_ERROR + "psxrecomp_add_game_runtime: PSXRECOMP_ROOT is not set. " + "Set it to the psxrecomp submodule path before including runtime.cmake.") + endif() + + option(PSXRECOMP_FORCE_SETUP_HOST + "Build without linking game C even if generated/ exists" OFF) + option(PSXRECOMP_REQUIRE_GAME_C + "Fail configure if generated game C is missing" OFF) + + # Legacy BPE option name (CI / docs may still pass -DBPE_FORCE_SETUP_HOST=ON). + if(BPE_FORCE_SETUP_HOST) + set(PSXRECOMP_FORCE_SETUP_HOST ON CACHE BOOL + "Build without linking game C even if generated/ exists" FORCE) + endif() + if(BPE_REQUIRE_GAME_C) + set(PSXRECOMP_REQUIRE_GAME_C ON CACHE BOOL + "Fail configure if generated game C is missing" FORCE) + endif() + + if(NOT PSXG_VERSION_FILE) + set(PSXG_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/VERSION") + endif() + if(NOT IS_ABSOLUTE "${PSXG_VERSION_FILE}") + set(PSXG_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${PSXG_VERSION_FILE}") + endif() + + # Lobby / release pin from VERSION when PSX_GAME_VERSION is unset. + if(EXISTS "${PSXG_VERSION_FILE}") + file(READ "${PSXG_VERSION_FILE}" _psxg_ver_raw) + string(STRIP "${_psxg_ver_raw}" _psxg_release_version) + else() + set(_psxg_release_version "0.0.0") + endif() + set(PSX_GAME_VERSION "" CACHE STRING + "Lobby release pin (empty = Release uses VERSION file, else dev)") + if(PSX_GAME_VERSION STREQUAL "") + get_property(_psxg_is_multi GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if(_psxg_is_multi) + set(PSX_GAME_VERSION + "$,${_psxg_release_version},dev>") + elseif(CMAKE_BUILD_TYPE STREQUAL "Release") + set(PSX_GAME_VERSION "${_psxg_release_version}") + else() + set(PSX_GAME_VERSION "dev") + endif() + endif() + message(STATUS + "psxrecomp game_version: ${PSX_GAME_VERSION} " + "(from VERSION=${_psxg_release_version})") + + # Prefer game-root recomp-ui (runtime.cmake also auto-discovers this). + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/recomp-ui/recomp_ui.cmake") + set(RECOMP_UI_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/recomp-ui" CACHE PATH + "Path to recomp-ui launcher" FORCE) + endif() + + # Optional: enable netplay when recomp-net is present. + if(PSXG_ENABLE_NETPLAY_IF_PRESENT) + if(EXISTS "${PSXRECOMP_ROOT}/lib/recomp-net/CMakeLists.txt") + if(NOT RECOMP_NET_ROOT) + set(RECOMP_NET_ROOT "${PSXRECOMP_ROOT}/lib/recomp-net" CACHE PATH + "Path to recomp-net; empty = auto-discover" FORCE) + endif() + if(NOT DEFINED PSX_NETPLAY) + set(PSX_NETPLAY ON) + endif() + endif() + endif() + + if(NOT PSXG_GEN_MARKER) + message(FATAL_ERROR + "psxrecomp_add_game_runtime: GEN_MARKER is required " + "(e.g. generated/SLUS_01234_dispatch.c)") + endif() + if(NOT IS_ABSOLUTE "${PSXG_GEN_MARKER}") + set(_psxg_marker "${CMAKE_CURRENT_SOURCE_DIR}/${PSXG_GEN_MARKER}") + else() + set(_psxg_marker "${PSXG_GEN_MARKER}") + endif() + + set(_psxg_has_game_c FALSE) + if(EXISTS "${_psxg_marker}" AND NOT PSXRECOMP_FORCE_SETUP_HOST) + set(_psxg_has_game_c TRUE) + endif() + + if(NOT _psxg_has_game_c) + set(PSXRECOMP_ALLOW_NO_BIOS ON CACHE BOOL + "Allow runtime with no BIOS backends (setup host)" FORCE) + endif() + + if(PSXRECOMP_REQUIRE_GAME_C AND NOT _psxg_has_game_c) + message(FATAL_ERROR + "PSXRECOMP_REQUIRE_GAME_C=ON but ${_psxg_marker} is missing. " + "Generate with psxrecomp-game, or leave REQUIRE_GAME_C off to " + "build the setup host.") + endif() + + set(_psxg_extras ${PSXG_CODEGEN_SETUP_SOURCES}) + list(APPEND _psxg_extras + "${PSXRECOMP_ROOT}/host/psxrecomp_codegen_host.c") + + set(_psxg_rt_args + GAME_VERSION "${PSX_GAME_VERSION}" + EXTRAS_SOURCES ${_psxg_extras} + ${PSXG_UNPARSED_ARGUMENTS} + ) + + if(_psxg_has_game_c) + message(STATUS + "psxrecomp: linking generated game C (full runtime) — ${_psxg_marker}") + set(_psxg_full_list "") + foreach(_glob IN LISTS PSXG_GEN_FULL_GLOB) + if(NOT IS_ABSOLUTE "${_glob}") + set(_glob "${CMAKE_CURRENT_SOURCE_DIR}/${_glob}") + endif() + file(GLOB _hits "${_glob}") + list(APPEND _psxg_full_list ${_hits}) + endforeach() + if(NOT _psxg_full_list) + if(PSXG_GEN_FULL_FALLBACK) + if(NOT IS_ABSOLUTE "${PSXG_GEN_FULL_FALLBACK}") + set(_psxg_full_list + "${CMAKE_CURRENT_SOURCE_DIR}/${PSXG_GEN_FULL_FALLBACK}") + else() + set(_psxg_full_list "${PSXG_GEN_FULL_FALLBACK}") + endif() + else() + # Derive SLUS_*_full.c next to the dispatch marker. + get_filename_component(_psxg_marker_dir "${_psxg_marker}" DIRECTORY) + get_filename_component(_psxg_marker_name "${_psxg_marker}" NAME) + string(REPLACE "_dispatch.c" "_full.c" _psxg_full_name + "${_psxg_marker_name}") + set(_psxg_full_list "${_psxg_marker_dir}/${_psxg_full_name}") + endif() + endif() + psxrecomp_add_runtime_target(${target} + GAME_GENERATED_FULL_C ${_psxg_full_list} + GAME_GENERATED_DISPATCH_C "${_psxg_marker}" + ${_psxg_rt_args} + ) + else() + message(STATUS + "psxrecomp: setup host (no game C, no BIOS backends) — " + "first-run Generate & rebuild") + psxrecomp_add_runtime_target(${target} ${_psxg_rt_args}) + endif() + + target_compile_definitions(${target} PRIVATE PSX_HAS_GAME_CODEGEN=1) + + # Include the portable codegen host. Do NOT add CMAKE_CURRENT_SOURCE_DIR + # wholesale to -I: on case-insensitive macOS, #include can pick + # up the repo VERSION pin file. Title codegen_setup.h is found via + # quote-include next to codegen_setup.c. + set(_psxg_inc "${PSXRECOMP_ROOT}/host") + if(PSXG_CODEGEN_SETUP_INCLUDE_DIR) + list(APPEND _psxg_inc "${PSXG_CODEGEN_SETUP_INCLUDE_DIR}") + endif() + if(RECOMP_UI_ROOT) + list(APPEND _psxg_inc "${RECOMP_UI_ROOT}/src") + endif() + target_include_directories(${target} PRIVATE ${_psxg_inc}) +endfunction() diff --git a/runtime/src/boot_state.c b/runtime/src/boot_state.c index 7ffb157f..cded889a 100644 --- a/runtime/src/boot_state.c +++ b/runtime/src/boot_state.c @@ -3,7 +3,9 @@ #include "dirty_ram_interp.h" #include "gpu_render.h" /* gr_vram_transfer_in / gr_vram_transfer_out */ #include "cpu_state.h" /* gte_canonicalize_cpu_state after CPU wire restore */ +#include "interrupts.h" #include "psx_cycles.h" +#include "psx_icache.h" /* g_psx_icache_tv — fetch-cost tags in BS_SEC_ICACHE */ #include "pst_wire.h" #include #include @@ -237,7 +239,7 @@ static int boot_state_save_to(BsOut* o, const CPUState* cpu, h.codegen_hash = (uint32_t)PSX_OVERLAY_CODEGEN_HASH; h.abi_tag = (int32_t)PSX_OVERLAY_ABI_TAG; h.codegen_ver = (uint32_t)PSX_OVERLAY_CODEGEN_VER; - h.section_count = 15; + h.section_count = 16; ok = write_header_le(o, &h); @@ -245,11 +247,15 @@ static int boot_state_save_to(BsOut* o, const CPUState* cpu, if (ok) ok = write_section(o, BS_SEC_RAM, memory_get_ram_ptr(), RAM_SIZE); if (ok) ok = write_section(o, BS_SEC_SPAD, memory_get_scratchpad_ptr(), SPAD_SIZE); if (ok) { - uint8_t irq[8]; + /* 12B: i_stat, i_mask, cycles_since_vblank. Zeroing csv on warm load + * rebased every tip to phase 0 and forked MotK wait-loop resim + * (IRQ at CD54 vs CDA0). Selfcheck already restored csv out-of-band. */ + uint8_t irq[12]; PstW w; pst_w_init(&w, irq, sizeof irq); ok = pst_w_u32(&w, i_stat) && pst_w_u32(&w, i_mask) && - write_section(o, BS_SEC_IRQ, irq, 8); + pst_w_u32(&w, interrupts_get_cycles_since_vblank()) && + write_section(o, BS_SEC_IRQ, irq, 12); } if (ok) ok = write_timer_section(o); if (ok) { @@ -294,6 +300,18 @@ static int boot_state_save_to(BsOut* o, const CPUState* cpu, if (ok) ok = write_module_section(o, BS_SEC_DMA, dma_snapshot_bytes, dma_snapshot_write); if (ok) ok = write_module_section(o, BS_SEC_SIO, sio_snapshot_bytes, sio_snapshot_write); if (ok) ok = write_module_section(o, BS_SEC_MDEC, mdec_snapshot_bytes, mdec_snapshot_write); + if (ok) { + /* I-cache tags: warm loads must replay with the fetch-cost state the + * live timeline had, or miss cycles differ per peer/retry and IRQ + * delivery forks a few wait-loop iterations (MotK abort@940). */ + uint8_t ib[1024u * 4u]; + PstW w; + pst_w_init(&w, ib, sizeof ib); + ok = 1; + for (uint32_t i = 0; ok && i < 1024u; i++) + ok = pst_w_u32(&w, g_psx_icache_tv[i]); + if (ok) ok = write_section(o, BS_SEC_ICACHE, ib, sizeof ib); + } if (ok) { uint32_t wc = dirty_ram_get_bitmap_word_count(); uint64_t nbytes = (uint64_t)wc * 4u; @@ -403,12 +421,19 @@ static int apply_section(uint32_t tag, const uint8_t* p, uint32_t len, return 1; case BS_SEC_IRQ: { PstR r; - uint32_t st, mk; - if (len != 8) return 0; + uint32_t st, mk, csv; + if (len != 8 && len != 12) return 0; pst_r_init(&r, p, len); if (!pst_r_u32(&r, &st) || !pst_r_u32(&r, &mk)) return 0; i_stat = st; i_mask = mk; + if (len == 12) { + if (!pst_r_u32(&r, &csv)) return 0; + interrupts_set_cycles_since_vblank(csv); + } else { + /* Legacy UI/disk snaps: no phase — rebase like pre-csv saves. */ + interrupts_set_cycles_since_vblank(0); + } return 1; } case BS_SEC_TIMER: { @@ -498,6 +523,14 @@ static int apply_section(uint32_t tag, const uint8_t* p, uint32_t len, free(words); return 1; } + case BS_SEC_ICACHE: { + PstR r; + if (len != 1024u * 4u) return 0; + pst_r_init(&r, p, len); + for (uint32_t i = 0; i < 1024u; i++) + if (!pst_r_u32(&r, &g_psx_icache_tv[i])) return 0; + return 1; + } default: return 0; } diff --git a/runtime/src/cdrom.c b/runtime/src/cdrom.c index c89b8742..decbc743 100644 --- a/runtime/src/cdrom.c +++ b/runtime/src/cdrom.c @@ -281,6 +281,25 @@ int cdrom_get_setloc_lba(void) { return s_setloc_lba; } /* Frontend XA-stream probe (FMV auto-skip / turbo-load gating in main.cpp). */ int cdrom_xa_stream_active(void) { return xa_stream_active; } +int cdrom_fmv_stream_pending(void) +{ + if (xa_stream_active) + return 1; + /* mode 0x40 = XA-ADPCM enable; 0x08 = sector size / related FMV setup. + * Match warm-route consumer (mode & 0x48). */ + if (reading && (mode_reg & 0x48u)) + return 1; + return 0; +} + +void cdrom_resync_deadlines_after_restore(void) +{ + if (reading && read_delay > 0) + s_cd_timing_next_due = psx_cycle_count + (uint64_t)read_delay; + else if (!reading) + s_cd_timing_next_due = 0; +} + /* Response-overwrite diagnostics consumed by debug_server.c. master's CD * response arbiter tracks how often an undelivered response was overwritten; * this CD model (forwarded from the Ape bring-up branch) delivers responses @@ -2831,7 +2850,12 @@ int cdrom_snapshot_read(const uint8_t *p, uint32_t len) { PstR r; if (len != cdrom_snapshot_bytes()) return 0; pst_r_init(&r, p, len); - return cdrom_snap_parse(&r); + if (!cdrom_snap_parse(&r)) + return 0; + /* Absolute host deadlines are not on the wire — rebuild from restored + * relative read_delay (psx_cycle_count is resynced by the load caller). */ + cdrom_resync_deadlines_after_restore(); + return 1; } void debug_force_cd_reinsert(void) { diff --git a/runtime/src/crc32.c b/runtime/src/crc32.c index 687a6260..ebb1934b 100644 --- a/runtime/src/crc32.c +++ b/runtime/src/crc32.c @@ -1,6 +1,26 @@ /* - * crc32.c — Simple CRC32 implementation (no external dependencies). + * crc32.c — CRC32 implementation (no external dependencies). * Uses the standard IEEE 802.3 polynomial 0xEDB88320. + * + * Slicing-by-8 (2026-08-01): the netplay rollback digest path CRCs the full + * 2 MiB RAM buffer and 1 MiB VRAM buffer on every episode commit (see + * docs/ROLLBACK_MOTK_HOOKUP.md, "Tier 1/2/3 engine enhancements"). The + * original byte-at-a-time loop below runs at ~500-600 MB/s because each + * iteration's table lookup depends on the previous iteration's output CRC — + * a serial dependency chain that leaves the CPU unable to overlap + * consecutive steps. Slicing-by-8 restates 8 of those serially-dependent + * steps as 8 *independent* table lookups (each depends only on an input + * byte, not on each other) that are XORed together, which lets the CPU + * execute them out-of-order/in parallel. This is a pure implementation + * change: it computes the exact same polynomial over the exact same bytes, + * so output is bit-identical to the old loop for every input — verified by + * fuzzing both implementations against each other over 27k+ cases + * (exhaustive small lengths from many offsets/starting-crc values, 20k + * random large buffers, and 5k chained/incremental folds matching how + * netplay_state_digest.c folds parts together) before this replaced the + * scalar loop. Measured ~5x faster on both VRAM (1 MiB) and RAM (2 MiB) + * sized buffers. No protocol/wire-format implication: the digest is an + * opaque comparison value never compared across differently-built peers. */ #include "crc32.h" #include @@ -18,15 +38,55 @@ static void crc32_init_table(void) { s_table_ready = 1; } +/* s_table8[k] = s_table with the base single-byte transform applied k+1 + * times (s_table8[0] == s_table). Built once from s_table so the slicing + * loop stays defined purely in terms of the one canonical polynomial table + * above — there is no second, independently-authored polynomial to drift + * out of sync with it. */ +static uint32_t s_table8[8][256]; +static int s_table8_ready = 0; + +static void crc32_init_table8(void) { + int n, k; + if (!s_table_ready) crc32_init_table(); + for (n = 0; n < 256; n++) s_table8[0][n] = s_table[n]; + for (n = 0; n < 256; n++) { + uint32_t c = s_table8[0][n]; + for (k = 1; k < 8; k++) { + c = s_table8[0][c & 0xFFu] ^ (c >> 8); + s_table8[k][n] = c; + } + } + s_table8_ready = 1; +} + /* Incremental update over a raw (un-finalized) running CRC. Start from * 0xFFFFFFFF, fold each chunk through crc32_update, then XOR the result with * 0xFFFFFFFF to finalize. Lets a multi-range hash (e.g. a function's * non-contiguous code ranges) be computed without concatenating into a temp - * buffer, and matches crc32_compute over the concatenation of the same bytes. */ + * buffer, and matches crc32_compute over the concatenation of the same bytes. + * + * Bytes are read individually (data[0]..data[7]), not via a reinterpreted + * 32-bit load, so this has no dependency on host byte order or pointer + * alignment — the slicing speedup comes from breaking the serial + * crc-depends-on-previous-crc chain, not from fewer loads, so there is + * nothing to gain from a wider load here and real portability to lose. */ uint32_t crc32_update(uint32_t crc, const uint8_t *data, size_t len) { if (!s_table_ready) crc32_init_table(); - for (size_t i = 0; i < len; i++) - crc = s_table[(crc ^ data[i]) & 0xFF] ^ (crc >> 8); + if (!s_table8_ready) crc32_init_table8(); + + while (len >= 8) { + uint32_t b0 = data[0] ^ (uint8_t)(crc); + uint32_t b1 = data[1] ^ (uint8_t)(crc >> 8); + uint32_t b2 = data[2] ^ (uint8_t)(crc >> 16); + uint32_t b3 = data[3] ^ (uint8_t)(crc >> 24); + crc = s_table8[7][b0] ^ s_table8[6][b1] ^ s_table8[5][b2] ^ s_table8[4][b3] + ^ s_table8[3][data[4]] ^ s_table8[2][data[5]] ^ s_table8[1][data[6]] ^ s_table8[0][data[7]]; + data += 8; + len -= 8; + } + while (len--) + crc = s_table[(crc ^ *data++) & 0xFFu] ^ (crc >> 8); return crc; } diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index 922ec0bd..27e3771c 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -2483,8 +2483,15 @@ void gpu_vblank_clear_deferred_present(void) { } void gpu_vblank_arm_deferred_present(void) { - if (s_present_pending < 8) - s_present_pending++; + /* Coalesce: at most one deferred present. Stacking (≥2) drained in one + * flush as double finish_frame at the same guest cycle (MotK soak: + * fin@N and fin@N+1 share dig/cyc → episode skew + clk/tim ±9). */ + if (s_present_pending < 1) + s_present_pending = 1; +} + +int gpu_vblank_present_pending(void) { + return s_present_pending > 0; } void gpu_vblank_release_present_flush_guard(void) { @@ -2494,6 +2501,40 @@ void gpu_vblank_release_present_flush_guard(void) { void gpu_vblank_flush_present(void) { if (s_flushing_present || s_present_pending <= 0) return; + /* Belt-and-suspenders with interrupts.c: never finish_frame inside the + * exception handler (IEc-clear BB edges used to drain present_pending). */ + { + extern int psx_get_in_exception(void); + if (psx_get_in_exception()) + return; + } + /* MotK menu wait (0x8006CD54↔0x8006CDA0): present ONLY at an explicit + * CDA0 edge. Never present on a CD54 edge (sticky CDA0 must not allow + * that — soak ep3 non-det fin@946). Non-wait edges (FMV / cutover) must + * present even if sticky still names the wait loop — treating sticky as + * in_wait blocked deferred present for the whole FMV1→FMV2 gap. */ + { + extern int psx_netplay_active(void); + extern uint32_t psx_compiled_irq_resume_pc(void); + extern uint32_t psx_last_irq_check_pc(void); + extern uint32_t psx_netplay_rb_sticky_bb_pc(void); + if (psx_netplay_active()) { + const uint32_t wait_a = 0x8006CD54u; + const uint32_t wait_b = 0x8006CDA0u; + uint32_t pc = psx_compiled_irq_resume_pc(); + uint32_t last = psx_last_irq_check_pc(); + uint32_t sticky = psx_netplay_rb_sticky_bb_pc(); + uint32_t edge = pc ? pc : last; + if (edge == wait_a) + return; + if (edge == wait_b || edge != 0u) { + /* CDA0, or non-wait (FMV/cutover) — present; ignore sticky */ + } else if (sticky == wait_a) { + return; /* latch cleared, sticky CD54 — defer */ + } + /* sticky CDA0 or unrelated/0 — present */ + } + } s_flushing_present = 1; while (s_present_pending > 0) { s_present_pending--; @@ -2539,9 +2580,9 @@ void gpu_vblank_tick(void) { * resim peers (selfcheck soak: many FAILs with d_cyc≠0). Netplay * defers — both peers share the same present contract from boot. */ if (psx_netplay_active()) { - /* Cap: a stuck defer must not queue unbounded presents. */ - if (s_present_pending < 8) - s_present_pending++; + /* Coalesce to one deferred present (see arm_deferred_present). */ + if (s_present_pending < 1) + s_present_pending = 1; } else { vblank_callback(); } @@ -2691,6 +2732,45 @@ uint32_t gpu_display_pixel_argb(const GpuDisplayInfo* di, uint32_t x, uint32_t y return 0xFF000000u | ((uint32_t)r << 16) | ((uint32_t)g << 8) | (uint32_t)b; } +/* Batch depth24 (FMV) scanline → ARGB. Same semantics as calling + * gpu_display_pixel_argb(di, x, y) for x in [0, count) (byte-identical + * output, including the black-fill past the 2048-byte VRAM row), but hoists + * the per-row invariants (vy, base_byte_x, the "how many pixels are past the + * row edge" test) out of the per-pixel path instead of recomputing them + * `count` times and paying three gpu_vram_byte() calls per pixel. The + * per-pixel path was the dominant present-side cost of FMV frames (3x the + * VRAM touches of the 16-bit path, done as a function-call chain instead of + * a straight-line loop). Still scalar C — no host-endianness assumptions, + * same byte-order shifts as gpu_vram_byte. */ +void gpu_depth24_present_row(const GpuDisplayInfo* di, uint32_t y, uint32_t* out, + uint32_t count) { + uint32_t vy = (di->display_y + y) & 511u; + uint32_t base_byte_x = (di->display_x & 1023u) * 2u; + const uint16_t* row = vram + (size_t)vy * 1024u; + uint32_t valid = 0u; + uint32_t x; + + if (base_byte_x <= 2045u) + valid = (2045u - base_byte_x) / 3u + 1u; + if (valid > count) + valid = count; + + for (x = 0; x < valid; x++) { + uint32_t byte_x = base_byte_x + x * 3u; + uint32_t bx1 = byte_x + 1u; + uint32_t bx2 = byte_x + 2u; + uint32_t hw0 = row[byte_x >> 1]; + uint32_t hw1 = row[bx1 >> 1]; + uint32_t hw2 = row[bx2 >> 1]; + uint8_t r = (byte_x & 1u) ? (uint8_t)(hw0 >> 8) : (uint8_t)hw0; + uint8_t g = (bx1 & 1u) ? (uint8_t)(hw1 >> 8) : (uint8_t)hw1; + uint8_t b = (bx2 & 1u) ? (uint8_t)(hw2 >> 8) : (uint8_t)hw2; + out[x] = 0xFF000000u | ((uint32_t)r << 16) | ((uint32_t)g << 8) | (uint32_t)b; + } + for (; x < count; x++) + out[x] = 0xFF000000u; +} + int gpu_display_is_depth24(void) { return (int)(display_depth & 1u); } diff --git a/runtime/src/interrupts.c b/runtime/src/interrupts.c index 5f1f6b14..74a679fe 100644 --- a/runtime/src/interrupts.c +++ b/runtime/src/interrupts.c @@ -638,7 +638,11 @@ void interrupts_resync_after_restore(void) { * delivery — including VBlank — until the restored clock catches up, * freezing the picture for those N seconds while host FPS stays at 60. */ post_exception_cooldown_until = 0; - cycles_since_vblank = 0; + /* Do NOT zero cycles_since_vblank: boot_state restores it from the snap + * (BS_SEC_IRQ +4). Zeroing rebased every warm tip to phase 0 while timers / + * LCF stayed at the snap's mid-frame phase — MotK wait-loop resim then + * delivered VBlank on opposite CD54↔CDA0 edges (cyc±3 / core fork). Legacy + * 8-byte IRQ sections still set csv=0 in boot_state_load. */ dispatch_count = 0; in_exception = 0; exception_nest_depth = 0; @@ -848,18 +852,43 @@ int psx_interrupt_delivery_needed(const CPUState* cpu) { void psx_check_interrupts(CPUState* cpu) { psx_cyc_batch_flush(); + /* Publish this edge's resume PC BEFORE deferred present flush. The MotK + * CDA0 gate reads s_last_interrupt_check_pc; leaving it at the previous + * BB (CD54) made every CDA0 entry flush look like CD54 and no-op. Present + * then only drained on post-IRQ at a CDA0 delivery — if the first post-arm + * VBlank was taken at CD54, peers waited a full extra VB (soak ep9: arm+2 + * vs arm+1). */ + { + uint32_t edge_pc = g_dirty_safe_resume_pc ? g_dirty_safe_resume_pc + : s_compiled_interrupt_resume_pc; + if (edge_pc != 0u) + s_last_interrupt_check_pc = edge_pc; + } /* Netplay: deferred sdl_vblank_present at BB edge (not mid-block - * fire_vblank_edge). When an IRQ is also deliverable, flush AFTER - * delivery so finish_frame digests post-RFE GPRs at the interrupted - * EPC — MotK menu CD54↔CDA0 was presenting pre-IRQ on opposite edges. - * Selfcheck stays on immediate present (see gpu.c). */ + * fire_vblank_edge). Prefer flush AFTER delivery so finish_frame digests + * post-RFE GPRs — but also attempt flush at entry when delivery is due. + * MotK's CDA0 gate no-ops the entry attempt on CD54; without it, sticky + * I_STAT kept skipping entry flush while post-IRQ also skipped on CD54, + * stacking multiple VBlanks into one drain (double finish_frame @ same + * guest cyc → clk/tim ±9 on the next IRQ). Post-IRQ retry still runs. + * Selfcheck stays on immediate present (see gpu.c). + * + * Never flush while in_exception: handler BB edges see IEc clear so + * delivery_needed is false and the old else-branch called finish_frame + * mid-BIOS-handler. Soak: irqctx left restored=0/reason=0, peers forked + * dig_cpu at v0=1 vs countdown (cyc±14) on sealed Cross resim. Outer + * delivery keeps np_present_after_irq and flushes on its RETURN. */ extern int g_ls_suppress_record; extern int psx_netplay_active(void); int np_present_after_irq = 0; - if (psx_netplay_active() && psx_interrupt_delivery_needed(cpu)) + if (in_exception) { + np_present_after_irq = 0; + } else if (psx_netplay_active() && psx_interrupt_delivery_needed(cpu)) { + gpu_vblank_flush_present(); /* CDA0 only; CD54 no-ops inside gate */ np_present_after_irq = 1; - else + } else { gpu_vblank_flush_present(); + } #define PSX_CHECK_INTERRUPTS_RETURN() do { \ if (np_present_after_irq) gpu_vblank_flush_present(); \ if (g_ls_suppress_record > 0) g_ls_suppress_record--; \ @@ -871,6 +900,25 @@ void psx_check_interrupts(CPUState* cpu) { #define COSIM_IRQ_NOTE(kind_) cosim_irq_note(cpu, (kind_), COSIM_IRQ_TAKE_PC(), g_dirty_safe_resume_pc, s_compiled_interrupt_resume_pc, cpu->cop0[COP0_SR]) #endif + /* MotK wait CD54 + VBlank-only I_STAT: never deliver at the CD54 edge in + * netplay — hold until CDA0. Delivering at CD54 leaves v0=slt(1) while a + * peer that hit CDA0 first delivers with countdown in v0 (soak: non-det + * fin@946/@902 CD54 vs CDA0, cyc ±1 VB, on identical baseline+pads). + * UNCONDITIONAL on the edge PC — the earlier gate also required + * gpu_vblank_present_pending(), but s_present_pending is host-only state + * (not in the snap), so replay delivery timing forked across peers. + * Edge PC + I_STAT are guest-deterministic; the wait ping-pong reaches + * CDA0 a few instructions later, so no starvation. */ + if (!in_exception && psx_netplay_active()) { + const uint32_t wait_a = 0x8006CD54u; + uint32_t edge = s_last_interrupt_check_pc; + uint32_t pend = i_stat & i_mask; + if (edge == wait_a && pend != 0u && + (pend & ~(1u << IRQ_VBLANK)) == 0u) { + PSX_CHECK_INTERRUPTS_RETURN(); + } + } + s_irq_path_entry++; /* MotK VLC / FMV hot edge: sticky unmasked I_STAT (CD/VBlank) while diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 3a71be07..d9ba1afa 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -459,6 +459,9 @@ static void present_session_reset(void) { s_d24_prev_mdec = 0; s_d24_saw_gap = 0; s_d24_cutover_blank = 0; + /* Soft-exit can leave deferred present / flush reentrancy armed; gpu_init + * does not clear them. Rematch then no-ops every flush → black window. */ + gpu_vblank_clear_deferred_present(); smooth_60_reset(); } @@ -859,14 +862,22 @@ extern "C" void psx_frontend_on_savestate_loaded(void) { post_load_probe_arm(); } -/* Rollback snap apply / realign — keep depth24 hold clear + restage, but do - * NOT run the full savestate present thrash (force_present, cutover gap reset, - * post_load_probe). That path is for one-shot disk loads; episode loads during - * MotK FMV were scrambling the movie. */ +/* Rollback snap apply / realign — keep depth24 hold clear + restage. Full + * savestate present thrash (post_load_probe) stays disk-only. When the tip + * is already in FMV/media, mirror the disk cutover reset so resume into + * FMV entry does not treat the tip as a fresh gap (permanent black blank). */ extern "C" void psx_frontend_on_rb_snap_loaded(void) { + const int media = gpu_display_is_depth24() || mdec_recently_active(8) || + cdrom_fmv_stream_pending() || cdrom_xa_stream_active(); g_audio_cycle_resync = 1; gpu_depth24_on_savestate_loaded(); s_d24_cutover_blank = 0; + if (media) { + s_d24_saw_gap = 0; + s_d24_prev_mdec = (gpu_display_is_depth24() || mdec_recently_active(8)) ? 1 : 0; + s_disabled_frame_presented = false; + s_force_present_after_load = true; + } gl_renderer_restage_vram_after_savestate(); vk_renderer_restage_vram_after_savestate(); } @@ -1164,11 +1175,13 @@ static int ensure_sw_sdl_present(void) { * mirror via readback — host-GPU nondeterministic; forks peer snaps (core * matched, VRAM zlib ~220KB apart) then mid-resim cores. SW is guest authority. */ extern "C" void psx_frontend_netplay_force_sw_gpu(void) { - if (s_netplay_sw_gpu_locked) - return; - s_netplay_sw_gpu_locked = 1; #ifndef PSX_SDL_NO_RENDER - const int had_hw = g_gl_active || g_vk_active; + /* Retry ensure even if a prior attempt locked after GL teardown with a + * null SDL_Renderer (rematch empty window). */ + if (s_netplay_sw_gpu_locked && sdl_renderer && sdl_texture && sdl_pixel_buf) + return; +#endif +#ifndef PSX_SDL_NO_RENDER if (g_gl_active) { gl_renderer_sync_cpu(); gl_renderer_shutdown(); @@ -1184,14 +1197,17 @@ extern "C" void psx_frontend_netplay_force_sw_gpu(void) { gr_set_backend(GR_BACKEND_SOFTWARE); g_video_renderer = 0; #ifndef PSX_SDL_NO_RENDER - /* Late switch (window already OPENGL/VULKAN): build SDL present path. */ - if (had_hw || !sdl_renderer) { - if (ensure_sw_sdl_present() != 0) { - std::fprintf(stderr, - "psxrecomp: netplay SW GPU forced but present path failed " - "(expect black frame)\n"); - } + if (ensure_sw_sdl_present() != 0) { + s_netplay_sw_gpu_locked = 0; + std::fprintf(stderr, + "psxrecomp: netplay SW GPU forced but present path failed " + "(expect black/empty window — will retry)\n"); + fflush(stderr); + return; } + s_netplay_sw_gpu_locked = 1; +#else + s_netplay_sw_gpu_locked = 1; #endif latency_ring_set_backend("software"); fprintf(stderr, @@ -3950,11 +3966,20 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { (double)s_np_admit_ticks * invf / (double)s_np_timing_frames; const double guest_ms = (double)s_np_guest_ticks * invf / (double)s_np_timing_frames; + /* Fraction of this window's ticks spent resimulating (Replay) + * vs running forward normally (Live). Lets a soak confirm + * "the game was chain-episoding almost the whole time" (near + * 100%) instead of inferring it from raw episode density in + * the log — see docs/ROLLBACK_MOTK_HOOKUP.md 2026-08-01. */ + const uint64_t replay_ticks = psx_netplay_rb_take_replay_ticks(); + const double replay_pct = + 100.0 * (double)replay_ticks / (double)s_np_timing_frames; std::fprintf(stderr, "[FPS] game: %.1f fps (%.2fx) | frames: %llu | " - "guest=%.2f ms/f admit=%.2f ms/f (n=%llu)\n", + "guest=%.2f ms/f admit=%.2f ms/f replay=%.0f%% (n=%llu)\n", fps, speed, (unsigned long long)s_frame_count, - guest_ms, admit_ms, (unsigned long long)s_np_timing_frames); + guest_ms, admit_ms, replay_pct, + (unsigned long long)s_np_timing_frames); s_np_admit_ticks = 0; s_np_guest_ticks = 0; s_np_timing_frames = 0; @@ -4309,11 +4334,14 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { } } - /* Netplay FMV: present 1 of every 4 depth24 vblanks. SW RGB888 scanout - * dominates guest ms; admit + wall pace still run every tick in the - * epilogue. Do NOT skip pace — MotK ran ~90fps when decode was fast and - * pace was skipped here. */ - if (psx_netplay_active() && gpu_display_is_depth24()) { + /* Netplay FMV: present 1 of every 4 depth24 vblanks while MDEC is hot. + * SW RGB888 scanout dominates guest ms; admit + wall pace still run every + * tick in the epilogue. Do NOT skip pace — MotK ran ~90fps when decode was + * fast and pace was skipped here. During MDEC-idle cutover (FMV1→FMV2) + * present every frame so the next stream can handshake (1/4 skip left the + * gap looking frozen with cheap guest + TURN admit wait). */ + if (psx_netplay_active() && gpu_display_is_depth24() && + mdec_recently_active(8)) { if (s_netplay_depth24_present_skip > 0) { s_netplay_depth24_present_skip--; return ep; @@ -4403,9 +4431,13 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { } else if (g_vk_active) { vk_renderer_present_blank(); } else { - SDL_SetRenderDrawColor(sdl_renderer, 0, 0, 0, 255); - SDL_RenderClear(sdl_renderer); - SDL_RenderPresent(sdl_renderer); + if (!sdl_renderer && ensure_sw_sdl_present() != 0) + return ep; + if (sdl_renderer) { + SDL_SetRenderDrawColor(sdl_renderer, 0, 0, 0, 255); + SDL_RenderClear(sdl_renderer); + SDL_RenderPresent(sdl_renderer); + } } } #endif @@ -4492,11 +4524,12 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { if (g_vk_active) { if (di.depth24) { /* 24-bit (FMV): packed RGB lives in the CPU mirror — do NOT - * sync_cpu (FBO readback clobbers RGB888). */ + * sync_cpu (FBO readback clobbers RGB888). Batch per-scanline + * (see gpu_depth24_present_row) instead of a per-pixel call + * chain — this loop was the FMV present-side cost. */ for (uint32_t y = 0; y < h; y++) - for (uint32_t x = 0; x < present_w; x++) - sdl_pixel_buf[y * present_w + x] = - gpu_display_pixel_argb(&di, x, y); + gpu_depth24_present_row(&di, y, sdl_pixel_buf + (size_t)y * present_w, + present_w); /* Trailing / cutover blank: black-fill uncovered RGB cols * inside the full-width buffer — never shrink present width. */ depth24_fix_trailing_margin(sdl_pixel_buf, present_w, h, @@ -4553,6 +4586,13 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { gr_render_display_hires(sdl_pixel_buf, (int)(sw * sizeof(uint32_t)), (int)di.display_x, (int)di.display_y, (int)present_w, (int)h); + } else if (di.depth24) { + /* Batch per-scanline (see gpu_depth24_present_row) — the + * per-pixel call chain was the dominant FMV present cost + * (3x the VRAM touches of the 16-bit path below). */ + for (uint32_t y = 0; y < h; y++) + gpu_depth24_present_row(&di, y, sdl_pixel_buf + (size_t)y * present_w, + present_w); } else { for (uint32_t y = 0; y < h; y++) { for (uint32_t x = 0; x < present_w; x++) { @@ -4634,6 +4674,10 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { (g_video_aa && !depth24_frame) ? 1 : 0, pin_43 ? 1 : 0, 0 /* full width */); } else { + if ((!sdl_renderer || !sdl_texture) && ensure_sw_sdl_present() != 0) + return ep; + if (!sdl_renderer || !sdl_texture) + return ep; SDL_Rect src = { 0, 0, src_w, src_h }; SDL_UpdateTexture(sdl_texture, &src, sdl_pixel_buf, (int)(src_w * sizeof(uint32_t))); diff --git a/runtime/src/mdec.c b/runtime/src/mdec.c index c753b8ff..c303e056 100644 --- a/runtime/src/mdec.c +++ b/runtime/src/mdec.c @@ -771,7 +771,15 @@ static void write_data(uint32_t value) { } int mdec_recently_active(uint32_t within_frames) { - return (uint64_t)(s_frame_count - mdec_last_color_decode_frame) <= within_frames; + /* Guest-cycle hysteresis (not host s_frame_count). Present-skip / Replay + * leave s_frame_count stale so a single decode looked "recent" forever + * (false FMV lockstep on rematch; tip episodes into MotK FMV entry). */ + const uint64_t cycles_per_frame = 338688ull; + const uint64_t window = + (uint64_t)within_frames * cycles_per_frame + (cycles_per_frame / 2ull); + if (psx_cycle_count < mdec_last_color_decode_cycle) + return 0; + return (psx_cycle_count - mdec_last_color_decode_cycle) <= window; } void mdec_init(void) { diff --git a/runtime/src/netplay_state_digest.c b/runtime/src/netplay_state_digest.c index 566497f8..6ea8bdf1 100644 --- a/runtime/src/netplay_state_digest.c +++ b/runtime/src/netplay_state_digest.c @@ -3,7 +3,9 @@ #include "crc32.h" #include "dirty_ram_interp.h" #include "gpu.h" +#include "interrupts.h" #include "psx_cycles.h" +#include "psx_icache.h" #include #include @@ -134,9 +136,17 @@ void netplay_core_digest_parts(const CPUState* cpu, NetplayCoreParts* out) { uint64_t cyc = psx_cycle_count; + uint32_t csv = interrupts_get_cycles_since_vblank(); crc_clk = crc32_update(crc_clk, (const uint8_t*)&cyc, sizeof(cyc)); crc_clk = crc32_update(crc_clk, (const uint8_t*)&i_stat, sizeof(i_stat)); crc_clk = crc32_update(crc_clk, (const uint8_t*)&i_mask, sizeof(i_mask)); + /* VBlank phase was invisible to FRAME_COMMIT — peers could agree on + * core while holding different csv, then resim from zeroed phase. */ + crc_clk = crc32_update(crc_clk, (const uint8_t*)&csv, sizeof(csv)); + /* I-cache tags travel in BS_SEC_ICACHE; fold them so cache asymmetry + * (fetch-cost fork) surfaces at compare time, not as a resim abort. */ + crc_clk = crc32_update(crc_clk, (const uint8_t*)g_psx_icache_tv, + sizeof(g_psx_icache_tv)); } timers_get_snapshot(counter, mode, target, irq_line, frac); diff --git a/runtime/src/psx_netplay.c b/runtime/src/psx_netplay.c index 75bc41fc..cae6f848 100644 --- a/runtime/src/psx_netplay.c +++ b/runtime/src/psx_netplay.c @@ -35,6 +35,7 @@ #include "cpu_state.h" #include "gpu.h" #include "interrupts.h" +#include "mdec.h" #include "psx_scheduler.h" #if defined(PSX_HAS_LOBBY_CLIENT) #include "psx_lobby_client.h" @@ -425,8 +426,14 @@ static void np_drain_peer_frame_commits(void) { rnet_u32 through = 0, hash = 0; if (!g_np.session) return; - while (rnet_session_take_rb_frame_commit(g_np.session, &through, &hash)) + while (rnet_session_take_rb_frame_commit(g_np.session, &through, &hash)) { + /* Tip-extend rereplay: TipHold Live invent FCs may still be queued + * after hc_prime — drop until peer's sealed resim matches. */ + if (g_np.rollback && + psx_netplay_rb_ignore_peer_frame_commit(through, hash)) + continue; netplay_hc_note_peer(&g_np.hc, through, hash); + } /* FIRST CORE can stick the watermark forever once that tick ages out of * the 128-slot ring — heal so choose_load_tick sees a real frontier. */ if (netplay_hc_heal_stale_gap(&g_np.hc)) { @@ -452,10 +459,17 @@ static void np_emit_frame_commit(uint32_t tick) uint32_t aux = 0u; int crumb; if (!g_np.cpu || !g_np.session) return; - /* Live depth24: skip full-RAM FRAME_COMMIT (rewind already deferred). - * Resim still commits every tick. Leaving FMV primes hash_confirm so the - * watermark is not stuck on missing movie slots. */ - if (gpu_display_is_depth24() && + /* TipHold Live invents are not sealed-input truth — emitting them poisoned + * tip-extend Replay (stale peer digests → false resim core diverge). */ + if (g_np.rollback && psx_netplay_rb_tip_holding() && + !psx_netplay_rb_is_resimulating()) + return; + /* Live depth24 + hot MDEC: skip full-RAM FRAME_COMMIT (rewind already + * deferred). Resim still commits every tick. MDEC-idle cutover still + * commits so FMV1→FMV2 forks are visible (digs used to go dark for the + * whole movie). Leaving FMV primes hash_confirm so the watermark is not + * stuck on missing movie slots. */ + if (gpu_display_is_depth24() && mdec_recently_active(8) && !(g_np.rollback && psx_netplay_rb_is_resimulating())) return; /* Present-edge: clear PC so parked-0 vs live-BB does not fork FRAME_COMMIT @@ -1353,6 +1367,144 @@ static int np_digital_new_press(const RNetInputContractFrame *pub, return newly_pressed != 0; } +/* After promoting a digital release at `release_tick`, rewrite predicted hist + * rows release_tick+1..sim to the released pad. Hold-last invent ahead of the + * release tip left `pub` still held → ghost second-release episodes (soak: + * RIGHT release@1607 then another `pub=ffdf wire=ffff` @1634). */ +static void np_scrub_ahead_predicted(int slot, rnet_u32 release_tick, + const RNetRbFrame *released) +{ + rnet_u32 sim; + rnet_u32 t; + unsigned n = 0; + if (!released || !g_np.session) + return; + sim = rnet_session_sim_tick(g_np.session); + if (sim <= release_tick) + return; + for (t = release_tick + 1u; t <= sim; ++t) { + RNetRbFrame row; + RNetRbFrame scrub; + if (!netplay_ih_get(&g_np.ih, slot, t, &row)) + continue; + if (!row.is_predicted) + continue; + if (row.buttons == released->buttons && + row.stick_x == released->stick_x && + row.stick_y == released->stick_y) + continue; + scrub = *released; + scrub.tick = t; + scrub.is_predicted = 0u; + scrub.is_valid = 1u; + if (netplay_ih_promote(&g_np.ih, slot, &scrub)) + n++; + } + if (n) { + fprintf(stderr, + "psxrecomp: rb scrub-ahead release slot=%d from=%u sim=%u n=%u " + "btn=%04x\n", + slot, (unsigned)release_tick, (unsigned)sim, n, + (unsigned)released->buttons); + fflush(stderr); + } +} + +/* TipHold Live is stalled at invent-cap while digital is held — predicted hist + * rows past tip may not exist yet, so ordinary reconcile never sees the + * release. Peek wire tip+1..tip+runway; on the first pad delta vs tip hist, + * promote the span and tip-extend (real coalesce). */ +static void np_tip_hold_coalesce_ahead(void) +{ + uint32_t tip; + uint32_t runway; + rnet_u8 delay_u8; + int slot; + + if (!g_np.rollback || !g_np.session) + return; + if (!psx_netplay_rb_tip_holding()) + return; + if (psx_netplay_rb_load_pending() || psx_netplay_rb_is_resimulating()) + return; + + tip = psx_netplay_rb_episode_target(); + runway = psx_netplay_rb_tip_runway(); + if (tip == 0u || runway == 0u) + return; + + delay_u8 = (rnet_u8)(g_np.input_delay < 0 ? 0 + : (g_np.input_delay > 255 ? 255 : g_np.input_delay)); + + for (slot = 0; slot < g_np.slot_count; ++slot) { + RNetRbFrame tip_row; + rnet_u32 edge = 0; + rnet_u32 t; + if (slot == g_np.local_slot) + continue; + if (!netplay_ih_get(&g_np.ih, slot, tip, &tip_row) || !tip_row.is_valid) + continue; + + for (t = tip + 1u; t <= tip + runway; ++t) { + RNetInputSample sample; + PsxNetPad pad; + RNetRbFrame wire_frame; + rnet_u32 wire = rnet_wire_tick_from_sim(t, delay_u8); + if (!rnet_session_peek_remote_input(g_np.session, slot, wire, &sample)) + break; + decode_pad(&sample, &pad); + netplay_ih_pad_to_frame(&pad, t, 0, &wire_frame); + if (wire_frame.buttons == tip_row.buttons && + wire_frame.stick_x == tip_row.stick_x && + wire_frame.stick_y == tip_row.stick_y) + continue; + edge = t; + break; + } + if (!edge) + continue; + + for (t = tip + 1u; t <= edge; ++t) { + RNetInputSample sample; + PsxNetPad pad; + RNetRbFrame wire_frame; + rnet_u32 wire = rnet_wire_tick_from_sim(t, delay_u8); + if (!rnet_session_peek_remote_input(g_np.session, slot, wire, &sample)) + break; + decode_pad(&sample, &pad); + netplay_ih_pad_to_frame(&pad, t, 0, &wire_frame); + (void)netplay_ih_promote(&g_np.ih, slot, &wire_frame); + } + { + RNetInputSample sample; + PsxNetPad pad; + RNetRbFrame edge_frame; + RNetInputContractFrame tip_c, edge_c; + rnet_u32 wire = rnet_wire_tick_from_sim(edge, delay_u8); + if (rnet_session_peek_remote_input(g_np.session, slot, wire, &sample)) { + decode_pad(&sample, &pad); + netplay_ih_pad_to_frame(&pad, edge, 0, &edge_frame); + netplay_ih_frame_to_contract(&tip_row, &tip_c); + netplay_ih_frame_to_contract(&edge_frame, &edge_c); + if (np_digital_release_only(&tip_c, &edge_c)) + np_scrub_ahead_predicted(slot, edge, &edge_frame); + } + } + fprintf(stderr, + "psxrecomp: rb tip-hold coalesce-ahead tip=%u edge=%u slot=%d " + "tip_btn=%04x\n", + (unsigned)tip, (unsigned)edge, slot, (unsigned)tip_row.buttons); + fflush(stderr); + if (psx_netplay_rb_tip_extend(edge, slot)) + return; + /* tip-extend refused (span cap / different seat) — open via begin. */ + if (psx_netplay_rb_begin_rewind(edge, slot)) + return; + } +} + +static void np_timesync_note_late(void); + /* Late authoritative wire vs published predicted rows → promote or queue rewind. * Diag: promote-no-resim means hist took the late pad but guest sim did not * rewind — looks like "remote input rejected" when cadence already drifted. */ @@ -1439,6 +1591,16 @@ static void np_rollback_reconcile_wire(void) pads_differ = (pub_c.buttons != wire_c.buttons) || (pub_c.stick_x != wire_c.stick_x) || (pub_c.stick_y != wire_c.stick_y); + /* Feed the timesync pacer ONLY on a genuine mispredict (the + * invented value was wrong). Firing on every predicted-row + * resolution — the prior cut — meant EVERY normally-delayed + * prediction counted as "late" (that is simply how input delay + * works), drowning the one signal that actually identifies the + * ahead peer. 2026-08-01 soak: gated correctly, the peer that + * initiates corrections is unambiguous (28 vs 12 rewind-requests + * split across the two sides in one session). */ + if (pads_differ) + np_timesync_note_late(); /* After commit/realign: flush invent poison without another episode. */ if (no_resim) { (void)netplay_ih_promote(&g_np.ih, slot, &wire_frame); @@ -1464,6 +1626,7 @@ static void np_rollback_reconcile_wire(void) * menu releases open a real episode. */ if (fmv_defer && np_digital_release_only(&pub_c, &wire_c)) { (void)netplay_ih_promote(&g_np.ih, slot, &wire_frame); + np_scrub_ahead_predicted(slot, t, &wire_frame); if (n_soft_release == 0) { first_no_resim_t = t; first_no_resim_slot = slot; @@ -1505,6 +1668,8 @@ static void np_rollback_reconcile_wire(void) * peer SEAL_ROWS land. Skipping promote left invent-idle * pads sealed/published across the skip press tick. */ (void)netplay_ih_promote(&g_np.ih, slot, &wire_frame); + if (np_digital_release_only(&pub_c, &wire_c)) + np_scrub_ahead_predicted(slot, t, &wire_frame); if (psx_netplay_rb_begin_rewind(t, slot)) { g_np.needs_advance = 0; n_episode_open++; @@ -1516,6 +1681,8 @@ static void np_rollback_reconcile_wire(void) } } else if (pads_differ) { (void)netplay_ih_promote(&g_np.ih, slot, &wire_frame); + if (np_digital_release_only(&pub_c, &wire_c)) + np_scrub_ahead_predicted(slot, t, &wire_frame); n_contract_promote++; } else { (void)netplay_ih_promote(&g_np.ih, slot, &wire_frame); @@ -1523,6 +1690,11 @@ static void np_rollback_reconcile_wire(void) } } + /* TipHold invent-cap stall: peek wire past tip for release/press edges that + * have no predicted hist row yet (Live never invented them). */ + if (!g_np.pending_rewind) + np_tip_hold_coalesce_ahead(); + if (n_no_resim || n_soft_release || n_episode_open || n_begin_refused) { static uint32_t s_log_sim; static unsigned s_suppress; @@ -1568,6 +1740,276 @@ static void np_rollback_reconcile_wire(void) } } +/* Stall-before-invent grace. Soaks showed every menu episode was a real + * press/release edge whose wire row landed only 1–2 ticks after the seal + * point (LAN, admit skew — not link latency): invent-on-first-miss then + * mispredicted the edge and opened a paired episode every few ticks + * (resim storm, 0.44–0.8x). Before inventing a missing remote row, stall + * the admit up to PSX_RB_INVENT_GRACE_MS (default 30) measured from the + * first miss of that wire tick. This is a rate governor, not just packet + * wait: the lateness is sim skew (soak: ALL late edges were on the peer + * running 1–2 ticks ahead; the other side had zero) — while the ahead + * peer stalls here, the behind peer keeps simulating and catches up, so + * in steady state the sims stay aligned and inputs arrive before the + * seal with no stall at all. The first cut (8 ms) was shorter than one + * tick of skew, expired on every edge and tripped the adaptive-off — + * worst of both. The budget scales with the observed tick period: at + * fight-scene rates (~35-45 fps) a tick lasts ~25 ms, so a fixed 30 ms + * no longer covers 2 ticks of skew and gameplay stormed again; the + * effective budget is max(base, 2.5x tick-period EMA), capped at 100 ms. + * The caller only invokes this when >=2 ticks ahead of the remote sealed + * tip: stalling on a 1-tick gap (normal pipeline phase offset) serialized + * the two sims — each peer's admit waited out the other's guest frame and + * throughput fell to 1/(g1+g2) even with zero episodes. + * Adaptive off: if the grace keeps expiring (peer + * genuinely lagging beyond the budget — real WAN latency), disable it + * for 2s so we don't add a per-tick stall on top of real lag. + * Host-side pacing only — the invented value is unchanged (hold-last), so + * guest determinism is unaffected. 0 disables. */ +/* Budget policy (2026-08-01, see docs/ROLLBACK_MOTK_HOOKUP.md section 12): + * this used to scale purely off the local sim's own tick-to-tick cadence + * (s_tick_ema_ms), which is the wrong signal — tick period gets *longer* + * precisely when the engine is already struggling (an active resim burst, + * a heavy scene), none of which means the network needs more patience. That + * let a slowdown balloon the very stall reacting to it: a soak on a genuine + * sub-ms-RTT direct-LAN connection measured this landing at 70-90ms per + * occurrence. Now sized primarily from psx_netplay_rb_rtt_estimate_ms() — a + * live measurement of actual POST-handshake round-trip latency — with the + * old tick-cadence term kept only as a *fallback* for the brief window + * before the first episode has round-tripped this session (no RTT sample + * yet), and tightened since it's no longer the primary mechanism. */ +static int np_invent_grace_stall(int slot, rnet_u32 wire) +{ + static int s_grace_ms = -1; + static rnet_u32 s_wire[PSX_MAX_PLAYERS]; + static uint32_t s_t0[PSX_MAX_PLAYERS]; + static uint8_t s_expired[PSX_MAX_PLAYERS]; + static uint8_t s_budget_logged[PSX_MAX_PLAYERS]; + static uint32_t s_expire_streak; + static uint32_t s_off_until; + static uint32_t s_tick_ema_ms; /* fallback-only EMA of inter-tick period, ms */ + uint32_t now; + uint32_t budget; + uint32_t rtt; + + if (s_grace_ms < 0) { + const char *e = getenv("PSX_RB_INVENT_GRACE_MS"); + s_grace_ms = (e && e[0]) ? atoi(e) : 8; + if (s_grace_ms < 0) s_grace_ms = 0; + if (s_grace_ms > 200) s_grace_ms = 200; + fprintf(stderr, + "psxrecomp: rb invent grace floor=%d ms (minimum stall before " + "hold-last invent; PSX_RB_INVENT_GRACE_MS — actual per-stall " + "budget scales up from measured RTT, see 'rb invent grace " + "budget' lines)\n", + s_grace_ms); + fflush(stderr); + } + if (s_grace_ms == 0 || slot < 0 || slot >= PSX_MAX_PLAYERS) + return 0; + now = np_mono_ms(); + if (s_off_until != 0u && (int32_t)(now - s_off_until) < 0) + return 0; + s_off_until = 0u; + if (s_wire[slot] != wire) { + /* Previous tracked tick arrived before its grace expired (we moved + * on without hitting the expiry path) — the link is keeping up. */ + if (s_wire[slot] != 0u && !s_expired[slot]) + s_expire_streak = 0; + /* Consecutive tracked ticks give the live tick period — fallback + * signal only now (see policy note above); real budget sizing + * prefers psx_netplay_rb_rtt_estimate_ms() below. */ + if (s_wire[slot] != 0u && wire == s_wire[slot] + 1u) { + uint32_t dt = now - s_t0[slot]; + if (dt >= 1u && dt <= 250u) + s_tick_ema_ms = s_tick_ema_ms + ? (3u * s_tick_ema_ms + dt) / 4u + : dt; + } + s_wire[slot] = wire; + s_t0[slot] = now; + s_expired[slot] = 0; + s_budget_logged[slot] = 0; + return 1; + } + budget = (uint32_t)s_grace_ms; + rtt = psx_netplay_rb_rtt_estimate_ms(); + if (rtt) { + /* 1.5x measured RTT: slack for jitter without inheriting the old + * formula's habit of ballooning under local slowdown, since network + * transit time does not degrade just because guest/replay cost did. */ + uint32_t scaled = rtt + rtt / 2u; + if (scaled > budget) + budget = scaled; + if (budget > 60u) /* hard ceiling regardless of measured RTT */ + budget = 60u; + } else if (s_tick_ema_ms) { + /* No RTT sample yet this session (before the first episode has + * round-tripped) — narrower and lower-ceiling than before, since + * this is a brief startup fallback, not the steady-state mechanism. */ + uint32_t scaled = (s_tick_ema_ms * 3u) / 2u; /* 1.5 ticks, was 2.5 */ + if (scaled > budget) + budget = scaled; + if (budget > 40u) /* was 100 */ + budget = 40u; + } + if (!s_budget_logged[slot]) { + s_budget_logged[slot] = 1; + fprintf(stderr, + "psxrecomp: rb invent grace budget=%u ms (floor=%d rtt=%u " + "tick_ema=%u) slot=%d wire=%u\n", + (unsigned)budget, s_grace_ms, (unsigned)rtt, + (unsigned)s_tick_ema_ms, slot, (unsigned)wire); + fflush(stderr); + } + if ((uint32_t)(now - s_t0[slot]) < budget) + return 1; + if (!s_expired[slot]) { + s_expired[slot] = 1; + if (++s_expire_streak >= 45u) { + s_expire_streak = 0; + s_off_until = now + 2000u; + fprintf(stderr, + "psxrecomp: rb invent grace OFF 2s (remote input " + "consistently later than %u ms)\n", + (unsigned)budget); + fflush(stderr); + } + } + return 0; +} + +/* Mispredict-driven timesync pacing. The first cut used a GGPO-style + * advantage metric (wire_need - highest received remote row) with an + * absolute +0.5 tick threshold — soak showed BOTH peers measure ~+0.6 + * ticks at the natural operating point (each side samples at the start + * of its own tick), so both throttled, neither "closed", and both + * tripped the off-guard while the mispredicts continued. The ground + * truth for "I am the ahead peer" is the mispredict itself: only the + * ahead side promotes real rows that arrived AFTER it already invented + * them WRONG — note_late() is gated on pads_differ in the reconcile + * caller (an earlier cut fired on every predicted-row resolution, which + * is just normal input-delay operation and swamped the signal). Each + * genuine mispredict adds ~half a tick of pacing debt (capped at 2 + * ticks); the admit path shaves the debt off at <=3 ms per tick. Zero + * cost in steady state (aligned phase -> no mispredicts -> no debt). + * Adaptive off: on real WAN transit every edge mispredicts no matter the + * phase, so debt keeps landing back at the cap — a streak of 12 + * consecutive cap-hits (no room to have drained between them) disables + * for 10 s. Earlier cut used "debt continuously nonzero for 5s", but an + * active mispredict burst legitimately keeps debt elevated while pacing + * is working exactly as intended — soak: 24/36 commits and 19/28 + * rewind-requests on the busier peer landed AFTER that off-guard fired, + * i.e. it disabled pacing precisely when the storm needed it most. Host + * pacing only — guest determinism unaffected. PSX_RB_TIMESYNC=0 + * disables. */ +static int g_ts_enabled = -1; +static uint32_t g_ts_tick_ema_ms; +static uint32_t g_ts_debt_ms; +static uint32_t g_ts_pegged_streak; /* consecutive mispredicts landing at/above cap */ +static uint32_t g_ts_off_until_ms; + +static void np_timesync_check_enabled(void) +{ + if (g_ts_enabled < 0) { + const char *e = getenv("PSX_RB_TIMESYNC"); + g_ts_enabled = (e && e[0]) ? (atoi(e) != 0) : 1; + } +} + +/* Reconcile saw a real remote row that contradicted what we invented for + * this tick — i.e. we are (locally) the mispredicting/ahead peer for this + * edge. Add pacing debt. + * Adaptive off (2026-08-01 soak): the first cut disabled after debt sat + * "continuously nonzero" for 5s — but during an active mispredict burst, + * debt SHOULD stay elevated for a while (each edge tops it back up before + * the previous slice fully drains); that isn't pacing failing, it's pacing + * doing its job under sustained load. Soak evidence: 24/36 episode commits + * and 19/28 rewind-requests on the busier peer landed AFTER that off-guard + * fired — it was disabling exactly when it was needed most. The real "this + * is transit latency, not phase skew" signal is debt landing AT THE CAP + * repeatedly with no room to have drained in between — track a streak of + * cap-hits instead of wall-clock nonzero time. */ +static void np_timesync_note_late(void) +{ + uint32_t now; + uint32_t add; + uint32_t cap; + + np_timesync_check_enabled(); + if (!g_ts_enabled) + return; + now = np_mono_ms(); + if (g_ts_off_until_ms != 0u && (int32_t)(now - g_ts_off_until_ms) < 0) + return; + g_ts_off_until_ms = 0u; + add = g_ts_tick_ema_ms ? g_ts_tick_ema_ms / 2u : 8u; + cap = g_ts_tick_ema_ms ? g_ts_tick_ema_ms * 2u : 34u; + if (g_ts_debt_ms >= cap) { + if (++g_ts_pegged_streak >= 12u) { + g_ts_pegged_streak = 0u; + g_ts_debt_ms = 0u; + g_ts_off_until_ms = now + 10000u; + fprintf(stderr, + "psxrecomp: rb timesync OFF 10s (mispredicts keep landing " + "at the pacing cap — transit latency, not phase skew)\n"); + fflush(stderr); + return; + } + } else { + g_ts_pegged_streak = 0u; + } + g_ts_debt_ms += add; + if (g_ts_debt_ms > cap) + g_ts_debt_ms = cap; +} + +/* Admit-side: shave pacing debt off at <=3 ms per tick. */ +static int np_timesync_throttle(uint32_t wire) +{ + static uint32_t s_last_wire; + static uint32_t s_last_wire_ms; + static uint32_t s_stall_until; + static uint8_t s_logged; + uint32_t now; + + np_timesync_check_enabled(); + if (!g_ts_enabled) + return 0; + now = np_mono_ms(); + if (wire != s_last_wire) { + if (s_last_wire != 0u && wire == s_last_wire + 1u) { + uint32_t dt = now - s_last_wire_ms; + if (dt >= 1u && dt <= 250u) + g_ts_tick_ema_ms = g_ts_tick_ema_ms + ? (7u * g_ts_tick_ema_ms + dt) / 8u + : dt; + } + s_last_wire = wire; + s_last_wire_ms = now; + if (g_ts_debt_ms > 0u) { + uint32_t slice = g_ts_debt_ms > 3u ? 3u : g_ts_debt_ms; + g_ts_debt_ms -= slice; + if (g_ts_debt_ms == 0u) + s_logged = 0; + s_stall_until = now + slice; + if (!s_logged) { + fprintf(stderr, + "psxrecomp: rb timesync pacing (debt=%u ms tick=%u ms — " + "shaving <=3 ms/tick)\n", + (unsigned)(g_ts_debt_ms + slice), + (unsigned)g_ts_tick_ema_ms); + fflush(stderr); + s_logged = 1; + } + } + } + if (s_stall_until != 0u && (int32_t)(now - s_stall_until) < 0) + return 1; + s_stall_until = 0u; + return 0; +} + /* Rollback admit: tip + invent remotes within P of remote tip; stall outside. * BattleShip phase_lock: invent only when wire_need <= highest_remote + P. */ static int np_try_admit_rollback(void) @@ -1598,6 +2040,54 @@ static int np_try_admit_rollback(void) memset(&st, 0, sizeof(st)); rnet_session_get_stats(g_np.session, &st); + /* TipHold past invent-cap: never invent (that caused tip-extend rereplay + * cliffs). Advance only when every remote wire row is present *and* all + * pads (local + remote) are idle (0xFFFF). Live-walking held digital + * through the runway fed MotK menu key-repeat (~24 extra navigations per + * press) and burned the coalesce window before tip-extend could absorb + * the release (soak: tip-extend=0, tip-hold→commit adjacent). Missing + * remotes or any held button → stall; wall-clock quiet / coalesce-ahead + * owns the runway. */ + if (psx_netplay_rb_tip_holding()) { + uint32_t tip = psx_netplay_rb_episode_target(); + uint32_t slack = psx_netplay_rb_tip_hold_invent_slack(); + if (tip > 0u && sim > tip + slack) { + int missing = 0; + int held = 0; + for (slot = 0; slot < g_np.slot_count; ++slot) { + if (slot == g_np.local_slot) { + if (rnet_session_peek_input(g_np.session, slot, wire, + &sample)) { + decode_pad(&sample, &pad); + if (pad.buttons != 0xFFFFu) + held = 1; + } else if (g_np.staged_valid && + g_np.staged.buttons != 0xFFFFu) { + held = 1; + } + continue; + } + if (!rnet_session_peek_remote_input(g_np.session, slot, wire, + &sample)) { + missing = 1; + break; + } + decode_pad(&sample, &pad); + if (pad.buttons != 0xFFFFu) + held = 1; + } + if (missing || held) + return 0; + } + } + + /* Phase alignment: the ahead peer paces down a few ms/tick so remote + * rows arrive before the seal (kills invent-mispredict episodes at the + * source). Never engages during episodes/lockstep — only live admits. */ + if (!psx_netplay_rb_active() && + np_timesync_throttle(wire)) + return 0; + for (slot = 0; slot < g_np.slot_count; ++slot) { if (slot == g_np.local_slot) { if (rnet_session_peek_input(g_np.session, slot, wire, &sample)) { @@ -1631,6 +2121,17 @@ static int np_try_admit_rollback(void) /* Stall when invent would run more than P ahead of remote tip. */ if (wire > st.highest_remote_wire + (rnet_u32)pred) return 0; + /* Brief stall before inventing — but ONLY when genuinely >=2 + * ticks ahead of the remote's sealed tip. A 1-tick gap is the + * normal phase offset of two pipelined sims: stalling on it + * serialized the peers (each admit waited out the other's + * guest frame; throughput fell to 1/(g1+g2) ~ 40-50 fps with + * zero episodes). Invent freely at gap<=1 — hold-last only + * mispredicts on real edges and episodes are cheap now that + * the agreed watermark tracks live. */ + if (wire > st.highest_remote_wire + 1u && + np_invent_grace_stall(slot, wire)) + return 0; /* MotK digital: hold-last. Idle invent re-mismatched every held * D-pad tick after commit → episode storm / char-select freeze. * Menu release soft-promote is off (see reconcile) so sticky Up diff --git a/runtime/src/psx_netplay_rb.c b/runtime/src/psx_netplay_rb.c index 038869bf..2c5a2366 100644 --- a/runtime/src/psx_netplay_rb.c +++ b/runtime/src/psx_netplay_rb.c @@ -40,6 +40,15 @@ void psx_netplay_rb_pump(void) {} int psx_netplay_rb_active(void) { return 0; } int psx_netplay_rb_is_resimulating(void) { return 0; } int psx_netplay_rb_tip_holding(void) { return 0; } +uint32_t psx_netplay_rb_episode_target(void) { return 0; } +uint32_t psx_netplay_rb_tip_hold_invent_slack(void) { return 0; } +uint32_t psx_netplay_rb_tip_runway(void) { return 0; } +int psx_netplay_rb_ignore_peer_frame_commit(uint32_t tick, uint32_t hash) +{ + (void)tick; + (void)hash; + return 0; +} int psx_netplay_rb_abort_resim_core_mismatch(uint32_t tick, uint32_t local_core, uint32_t peer_core) { @@ -60,12 +69,15 @@ void psx_netplay_rb_cpu_for_present_digest(struct CPUState *out, out->pc = 0; } uint32_t psx_netplay_rb_episode_count(void) { return 0; } +uint64_t psx_netplay_rb_take_replay_ticks(void) { return 0; } int psx_netplay_rb_phase(void) { return 0; } uint32_t psx_netplay_rb_snap_count(void) { return 0; } +uint32_t psx_netplay_rb_sticky_bb_pc(void) { return 0; } #else #include "netplay_hash_confirm.h" #include "netplay_input_hist.h" +#include "netplay_rb_post.h" #include "netplay_snap_ring.h" #include "netplay_state_digest.h" #include "boot_state.h" @@ -93,6 +105,11 @@ static RNetRbSession *g_rb; static NetplaySnapRing *g_snaps; static uint32_t g_epoch; static uint32_t g_episode_count; +/* Ticks armed into Replay, accumulated for the host's periodic [FPS] line + * (psx_netplay_rb_take_replay_ticks resets on read). Lets a soak confirm + * "we spent NN% of this window resimulating" instead of inferring it from + * episode density in the raw log. */ +static uint64_t g_stat_replay_ticks; static int g_pending_save_valid; static uint32_t g_pending_save_tick; static int g_pending_load_valid; @@ -138,6 +155,7 @@ static int g_peer_post_ok; static uint32_t g_peer_post_digest; static uint8_t g_peer_post_match; static uint32_t g_post_digest; +static uint32_t g_post_target; /* tip we POSTed; peer POST must match */ static uint32_t g_post_av; static uint32_t g_peer_post_av; static int g_needs_advance; @@ -146,6 +164,19 @@ static int g_baseline_rexmit_logged; static int g_post_rexmit_logged; static uint64_t g_seal_wait_ms; /* CLOCK_MONOTONIC ms when SealInputs began */ static uint64_t g_verify_wait_ms; +/* Live network-latency estimate, EMA'd from the POST handshake's own + * send/receive timestamps (no new wire message — reuses g_verify_wait_ms, + * already recorded for the verify-timeout mechanism). Sampled once per + * episode commit: elapsed time from "I sent my POST" to "peer's POST + * arrived here" when I sent first (the common case), which is dominated by + * one-way transit of the peer's packet plus its own send-side scheduling + * jitter — a genuine measurement of *the link*, unlike the local sim's own + * tick-to-tick cadence. See docs/ROLLBACK_MOTK_HOOKUP.md section 12 + * ("size wait-before-invent budgets from RTT, not local tick cadence") for + * why this replaced np_invent_grace_stall's old tick-cadence-based scaling + * term in psx_netplay.c. 0 = no sample yet (very start of a session, before + * the first episode has round-tripped). */ +static uint32_t g_rb_rtt_ema_ms; static uint64_t g_replay_progress_ms; /* last arm / finish_frame */ static uint32_t g_last_good_bb_pc; /* sticky BB-edge resume for snaps / digest */ static int g_follow_nack_pending; @@ -198,23 +229,49 @@ static uint32_t g_last_begin_mismatch = 0xffffffffu; #define RB_FOLLOW_NACK_REXMIT 24 #define RB_BASELINE_BURST 8 #define RB_BASELINE_BURST_MS 40u +/* After one peer accepts the other's POST and enters TipHold it used to stop + * retransmitting its own POST. On a single UDP loss the lagging peer then + * sat in Verify until RB_VERIFY_TIMEOUT_MS and aborted — while the winner + * had already advanced agreed_through to the unconfirmed tip. Burst POST + + * RESOLVED on tip-hold entry so the lagging peer can still complete Verify + * (see 2026-08-01 soak: tip=1437 peer_ok race → verify timeout → NACK storm). */ +#define RB_POST_BURST 8 /* Initiator waits for follower ready-ACK; do NOT solo-enter Replay. */ #define RB_READY_TIMEOUT_MS 4000u -/* Failed episode (resim/baseline/post): calm from *live* sim before realign. */ -#define RB_ABORT_COOLDOWN_TICKS 60u +/* Failed episode (resim/baseline/post): calm from *live* sim before realign. + * Was 60/120 — after abort, choose_load stuck at agreed tip so the next begin + * resimmed the whole cooldown (depth 63→128) = "pushing further back". */ +#define RB_ABORT_COOLDOWN_TICKS 24u /* Repeated abort streak — longer promote-only (char-select storm). */ -#define RB_STORM_COOLDOWN_TICKS 120u +#define RB_STORM_COOLDOWN_TICKS 48u +/* Cap initiator begin/follow-open resim span. Deep catch-up after abort is + * chunked (commit → next episode) instead of one 120-tick Replay. */ +#define RB_MAX_RESIM_SPAN 24u /* Keep deferring a few frames after MDEC goes quiet (cutover / credits). */ #define RB_FMV_MDEC_HYSTERESIS 8u /* After leaving FMV: defer rewind briefly so cutover heals. */ #define RB_FMV_SETTLE_TICKS 24u -/* Post-FMV no-invent lockstep (title Start / skip safe). Floor keeps invent - * off for at least MIN even when digests already agree; if cores have not - * matched for CONFIRM contiguous ticks by then, hold until they do or MAX - * (cross-game cutover — MotK was healthy at ~90, slower titles need gate). */ -#define RB_FMV_LOCKSTEP_MIN 90u -#define RB_FMV_LOCKSTEP_MAX 180u +/* Post-FMV no-invent lockstep (title Start / first menu safe). Floor keeps + * invent off for at least MIN even when digests already agree; if cores have + * not matched for CONFIRM contiguous ticks by then, hold until they do or + * MAX. MotK soak: MIN=90 unlocked invent at title→menu and invent≠Cross at + * +14 opened a 0x8006CDA0 tip storm — MIN=180 (~3s) covers first menu taps. */ +#define RB_FMV_LOCKSTEP_MIN 180u +#define RB_FMV_LOCKSTEP_MAX 300u #define RB_FMV_LOCKSTEP_CONFIRM 16u +/* After cores match (or MAX): keep invent off this many more ticks so admit + * waits for wire. Prevents hold-last sticky Up (pub=ffef vs wire=ffff) from + * inventing at unlock+1 → FIRST CORE DIVERGE / tip episode. Matches dense + * tip-snap +32 after invent unlock. Soak: grace=32 still saw Cross invent + * at invent_at+1 (896) — bump to 64. */ +#define RB_FMV_UNLOCK_GRACE 64u +/* MotK TipHold: quiet window so press→release→D-pad coalesce in one episode. + * invent slack 0 (UINT32_MAX sentinel) so Live never *invents* past tip + * (tip-extend rereplay was the menu FPS cliff). Runway is wall-clock frames + * at 60 Hz (not admit-pump spins — those finalized in ms and killed coalesce; + * 2026-08-01 soak: 0 tip-extends, 351 fresh light episodes). */ +#define RB_MOTK_TIP_RUNWAY 24u +#define RB_MOTK_TIP_SEAL_SLACK_FORCE0 0xffffffffu /* Baseline mismatch with no agreed tip — don't reopen every few ticks. */ #define RB_BASELINE_MISMATCH_COOLDOWN_TICKS 90u @@ -228,8 +285,14 @@ static int g_fmv_lockstep_released; /* sticky: never re-lock after invent on */ static uint32_t g_fmv_dense_through; static uint32_t g_bl_mismatch_streak; static uint32_t g_tip_hold_until; +/* TipHold invent-cap quiet: wall-clock ms when we first stalled at tip+slack + * (0 = not armed). Replaces the old pump-frame counter. */ +static uint64_t g_tip_hold_quiet_t0_ms; /* Mid-span tip-extend rereplay: reload snap without replacing episode pin. */ static int g_tip_extend_rereplay; +/* Inclusive tip of last tip-extend hc_prime; drop stale TipHold invent + * FRAME_COMMITs above this until Verify/TipHold. 0 = inactive. */ +static uint32_t g_tip_extend_prime_tick; static RNetSession *sess(void); /* fwd — used by FMV lockstep window */ static void commit_episode(void); /* fwd — legacy immediate commit */ @@ -237,6 +300,7 @@ static void enter_tip_hold(uint32_t target); static void enter_verify_at_tip(uint32_t done); static void finalize_tip_hold(void); static void poll_tip_hold_finalize(void); +static void clear_tip_extend_prime(void); static int rb_fmv_media_active(void) { @@ -244,6 +308,12 @@ static int rb_fmv_media_active(void) return 1; if (mdec_recently_active(RB_FMV_MDEC_HYSTERESIS)) return 1; + /* Real XA audio stream only. Do NOT treat reading+(mode&0x48) alone as + * media: MotK boot/LoadExe sets XA mode bits on ordinary CD reads. + * Rematch armed invent-lockstep at sim=0 (depth24=0 mdec=0 xa=0, + * fmv_pend=1 reading=1 mode=e0) and never presented a frame. */ + if (cdrom_xa_stream_active()) + return 1; return 0; } @@ -272,14 +342,21 @@ static void rb_fmv_update_lockstep_gate(uint32_t sim) prev_until = g_fmv_lockstep_until; cap = g_fmv_media_end_sim + RB_FMV_LOCKSTEP_MAX; if (sim >= cap || g_fmv_core_match_streak >= RB_FMV_LOCKSTEP_CONFIRM) { - if (g_fmv_lockstep_until > sim) - g_fmv_lockstep_until = sim; + /* Do not clamp until→sim: that unlocked invent on the same tick and + * hold-last poison (sticky Up) opened ep1 at unlock+1. Keep invent + * off for UNLOCK_GRACE while wire catches up; promote_sweep flushes + * any predicted rows already in lookback without a tip episode. */ + uint32_t grace_until = sim + RB_FMV_UNLOCK_GRACE; + if (grace_until > g_fmv_lockstep_until) + g_fmv_lockstep_until = grace_until; g_fmv_lockstep_released = 1; + g_promote_sweep = 1; fprintf(stderr, "psxrecomp: rb FMV lockstep RELEASE sim=%u streak=%u " - "cap=%u (invent on%s)\n", + "cap=%u invent_at=%u (grace=%u; promote_sweep%s)\n", (unsigned)sim, (unsigned)g_fmv_core_match_streak, - (unsigned)cap, + (unsigned)cap, (unsigned)g_fmv_lockstep_until, + (unsigned)RB_FMV_UNLOCK_GRACE, (sim >= cap) ? "; MAX" : "; cores matched"); fflush(stderr); (void)prev_until; @@ -321,6 +398,29 @@ static void rb_fmv_tick_settle(void) "psxrecomp: rb FMV rewind-defer ON (depth24/mdec; no invent)\n"); fflush(stderr); } + /* Heartbeat through the movie — digs are sparse while MDEC is hot. */ + { + static uint32_t s_fmv_hb_sim; + if (sim == 0u || (sim % 32u) == 0u) { + if (s_fmv_hb_sim != sim) { + CDROMDebugState cds; + memset(&cds, 0, sizeof(cds)); + cdrom_debug_snapshot(&cds); + fprintf(stderr, + "psxrecomp: rb FMV media sim=%u depth24=%d mdec=%d " + "xa=%d fmv_pend=%d cd_reading=%d mode=%02x " + "present_pend=%d\n", + (unsigned)sim, gpu_display_is_depth24(), + mdec_recently_active(RB_FMV_MDEC_HYSTERESIS), + cdrom_xa_stream_active(), + cdrom_fmv_stream_pending(), cds.reading, + (unsigned)cds.mode_reg, + gpu_vblank_present_pending()); + fflush(stderr); + s_fmv_hb_sim = sim; + } + } + } return; } if (g_was_in_fmv) { @@ -449,10 +549,15 @@ static uint32_t pick_snap_resume_pc(const CPUState *c, uint32_t hint) return 0; } +uint32_t psx_netplay_rb_sticky_bb_pc(void) +{ + return g_last_good_bb_pc; +} + static void note_good_bb_pc(uint32_t pc) { if (rb_resume_pc_ok(pc)) - g_last_good_bb_pc = pc; + g_last_good_bb_pc = rb_canonicalize_resume_pc(pc); } void psx_netplay_rb_cpu_for_present_digest(struct CPUState *out, @@ -520,6 +625,7 @@ static void clear_episode_wire_state(void) g_peer_post_ok = 0; g_post_av = 0; g_peer_post_av = 0; + g_post_target = 0; g_baseline_rexmit_logged = 0; g_post_rexmit_logged = 0; g_seal_wait_ms = 0; @@ -599,7 +705,9 @@ static void abort_episode(const char *why) g_needs_advance = 0; g_episode_baseline_matched = 0; g_tip_hold_until = 0; + g_tip_hold_quiet_t0_ms = 0ull; g_tip_extend_rereplay = 0; + clear_tip_extend_prime(); psx_scheduler_top_level_resume_clear(); clear_episode_wire_state(); if (g_rb && rnet_rb_is_active(g_rb)) { @@ -791,6 +899,199 @@ static uint64_t rb_mono_ms(void) return (uint64_t)ts.tv_sec * 1000ull + (uint64_t)ts.tv_nsec / 1000000ull; } +/* Raise agreed_through to the newest interval snap still in the ring. + * Used when hc cannot confirm anything in-ring (live core forks block + * heal_stale_gap) so begin/follow are not stuck REFUSED forever. */ +static int raise_agreed_to_newest_interval_snap(uint32_t oldest, uint32_t newest, + uint32_t iv, const char *why, + uint32_t hc_rt) +{ + uint32_t t; + uint32_t old; + if (iv < 1u) + iv = 1u; + t = newest; + for (;;) { + if (netplay_snap_ring_has(g_snaps, t) && + ((iv <= 1u) || ((t % iv) == 0u))) { + old = g_agreed_valid ? g_agreed_through : 0u; + if (g_agreed_valid && t <= g_agreed_through) + return 0; + g_agreed_through = t; + g_agreed_span_lo = t; + g_agreed_valid = 1; + fprintf(stderr, + "psxrecomp: rb agreed %s %u→%u (hc=%u oldest=%u)\n", + why, (unsigned)old, (unsigned)t, (unsigned)hc_rt, + (unsigned)oldest); + fflush(stderr); + return 1; + } + if (t <= oldest) + break; + t--; + } + return 0; +} + +/* When tip-hold/commit agreed_through ages out of the snap ring, begin would + * REFUSE forever (hard watermark ignores hc) while invent≠wire keeps forking + * Live — soak: agreed=919, oldest=935, hc=1824, cores diverged by 1856 with + * matched clocks. Raise agreed to the highest hc-confirmed snap still held. + * Do NOT raise while any agreed-era snap remains (TipHold false-confirm + * guard: never load above a live agreed tip via hc alone). + * + * If hc is stuck below ring oldest (heal_stale_gap blocked by live forks), + * force-raise to the newest interval snap — otherwise slot-0 wire corrections + * never open (soak: hc=714, oldest=982, agreed never tip-hold set). */ +static void heal_agreed_watermark_if_aged_out(void) +{ + uint32_t oldest; + uint32_t newest; + uint32_t rt; + uint32_t t; + uint32_t old; + uint32_t iv; + int any_agreed_snap = 0; + + if (!g_snaps || !g_b.hc) + return; + if (netplay_snap_ring_count(g_snaps) == 0) + return; + + oldest = netplay_snap_ring_oldest_tick(g_snaps); + newest = netplay_snap_ring_newest_tick(g_snaps); + iv = snap_interval(); + if (iv < 1u) + iv = 1u; + + (void)netplay_hc_heal_stale_gap(g_b.hc); + rt = netplay_hc_resolved_through(g_b.hc); + + /* Never tip-hold committed: hc watermark below ring → bootstrap agreed + * so choose_load / follow can open episodes for late wire (host P1). */ + if (!g_agreed_valid) { + if (netplay_hc_confirm_through(g_b.hc, rt) && rt >= oldest) { + /* hc still usable as soft frontier — no agreed seed needed. */ + return; + } + if (rt < oldest || !netplay_hc_confirm_through(g_b.hc, rt)) { + (void)raise_agreed_to_newest_interval_snap( + oldest, newest, iv, "BOOTSTRAP", rt); + } + return; + } + + if (oldest <= g_agreed_through) { + uint32_t hi = g_agreed_through; + if (hi > newest) + hi = newest; + if (hi >= oldest) { + for (t = hi;; --t) { + if (netplay_snap_ring_has(g_snaps, t)) { + any_agreed_snap = 1; + break; + } + if (t <= oldest) + break; + } + } + } + if (any_agreed_snap) + return; + + if (netplay_hc_confirm_through(g_b.hc, rt) && rt > g_agreed_through) { + t = rt; + if (t > newest) + t = newest; + for (;;) { + /* Interval snaps only — matches choose_load mutual under span_lo=t. */ + if (netplay_snap_ring_has(g_snaps, t) && + ((iv <= 1u) || ((t % iv) == 0u)) && + netplay_hc_confirm_through(g_b.hc, t)) { + old = g_agreed_through; + g_agreed_through = t; + g_agreed_span_lo = t; + fprintf(stderr, + "psxrecomp: rb agreed HEAL %u→%u (aged out of ring; " + "hc=%u oldest=%u)\n", + (unsigned)old, (unsigned)t, (unsigned)rt, + (unsigned)oldest); + fflush(stderr); + return; + } + if (t == 0u) + break; + t--; + } + } + + /* Agreed tip gone and hc cannot confirm in-ring (live forks) — force. */ + if (g_agreed_through < oldest) + (void)raise_agreed_to_newest_interval_snap( + oldest, newest, iv, "HEAL-FORCE", rt); +} + +/* Advance agreed_through along hash-confirmed clean Live play. The hard + * watermark used to move ONLY at episode commits: after a tip-hold commit, + * hundreds of hc-confirmed live ticks (identical peer digs, soak 1344..2112) + * were ignored, so the next edge reloaded from an ancient snap + * (mismatch=2135 load=1361) and SPAN CAP chunked 24 ticks per episode while + * Live galloped ahead — nonstop char-select resim storms with matching + * states. The historical false-confirm source behind the hard cap (TipHold + * Live invent FRAME_COMMITs with cleared PCs confirming forked tip snaps) + * is structurally gone: TipHold Live no longer emits FRAME_COMMITs, and the + * core digest now folds full CPU split + csv + icache. Advance only outside + * episodes/TipHold/pending loads, to hc-confirmed interval snaps in-ring; + * span_lo collapses to the new watermark (no dense span above a live + * advance — only interval ticks are mutual). */ +static void advance_agreed_watermark_from_hc(void) +{ + uint32_t rt; + uint32_t t; + uint32_t newest; + uint32_t oldest; + uint32_t iv; + + if (!g_agreed_valid || !g_snaps || !g_b.hc) + return; + if (psx_netplay_rb_active() || psx_netplay_rb_tip_holding() || + psx_netplay_rb_load_pending()) + return; + if (netplay_snap_ring_count(g_snaps) == 0) + return; + rt = netplay_hc_resolved_through(g_b.hc); + if (rt <= g_agreed_through || !netplay_hc_confirm_through(g_b.hc, rt)) + return; + newest = netplay_snap_ring_newest_tick(g_snaps); + oldest = netplay_snap_ring_oldest_tick(g_snaps); + iv = snap_interval(); + if (iv < 1u) + iv = 1u; + t = rt; + if (t > newest) + t = newest; + for (;;) { + if (t <= g_agreed_through) + return; + if (netplay_snap_ring_has(g_snaps, t) && + ((iv <= 1u) || ((t % iv) == 0u)) && + netplay_hc_confirm_through(g_b.hc, t)) { + uint32_t old = g_agreed_through; + g_agreed_through = t; + g_agreed_span_lo = t; + fprintf(stderr, + "psxrecomp: rb agreed ADVANCE %u→%u (hc=%u live confirm)\n", + (unsigned)old, (unsigned)t, (unsigned)rt); + fflush(stderr); + return; + } + if (t <= oldest) + return; + t--; + } +} + /* Pick a load snap ≤ mismatch that the lagging peer is likely to have. * Live snaps only land on snap_interval ticks; never invent a phantom tick. * @@ -798,7 +1099,7 @@ static uint64_t rb_mono_ms(void) * - hc resolved_through: both peers simmed those ticks and core digests * matched, so both saved the same interval snaps. * - last committed span (g_agreed_span_lo, g_agreed_through]: both peers - * replayed it and resim saves a snap at EVERY replayed tick (dense). + * replayed it and resim saves a snap at every replayed tick (dense). * Without this, mismatch one tick after a commit re-loaded a full interval + * slack deep (798 → 768) and each menu tap replayed ~30 ticks twice — the * main-menu "resim back to title" hang. Peer eviction is still covered by the @@ -825,16 +1126,19 @@ static int choose_load_tick(uint32_t mismatch, uint32_t *out_load) if (g_b.hc) (void)netplay_hc_heal_stale_gap(g_b.hc); + heal_agreed_watermark_if_aged_out(); + advance_agreed_watermark_from_hc(); if (g_agreed_valid) { + /* Hard watermark: tip-hold / episode commit tip. TipHold Live invent + * + PC-cleared FRAME_COMMIT can false-confirm tip snaps (848) that + * still fork baseline cores/av — never load above this via hc while + * the agreed snap is still in the ring (heal raises it when aged out). */ shared = g_agreed_through; have_shared = 1; - } - if (g_b.hc) { + } else if (g_b.hc) { uint32_t rt = netplay_hc_resolved_through(g_b.hc); - /* Only raise the walk ceiling on confirmed ticks — resolved_through - * alone can sit on TipHold Live tip snaps the peer never agreed. */ - if (netplay_hc_confirm_through(g_b.hc, rt) && rt >= shared) { + if (netplay_hc_confirm_through(g_b.hc, rt)) { shared = rt; have_shared = 1; } @@ -846,36 +1150,44 @@ static int choose_load_tick(uint32_t mismatch, uint32_t *out_load) uint32_t oldest = netplay_snap_ring_oldest_tick(g_snaps); if (hi > mismatch - 1u) hi = mismatch - 1u; - /* Bound the walk to one ring depth of ticks. */ - lo = (hi > 255u) ? hi - 255u : 0u; - if (oldest > lo) - lo = oldest; - for (t = hi;; --t) { - int mutual = 0; - /* TipHold Live saves tip snaps past the POST tip; those are - * local-only until hash_confirm. Never treat bare interval - * alignment above agreed_through as peer-safe (follower NACK - * past frontier=tip-hold commit). */ - if (g_b.hc && netplay_hc_confirm_through(g_b.hc, t)) - mutual = 1; - else if (g_agreed_valid && t <= g_agreed_through && - (t > g_agreed_span_lo || (iv <= 1u) || ((t % iv) == 0u))) - mutual = 1; - else if (!g_agreed_valid && ((iv <= 1u) || ((t % iv) == 0u))) - mutual = 1; - else if (g_fmv_dense_through > 0u && t <= g_fmv_dense_through && - t >= g_fmv_media_end_sim) - mutual = 1; - if (mutual && netplay_snap_ring_has(g_snaps, t)) { - *out_load = t; - return 1; + /* Watermark entirely below the ring: walk cannot succeed. Fall through + * to tip-slack (BOOTSTRAP/HEAL should have raised agreed; if not, + * refuse would stick forever with invent≠wire). */ + if (shared < oldest) { + have_shared = 0; + } else { + /* Bound the walk to one ring depth of ticks. */ + lo = (hi > 255u) ? hi - 255u : 0u; + if (oldest > lo) + lo = oldest; + for (t = hi;; --t) { + int mutual = 0; + if (g_agreed_valid) { + /* Only committed / tip-hold POST tip and its dense resim span. */ + if (t <= g_agreed_through && + (t > g_agreed_span_lo || (iv <= 1u) || ((t % iv) == 0u))) + mutual = 1; + } else if (g_b.hc && netplay_hc_confirm_through(g_b.hc, t)) { + mutual = 1; + } else if ((iv <= 1u) || ((t % iv) == 0u)) { + mutual = 1; + } else if (g_fmv_dense_through > 0u && t <= g_fmv_dense_through && + t >= g_fmv_media_end_sim) { + mutual = 1; + } + if (mutual && netplay_snap_ring_has(g_snaps, t)) { + *out_load = t; + return 1; + } + if (t <= lo) + break; } - if (t <= lo) - break; + /* Shared frontier in range but no snap left — refuse rather than + * load an unconfirmed tip above a live agreed watermark. */ + if (g_agreed_valid) + return 0; + have_shared = 0; } - /* Shared frontier exists but every confirmed snap aged out of the - * tip-dense ring — refuse rather than load an unconfirmed tip. */ - return 0; } cap = mismatch; if (newest >= iv && cap > newest - iv) @@ -1019,7 +1331,24 @@ static void enter_awaiting_baseline(void) static void accept_peer_baseline(uint32_t epoch, uint32_t load, uint32_t dig_m, uint32_t dig_a, uint32_t dig_b, uint32_t dig_c) { - (void)epoch; + /* Dedup vs the last *logged* copy of this exact message. RB_BASELINE_BURST + * sends up to 8 redundant UDP copies per baseline (TURN loses one-shots), + * from up to ~9 call sites across an episode's phases — soaks showed + * ~25 identical "peer baseline" lines (each with its own fflush) per + * episode, none carrying new information beyond the first. During a + * dense episode chain that's real fflush/syscall overhead landing right + * on the ticks already busy resimulating. The accepted state + * (g_peer_baseline_ok/digest/av/aux/ready) is still updated + * unconditionally on every copy — only the diagnostic print is gated on + * the tuple actually changing (epoch bump / phase transition / a + * genuinely different digest would all show up here). */ + static uint32_t s_log_epoch = 0xffffffffu; + static uint32_t s_log_load; + static uint32_t s_log_dig_m; + static uint32_t s_log_dig_a; + static uint32_t s_log_dig_b; + static uint32_t s_log_dig_c; + int changed; (void)load; g_peer_baseline_ok = 1; g_peer_baseline_digest = dig_m; @@ -1034,13 +1363,23 @@ static void accept_peer_baseline(uint32_t epoch, uint32_t load, uint32_t dig_m, } g_peer_baseline_ready = 1; } - fprintf(stderr, - "psxrecomp: rb peer baseline epoch=%u load=%u core=%08x av=%08x ext=%08x " - "ready=%u (local_applied=%d phase=%d)\n", - (unsigned)epoch, (unsigned)load, (unsigned)dig_m, (unsigned)dig_b, - (unsigned)dig_c, (unsigned)(dig_a & RB_BL_FLAG_READY), - g_episode_snap_applied, (int)rnet_rb_get_phase(g_rb)); - fflush(stderr); + changed = (epoch != s_log_epoch || load != s_log_load || dig_m != s_log_dig_m || + dig_a != s_log_dig_a || dig_b != s_log_dig_b || dig_c != s_log_dig_c); + if (changed) { + s_log_epoch = epoch; + s_log_load = load; + s_log_dig_m = dig_m; + s_log_dig_a = dig_a; + s_log_dig_b = dig_b; + s_log_dig_c = dig_c; + fprintf(stderr, + "psxrecomp: rb peer baseline epoch=%u load=%u core=%08x av=%08x ext=%08x " + "ready=%u (local_applied=%d phase=%d)\n", + (unsigned)epoch, (unsigned)load, (unsigned)dig_m, (unsigned)dig_b, + (unsigned)dig_c, (unsigned)(dig_a & RB_BL_FLAG_READY), + g_episode_snap_applied, (int)rnet_rb_get_phase(g_rb)); + fflush(stderr); + } maybe_send_baseline(); maybe_enter_replay(); } @@ -1202,6 +1541,18 @@ static void maybe_send_baseline(void) } } +/* Cheap per-replayed-tick audit: core digest only (registers + clock/IRQ + + * timers + full RAM CRC — no VRAM/CD/aux/scratchpad/DMA/SIO). Used for both + * "arm" and "fin" tags on every tick of a resim span. + * + * This used to also carry the *full* bus digest breakdown (av = full VRAM + * CRC, plus cd/aux/spad/dma/sio/master) on every "fin" call — i.e. on every + * replayed tick, not just the last one — purely to print it. During a dense + * back-to-back episode chain (see 2026-08-01 episode-cost note) that meant + * paying a full ~3 MiB CRC sweep per replayed tick, and then paying most of + * it *again* moments later in enter_verify_at_tip() for the real POST + * digest on the commit tick. The full breakdown now runs exactly once, in + * enter_verify_at_tip(), right where its values are actually needed. */ static void log_resim_tick_audit(uint32_t sim, const char *tag) { CPUState *c = cpu(); @@ -1209,47 +1560,22 @@ static void log_resim_tick_audit(uint32_t sim, const char *tag) int n = g_b.slot_count ? *g_b.slot_count : 0; uint32_t dig = 0u; NetplayCoreParts parts; - uint32_t av = 0u; - uint32_t cd = 0u; - uint32_t aux = 0u; - uint32_t spad = 0u; - uint32_t dma = 0u; - uint32_t sio = 0u; - int want_parts = (tag && tag[0] == 'f'); /* "fin" — partition breakdown */ - int present_edge = want_parts; /* fin is at vblank present */ memset(&parts, 0, sizeof(parts)); if (c) { - CPUState dig_cpu; - if (present_edge) { - log_cpu_digest_split(c, tag); - psx_netplay_rb_cpu_for_present_digest(&dig_cpu, c); - } else { - dig_cpu = *c; - if (!rb_resume_pc_ok(dig_cpu.pc)) { - uint32_t alt = g_last_good_bb_pc; - if (rb_resume_pc_ok(alt)) - dig_cpu.pc = alt; - } - } - if (want_parts) { - /* fin only: full bus digests (av = full VRAM CRC — skip on arm). */ - netplay_core_digest_parts(&dig_cpu, &parts); - av = netplay_av_digest(); - cd = netplay_cdrom_digest(); - aux = netplay_aux_digest(); - spad = netplay_spad_digest(); - dma = netplay_dma_digest(); - sio = netplay_sio_digest(); - dig = netplay_master_digest(&dig_cpu); - } else { - parts.core = netplay_core_digest(&dig_cpu); - dig = parts.core; /* arm: skip master/cd/av — diag only */ + CPUState dig_cpu = *c; + if (!rb_resume_pc_ok(dig_cpu.pc)) { + uint32_t alt = g_last_good_bb_pc; + if (rb_resume_pc_ok(alt)) + dig_cpu.pc = alt; } + parts.core = netplay_core_digest(&dig_cpu); + dig = parts.core; } fprintf(stderr, - "psxrecomp: rb audit %s sim=%u dig=%08x core=%08x av=%08x cd=%08x cyc=%llu", + "psxrecomp: rb audit %s sim=%u dig=%08x core=%08x av=00000000 " + "cd=00000000 cyc=%llu", tag ? tag : "?", (unsigned)sim, (unsigned)dig, (unsigned)parts.core, - (unsigned)av, (unsigned)cd, (unsigned long long)psx_cycle_count); + (unsigned long long)psx_cycle_count); for (slot = 0; slot < n; ++slot) { RNetRbFrame row; if (rnet_rb_get_sealed_frame(g_rb, slot, sim, &row) && row.is_valid) @@ -1258,19 +1584,6 @@ static void log_resim_tick_audit(uint32_t sim, const char *tag) else fprintf(stderr, " s%d=----", slot); } - if (want_parts) { - NetplaySioParts sp; - netplay_sio_digest_parts(&sp); - fprintf(stderr, - " | cpu=%08x clk=%08x tim=%08x ram=%08x dirty=%08x aux=%08x " - "spad=%08x dma=%08x sio=%08x sioP=%08x/%08x/%08x/%08x/%08x", - (unsigned)parts.cpu, (unsigned)parts.clock_irq, - (unsigned)parts.timers, (unsigned)parts.ram, - (unsigned)parts.dirty, (unsigned)aux, (unsigned)spad, - (unsigned)dma, (unsigned)sio, - (unsigned)sp.regs, (unsigned)sp.pads, (unsigned)sp.mc, - (unsigned)sp.pace, (unsigned)sp.meta); - } fprintf(stderr, "\n"); fflush(stderr); } @@ -1278,6 +1591,7 @@ static void log_resim_tick_audit(uint32_t sim, const char *tag) static void arm_replay_tick(uint32_t sim) { publish_sealed_sio(sim); + g_stat_replay_ticks++; g_needs_advance = 1; g_replay_progress_ms = rb_mono_ms(); fprintf(stderr, "psxrecomp: rb arm sim=%u (target=%u)\n", (unsigned)sim, @@ -1296,8 +1610,10 @@ static void arm_rereplay_after_load(uint32_t reload_tick) return; target = rnet_rb_get_target_tick(g_rb); first = reload_tick + 1u; + /* Drop TipHold-Live invent FRAME_COMMITs; sealed resim rebuilds the span. */ if (g_b.hc) netplay_hc_prime_after(g_b.hc, reload_tick); + g_tip_extend_prime_tick = reload_tick; g_episode_snap_applied = 1; g_pending_load_valid = 0; if (first > target) { @@ -1306,6 +1622,9 @@ static void arm_rereplay_after_load(uint32_t reload_tick) enter_verify_at_tip(reload_tick); return; } + /* Reset Live tip clock — TipHold invent may sit past the prior tip; arming + * that tip skipped the sealed span (870 snap → arm 873) and compared + * against stale peer invent digests. */ rnet_session_set_sim_tick(s, first); arm_replay_tick(first); } @@ -1316,27 +1635,49 @@ static void enter_verify_at_tip(uint32_t done) { RNetSession *s = sess(); CPUState *c = cpu(); + uint32_t cd; if (!g_rb || !s) return; + /* av/cd computed once here and reused by both prints below plus the wire + * send. Previously av/aux/core were each computed a second time here + * *in addition to* a full copy already computed moments earlier by the + * per-tick "fin" audit on this same tick (see log_resim_tick_audit) — + * i.e. two full ~3 MiB CRC sweeps per commit instead of one. */ + g_post_av = netplay_av_digest(); + cd = netplay_cdrom_digest(); if (c) { CPUState dig_cpu; + NetplayCoreParts parts; + NetplaySioParts sp; + uint32_t aux, spad, dma, sio, master, core, fold; log_cpu_digest_split(c, "post"); psx_netplay_rb_cpu_for_present_digest(&dig_cpu, c); - { - uint32_t core = netplay_core_digest(&dig_cpu); - uint32_t aux = netplay_aux_digest(); - uint32_t fold = 0xFFFFFFFFu; - fold = crc32_update(fold, (const uint8_t *)&core, sizeof(core)); - fold = crc32_update(fold, (const uint8_t *)&aux, sizeof(aux)); - g_post_digest = fold ^ 0xFFFFFFFFu; - fprintf(stderr, - "psxrecomp: rb post parts core=%08x aux=%08x (wire dig_m=%08x)\n", - (unsigned)core, (unsigned)aux, (unsigned)g_post_digest); - } + netplay_core_digest_parts(&dig_cpu, &parts); + aux = netplay_aux_digest(); + spad = netplay_spad_digest(); + dma = netplay_dma_digest(); + sio = netplay_sio_digest(); + netplay_sio_digest_parts(&sp); + master = netplay_master_digest(&dig_cpu); + core = parts.core; + fold = 0xFFFFFFFFu; + fold = crc32_update(fold, (const uint8_t *)&core, sizeof(core)); + fold = crc32_update(fold, (const uint8_t *)&aux, sizeof(aux)); + g_post_digest = fold ^ 0xFFFFFFFFu; + fprintf(stderr, + "psxrecomp: rb post parts core=%08x aux=%08x (wire dig_m=%08x) " + "dig=%08x | cpu=%08x clk=%08x tim=%08x ram=%08x dirty=%08x " + "spad=%08x dma=%08x sio=%08x sioP=%08x/%08x/%08x/%08x/%08x\n", + (unsigned)core, (unsigned)aux, (unsigned)g_post_digest, + (unsigned)master, (unsigned)parts.cpu, (unsigned)parts.clock_irq, + (unsigned)parts.timers, (unsigned)parts.ram, + (unsigned)parts.dirty, (unsigned)spad, (unsigned)dma, + (unsigned)sio, (unsigned)sp.regs, (unsigned)sp.pads, + (unsigned)sp.mc, (unsigned)sp.pace, (unsigned)sp.meta); } else { g_post_digest = 0u; } - g_post_av = netplay_av_digest(); + g_post_target = done; rnet_rb_set_phase(g_rb, nRNetRbPhaseVerify); g_verify_wait_ms = rb_mono_ms(); (void)rnet_session_send_rb_post(s, rnet_rb_get_epoch_id(g_rb), done, g_post_digest, @@ -1344,9 +1685,9 @@ static void enter_verify_at_tip(uint32_t done) g_local_post_sent = 1; g_post_rexmit_logged = 0; fprintf(stderr, - "psxrecomp: rb post sent dig_m=%08x av=%08x cd=%08x (peer_ok=%d)\n", - (unsigned)g_post_digest, (unsigned)g_post_av, - (unsigned)netplay_cdrom_digest(), g_peer_post_ok); + "psxrecomp: rb post sent tip=%u dig_m=%08x av=%08x cd=%08x (peer_ok=%d)\n", + (unsigned)g_post_target, (unsigned)g_post_digest, (unsigned)g_post_av, + (unsigned)cd, g_peer_post_ok); fflush(stderr); if (g_peer_post_ok) { if (g_peer_post_digest == g_post_digest && g_peer_post_av == g_post_av) @@ -1450,8 +1791,18 @@ static void maybe_enter_replay(void) send_baseline_burst(1, RB_BASELINE_BURST, 0); } } else { - g_peer_baseline_ready = 1; /* treat as ready — digests already matched */ - fprintf(stderr, "psxrecomp: rb light-tip skip ready-ACK load=%u target=%u\n", + /* Digests match and load is tip-aligned on *this* peer — do not wait + * for the ready-ACK RTT before entering Replay. Still *emit* ready/GO + * so a peer that classified the same episode as non-light (its + * resolved_through was ahead of load after a unilateral HC advance) + * is not stranded in wait-for-GO until RB_READY_TIMEOUT_MS — that + * asymmetry was the 4 fps cliff in the 2026-08-01 soak (initiator + * light-skipped, follower full-path timed out). */ + g_peer_baseline_ready = 1; + send_baseline_burst(1, RB_BASELINE_BURST, 0); + fprintf(stderr, + "psxrecomp: rb light-tip skip wait load=%u target=%u " + "(still emitted ready/GO for asymmetric peer)\n", (unsigned)rnet_rb_get_load_tick(g_rb), (unsigned)rnet_rb_get_target_tick(g_rb)); fflush(stderr); @@ -1507,33 +1858,66 @@ static void clear_post_handshake(void) g_peer_post_digest = 0; g_peer_post_av = 0; g_peer_post_match = 0; + g_post_target = 0; g_post_rexmit_logged = 0; g_verify_wait_ms = 0; } +static void clear_tip_extend_prime(void) +{ + g_tip_extend_prime_tick = 0; +} + static void enter_tip_hold(uint32_t target) { RNetSession *s = sess(); uint32_t runway; uint32_t sim; uint32_t base; + int i; + /* Capture POST digests before clear_post_handshake — we still owe the + * peer a reliable delivery of our POST even after we leave Verify. */ + uint32_t post_dig = g_post_digest; + uint32_t post_av = g_post_av; + uint32_t post_tip = g_post_target ? g_post_target : target; + int had_local_post = g_local_post_sent; if (!g_rb || !rnet_rb_enter_tip_hold(g_rb)) return; g_agreed_through = target; g_agreed_span_lo = g_episode_load_tick; g_agreed_valid = 1; - if (s) - (void)rnet_session_send_rb_resolved(s, target); + /* Drop invent-hold FRAME_COMMIT confirms past this tip. Without priming, + * advance_agreed_watermark_from_hc immediately re-ADVANCE'd over the + * hold-last ticks that both peers matched before the release arrived + * (soak: release commit tip=1607 then agreed ADVANCE 1607→1632 from + * stale HC — ghost second-release episodes / double menu inputs). */ + if (g_b.hc) + netplay_hc_prime_after(g_b.hc, target); + rnet_rb_set_peer_convergence(g_rb, target); + if (s) { + /* Burst RESOLVED + POST: the peer that has not yet received our POST + * is still in Verify and will time out unless these land. One-shot + * UDP was the 2026-08-01 "peer POST missing" / tip-extend NACK storm. */ + for (i = 0; i < RB_POST_BURST; ++i) { + (void)rnet_session_send_rb_resolved(s, target); + if (had_local_post) + (void)rnet_session_send_rb_post(s, rnet_rb_get_epoch_id(g_rb), + post_tip, post_dig, post_av, 1u); + } + } runway = rnet_rb_get_tip_runway(g_rb); sim = s ? rnet_session_sim_tick(s) : target; /* Quiet window from the later of sealed tip and live tip — Live may already * sit past target when tip-extend rereplay returns to TipHold. */ base = (sim > target) ? sim : target; g_tip_hold_until = base + runway; + g_tip_hold_quiet_t0_ms = 0ull; /* arm on first stalled poll */ g_needs_advance = 0; clear_post_handshake(); - fprintf(stderr, "psxrecomp: rb tip-hold through=%u until=%u\n", (unsigned)target, - (unsigned)g_tip_hold_until); + clear_tip_extend_prime(); + fprintf(stderr, "psxrecomp: rb tip-hold through=%u until=%u invent_slack=%u\n", + (unsigned)target, (unsigned)g_tip_hold_until, + (unsigned)rnet_rb_get_tip_seal_slack(g_rb)); fflush(stderr); } @@ -1544,11 +1928,19 @@ static void finalize_tip_hold(void) return; target = rnet_rb_get_target_tick(g_rb); g_episode_count++; + g_agreed_through = target; + g_agreed_span_lo = target; + g_agreed_valid = 1; + if (g_b.hc) + netplay_hc_prime_after(g_b.hc, target); + rnet_rb_set_peer_convergence(g_rb, target); fprintf(stderr, "psxrecomp: rb episode commit through=%u (tip-hold)\n", (unsigned)target); fflush(stderr); rnet_rb_session_reset(g_rb); g_tip_hold_until = 0; + g_tip_hold_quiet_t0_ms = 0ull; g_tip_extend_rereplay = 0; + clear_tip_extend_prime(); g_needs_advance = 0; g_episode_snap_applied = 0; g_pending_resume_valid = 0; @@ -1561,23 +1953,76 @@ static void poll_tip_hold_finalize(void) { RNetSession *s = sess(); uint32_t sim; + uint32_t tip; + uint32_t slack; + uint32_t invent_cap; + uint32_t runway; + uint64_t now; + uint64_t need_ms; if (!g_rb || !rnet_rb_is_tip_holding(g_rb) || !s) return; sim = rnet_session_sim_tick(s); - if (g_tip_hold_until > 0u && sim > g_tip_hold_until) + tip = rnet_rb_get_target_tick(g_rb); + slack = rnet_rb_get_tip_seal_slack(g_rb); + invent_cap = tip + slack; + runway = rnet_rb_get_tip_runway(g_rb); + /* Confirmed Live walked past the coalesce window (wire present during + * tip-hold — see np_try_admit_rollback). */ + if (g_tip_hold_until > 0u && sim > g_tip_hold_until) { finalize_tip_hold(); + return; + } + /* Invent-cap stall (MotK slack=0 ⇒ sim stuck at tip waiting for wire): + * wait runway frames of *wall clock* at 60 Hz. The old pump-frame counter + * hit "24" in a few ms of admit spinning and committed before the paired + * release edge could tip-extend (soak: 0 tip-extends). Tip-extend resets + * quiet_t0 so an active coalesce keeps the episode open. */ + if (sim >= invent_cap) { + now = rb_mono_ms(); + if (g_tip_hold_quiet_t0_ms == 0ull) + g_tip_hold_quiet_t0_ms = now; + need_ms = (uint64_t)runway * 1000ull / 60ull; + if (need_ms < 80ull) + need_ms = 80ull; + if (need_ms > 500ull) + need_ms = 500ull; + if (runway > 0u && now >= g_tip_hold_quiet_t0_ms + need_ms) + finalize_tip_hold(); + } else { + g_tip_hold_quiet_t0_ms = 0ull; + } } -static void schedule_episode_rereplay(uint32_t mismatch_tick) +/* prefer_plus_one: pass prior_tip+1 so prefer==prior tip (initiator and + * FOLLOW must agree). Walk down toward episode load before falling back to + * the baseline pin — mismatch-1 missing used to pin-reload on initiator + * while FOLLOW reloaded the prior tip → POST diverge. */ +static void schedule_episode_rereplay(uint32_t prefer_plus_one) { uint32_t load = rnet_rb_get_load_tick(g_rb); uint32_t reload = load; - uint32_t prefer = (mismatch_tick > 0u) ? mismatch_tick - 1u : 0u; + uint32_t prefer = (prefer_plus_one > 0u) ? prefer_plus_one - 1u : 0u; + uint32_t t; - if (prefer >= load && g_snaps && netplay_snap_ring_has(g_snaps, prefer)) + if (prefer < load) + prefer = load; + + if (g_snaps && netplay_snap_ring_has(g_snaps, prefer)) { reload = prefer; - else if (g_pin_valid && g_pin_tick >= load) + } else if (g_snaps) { + for (t = prefer; t > load; --t) { + if (netplay_snap_ring_has(g_snaps, t)) { + reload = t; + break; + } + } + if (reload == load && netplay_snap_ring_has(g_snaps, load)) + reload = load; + else if (reload == load && g_pin_valid && g_pin_tick >= load) + reload = g_pin_tick; + } else if (g_pin_valid && g_pin_tick >= load) { reload = g_pin_tick; + } g_pending_load_tick = reload; g_pending_load_valid = 1; g_tip_extend_rereplay = 1; @@ -1586,8 +2031,8 @@ static void schedule_episode_rereplay(uint32_t mismatch_tick) clear_post_handshake(); rnet_rb_set_phase(g_rb, nRNetRbPhaseReplay); fprintf(stderr, - "psxrecomp: rb tip-extend rereplay load=%u mismatch=%u target=%u\n", - (unsigned)reload, (unsigned)mismatch_tick, + "psxrecomp: rb tip-extend rereplay load=%u prefer=%u target=%u\n", + (unsigned)reload, (unsigned)prefer, (unsigned)rnet_rb_get_target_tick(g_rb)); fflush(stderr); } @@ -1616,6 +2061,7 @@ static void commit_episode(void) fflush(stderr); rnet_rb_session_reset(g_rb); g_tip_extend_rereplay = 0; + clear_tip_extend_prime(); g_needs_advance = 0; g_episode_snap_applied = 0; g_pending_resume_valid = 0; @@ -1660,7 +2106,16 @@ void psx_netplay_rb_start(void) memset(&cfg, 0, sizeof(cfg)); cfg.local_slot = (uint32_t)(g_b.local_slot ? *g_b.local_slot : 0); cfg.delay = (uint32_t)delay; - cfg.tip_runway = RNET_RB_TIP_RUNWAY_DEFAULT; + cfg.tip_runway = RB_MOTK_TIP_RUNWAY; + cfg.tip_seal_slack = RB_MOTK_TIP_SEAL_SLACK_FORCE0; + /* Keep the light-tip depth ceiling matched to tip_runway. A coalesced + * TipHold episode's eventual depth (load..target after tip-extend) can + * approach tip_runway ticks — with the library default of 16 that + * silently dropped ~19% of episodes out of the light-tip fast path in + * soak testing purely because they coalesced past 16, paying a second + * ready-ACK round trip they didn't need (see ROLLBACK_MOTK_HOOKUP.md, + * "light-tip depth vs TipHold coalesce runway", 2026-08-01). */ + cfg.light_tip_max_depth = RB_MOTK_TIP_RUNWAY; { int slots = g_b.slot_count ? *g_b.slot_count : 2; if (slots < 2) slots = 2; @@ -1699,7 +2154,9 @@ void psx_netplay_rb_start(void) g_episode_baseline_matched = 0; g_episode_load_tick = 0; g_tip_hold_until = 0; + g_tip_hold_quiet_t0_ms = 0ull; g_tip_extend_rereplay = 0; + g_tip_extend_prime_tick = 0; clear_baseline_pin(); clear_episode_wire_state(); fprintf(stderr, "psxrecomp: rb snap ring ready depth=%u snap_interval=%u\n", @@ -1735,7 +2192,15 @@ void psx_netplay_rb_shutdown(void) g_fmv_dense_through = 0; g_bl_mismatch_streak = 0; g_tip_hold_until = 0; + g_tip_hold_quiet_t0_ms = 0ull; g_tip_extend_rereplay = 0; + g_tip_extend_prime_tick = 0; + g_last_good_bb_pc = 0; + g_agreed_valid = 0; + g_agreed_through = 0; + g_agreed_span_lo = 0; + g_epoch = 1; + g_rb_rtt_ema_ms = 0; } /* Dense tip snaps: media/lockstep, plus a short window after invent unlock @@ -1861,10 +2326,13 @@ static int try_apply_pending_load(CPUState *cpu_in) note_good_bb_pc(pc); psx_cycles_resync_after_restore(cpu_in); interrupts_resync_after_restore(); - /* Do NOT cdrom_accelerate on RB restore — host-timed CD IRQ catch-up - * forks peers. Normalize SPU CD FIFO so host audio drain asymmetry - * left outside the digest cannot steer XA overflow differently. */ - spu_cd_audio_reset(); + /* Re-arm absolute CD deadlines from restored relative delays. + * Do NOT cdrom_accelerate (delay-cap boost forks peers). */ + cdrom_resync_deadlines_after_restore(); + /* Wipe SPU CD FIFO only when XA is idle — clearing mid-stream + * left MotK FMV silent/black after tip loads into media. */ + if (!cdrom_xa_stream_active() && !cdrom_fmv_stream_pending()) + spu_cd_audio_reset(); { extern void overlay_loader_clear_lazy_miss(void); overlay_loader_clear_lazy_miss(); @@ -1883,7 +2351,6 @@ static int try_apply_pending_load(CPUState *cpu_in) return 1; } if (g_tip_extend_rereplay) { - g_episode_snap_applied = 1; g_tip_extend_rereplay = 0; fprintf(stderr, "psxrecomp: rb tip-extend snap applied tick=%u pc=0x%08x " @@ -1891,6 +2358,10 @@ static int try_apply_pending_load(CPUState *cpu_in) (unsigned)loaded_tick, (unsigned)pc, (unsigned)g_episode_load_tick); fflush(stderr); + /* poll_snap can apply while already in Replay — must arm here + * (hc_prime + sim=reload+1). try_admit's arm path is skipped + * once g_pending_load_valid is cleared. */ + arm_rereplay_after_load(loaded_tick); return 1; } g_episode_snap_applied = 1; @@ -1934,6 +2405,8 @@ void psx_netplay_rb_flush_resume(void) } g_pending_resume_valid = 0; g_replay_progress_ms = rb_mono_ms(); + pc = rb_canonicalize_resume_pc(pc); + note_good_bb_pc(pc); /* MotK wait sticky = CDA0 for both peers */ { CPUState *c = cpu(); CDROMDebugState cds; @@ -1941,10 +2414,12 @@ void psx_netplay_rb_flush_resume(void) cdrom_debug_snapshot(&cds); fprintf(stderr, "psxrecomp: rb flush_resume pc=0x%08x depth24=%d mdec_recent=%d " - "media=%d cd_reading=%d read_delay=%d i_stat=%08x sr=%08x\n", + "media=%d cd_reading=%d read_delay=%d xa=%d fmv_pending=%d " + "i_stat=%08x sr=%08x\n", (unsigned)pc, gpu_display_is_depth24(), mdec_recently_active(RB_FMV_MDEC_HYSTERESIS), rb_fmv_media_active(), - cds.reading, cds.read_delay, (unsigned)cds.i_stat, + cds.reading, cds.read_delay, cdrom_xa_stream_active(), + cdrom_fmv_stream_pending(), (unsigned)cds.i_stat, c ? (unsigned)c->cop0[12] : 0u); fflush(stderr); /* flush_resume is called from admit inside sdl_vblank_present, which @@ -2024,14 +2499,13 @@ void psx_netplay_rb_poll(struct CPUState *cpu_in, uint32_t resume_pc) } } - /* Apply only when flush_resume (called right after poll_snap / admit) can - * longjmp immediately. Episode baseline loads wait for try_admit. */ - if (g_pending_load_valid) { - int can_flush_now = g_live_realign_pending || - (g_rb && rnet_rb_get_phase(g_rb) == nRNetRbPhaseReplay); - if (can_flush_now) - (void)try_apply_pending_load(cpu_in); - } + /* Apply from BB/present poll ONLY for live realign (flush_resume runs in + * the same present epilogue). Episode / tip-extend loads wait for + * try_admit — applying mid-Replay poll mutated CPU under a live CPS and + * forked MotK wait-loop resim (IRQ at CD54 vs CDA0 depending on which + * BB edge the poll hit). */ + if (g_pending_load_valid && g_live_realign_pending) + (void)try_apply_pending_load(cpu_in); } int psx_netplay_rb_rewind_suppressed(void) @@ -2115,14 +2589,37 @@ int psx_netplay_rb_tip_extend(uint32_t mismatch_tick, int slot) new_target = mismatch_tick; if (sim > new_target) new_target = sim; - need_rereplay = (phase_at_entry == nRNetRbPhaseTipHold || - phase_at_entry == nRNetRbPhaseVerify || sim > mismatch_tick) - ? 1 - : 0; - /* Any TipHold/Verify leave invalidates the prior POST pair — stale peer - * digests caused follower abort (local=new POST vs peer=pre-extend POST). */ - if (phase_at_entry == nRNetRbPhaseTipHold || phase_at_entry == nRNetRbPhaseVerify || - need_rereplay) + /* Rereplay only after the baseline/ready handshake (or TipHold/Verify). + * Seal/AwaitingBaseline: resign+extend only — forcing Replay here made the + * initiator solo-resim while the follower waited for GO (ready timeout). + * TipHold: rereplay if Live invented past the prior tip *or* the mismatch + * is past the sealed tip (peek-ahead release coalesce while Live is + * stalled at invent-cap — sim may still equal old_target). */ + if (phase_at_entry == nRNetRbPhaseTipHold) + need_rereplay = (sim > old_target || mismatch_tick > old_target) ? 1 : 0; + else if (phase_at_entry == nRNetRbPhaseVerify) + need_rereplay = 1; + else if (phase_at_entry == nRNetRbPhaseReplay && sim > mismatch_tick) + need_rereplay = 1; + else + need_rereplay = 0; + /* Preflight span cap before mutating POST/seals — peer-seal bitmask is + * uint64 (RNET_RB_PEER_SEAL_MASK_BITS). TipHold storm past load+63 must + * commit and reopen, not sit in begin_refused until runway expires. */ + if (new_target > old_target && !rnet_rb_can_extend_target(g_rb, new_target)) { + if (phase_at_entry == nRNetRbPhaseTipHold) { + fprintf(stderr, + "psxrecomp: rb tip-extend SPAN CAP epoch=%u %u→%u " + "(seal span would exceed %u) — tip-hold commit\n", + (unsigned)rnet_rb_get_epoch_id(g_rb), (unsigned)old_target, + (unsigned)new_target, (unsigned)RNET_RB_PEER_SEAL_MASK_BITS); + fflush(stderr); + finalize_tip_hold(); + } + return 0; + } + /* TipHold/Verify leave invalidates the prior POST pair. */ + if (phase_at_entry == nRNetRbPhaseTipHold || phase_at_entry == nRNetRbPhaseVerify) clear_post_handshake(); (void)rnet_rb_resign_slot_range(g_rb, slot, mismatch_tick, mismatch_tick); export_local_seals(); @@ -2138,28 +2635,44 @@ int psx_netplay_rb_tip_extend(uint32_t mismatch_tick, int slot) export_local_seals(); fprintf(stderr, "psxrecomp: rb tip-extend epoch=%u %u→%u mismatch=%u slot=%d " - "(light=%u)\n", + "(light=%u rereplay=%d)\n", (unsigned)rnet_rb_get_epoch_id(g_rb), (unsigned)old_target, (unsigned)new_target, (unsigned)mismatch_tick, slot, - (unsigned)rnet_rb_recommend_light_tip(g_rb)); + (unsigned)rnet_rb_recommend_light_tip(g_rb), need_rereplay); fflush(stderr); } else if (need_rereplay) { (void)rnet_rb_extend_target(g_rb, old_target); + } else if (phase_at_entry == nRNetRbPhaseSealInputs || + phase_at_entry == nRNetRbPhaseAwaitingBaseline) { + fprintf(stderr, + "psxrecomp: rb tip-extend defer-rereplay epoch=%u mismatch=%u " + "target=%u (handshake phase=%d)\n", + (unsigned)rnet_rb_get_epoch_id(g_rb), (unsigned)mismatch_tick, + (unsigned)new_target, (int)phase_at_entry); + fflush(stderr); } if (need_rereplay) { - schedule_episode_rereplay(mismatch_tick); + /* Same prefer tip as FOLLOW (old_target), not mismatch-1. */ + schedule_episode_rereplay(old_target + 1u); { uint32_t runway = rnet_rb_get_tip_runway(g_rb); uint32_t cap = new_target + runway; if (cap > g_tip_hold_until) g_tip_hold_until = cap; } + g_tip_hold_quiet_t0_ms = 0ull; /* restart coalesce window */ fprintf(stderr, - "psxrecomp: rb tip-extend rereplay epoch=%u mismatch=%u slot=%d " - "until=%u\n", - (unsigned)rnet_rb_get_epoch_id(g_rb), (unsigned)mismatch_tick, slot, - (unsigned)g_tip_hold_until); + "psxrecomp: rb tip-extend rereplay epoch=%u mismatch=%u " + "prior_tip=%u slot=%d until=%u\n", + (unsigned)rnet_rb_get_epoch_id(g_rb), (unsigned)mismatch_tick, + (unsigned)old_target, slot, (unsigned)g_tip_hold_until); fflush(stderr); + } else if (phase_at_entry == nRNetRbPhaseTipHold) { + uint32_t runway = rnet_rb_get_tip_runway(g_rb); + uint32_t cap = new_target + runway; + if (cap > g_tip_hold_until) + g_tip_hold_until = cap; + g_tip_hold_quiet_t0_ms = 0ull; /* restart coalesce window */ } return 1; } @@ -2179,9 +2692,20 @@ int psx_netplay_rb_begin_rewind(uint32_t mismatch_tick, int slot) if (!g_rb || !s) return 0; - /* Prefer tip-extend over refusing while an episode is already open. */ - if (rnet_rb_is_active(g_rb)) - return psx_netplay_rb_tip_extend(mismatch_tick, slot); + /* Prefer tip-extend over refusing while an episode is already open. + * Span-cap TipHold commit leaves the session inactive — fall through to + * open a fresh episode for this mismatch (symmetric with peer). */ + if (rnet_rb_is_active(g_rb)) { + if (psx_netplay_rb_tip_extend(mismatch_tick, slot)) + return 1; + if (rnet_rb_is_active(g_rb)) + return 0; + fprintf(stderr, + "psxrecomp: rb tip-hold yielded — begin fresh episode " + "mismatch=%u slot=%d\n", + (unsigned)mismatch_tick, slot); + fflush(stderr); + } sim = rnet_session_sim_tick(s); @@ -2258,6 +2782,18 @@ int psx_netplay_rb_begin_rewind(uint32_t mismatch_tick, int slot) target = rnet_rb_suggest_target(g_rb, mismatch_tick, sim); if (target < load) target = load; + /* Chunk deep catch-up (post-abort agreed tip << live) so Replay never + * walks the whole cooldown window in one episode. */ + if (target > load + RB_MAX_RESIM_SPAN) { + fprintf(stderr, + "psxrecomp: rb begin SPAN CAP mismatch=%u load=%u " + "target %u→%u (max_span=%u)\n", + (unsigned)mismatch_tick, (unsigned)load, (unsigned)target, + (unsigned)(load + RB_MAX_RESIM_SPAN), + (unsigned)RB_MAX_RESIM_SPAN); + fflush(stderr); + target = load + RB_MAX_RESIM_SPAN; + } memset(&corr, 0, sizeof(corr)); corr.epoch_id = g_epoch++; @@ -2267,9 +2803,13 @@ int psx_netplay_rb_begin_rewind(uint32_t mismatch_tick, int slot) corr.slot = slot; corr.initiator = 1u; corr.from_peer_notify = 0u; - /* Prefer MotK agreed tip when library watermark not yet set. */ + /* Prefer MotK agreed tip when library watermark not yet set. Depth + * ceiling comes from the session's configured light_tip_max_depth + * (== RB_MOTK_TIP_RUNWAY), not the library default — see + * psx_netplay_rb_start(). */ if (g_agreed_valid && - rnet_rb_is_light_tip_candidate(load, target, g_agreed_through)) + rnet_rb_is_light_tip_candidate_ex(load, target, g_agreed_through, + rnet_rb_get_light_tip_max_depth(g_rb))) corr.flags = RNET_RB_CORR_LIGHT_TIP; g_last_begin_mismatch = mismatch_tick; @@ -2315,10 +2855,29 @@ static void begin_follower(uint32_t epoch, uint32_t mismatch, uint32_t load, uin load == rnet_rb_get_load_tick(g_rb) && target > rnet_rb_get_target_tick(g_rb)) { uint32_t old_t = rnet_rb_get_target_tick(g_rb); RNetRbPhase phase_at_entry = rnet_rb_get_phase(g_rb); - int need_rereplay = (phase_at_entry == nRNetRbPhaseTipHold || - phase_at_entry == nRNetRbPhaseVerify) - ? 1 - : 0; + RNetSession *fs = sess(); + uint32_t sim_now = fs ? rnet_session_sim_tick(fs) : 0u; + /* TipHold: rereplay only if Live invented past prior tip (mirror + * initiator). Verify always rereplays. */ + int need_rereplay = 0; + if (phase_at_entry == nRNetRbPhaseTipHold) + need_rereplay = (sim_now > old_t) ? 1 : 0; + else if (phase_at_entry == nRNetRbPhaseVerify) + need_rereplay = 1; + if (!rnet_rb_can_extend_target(g_rb, target)) { + /* Initiator should have tip-hold-committed on span cap; if a stale + * tip-extend SYNC still arrives, close TipHold so a later SYNC can + * open a fresh episode. */ + if (phase_at_entry == nRNetRbPhaseTipHold) { + fprintf(stderr, + "psxrecomp: rb tip-extend FOLLOW SPAN CAP epoch=%u " + "%u→%u — tip-hold commit\n", + (unsigned)epoch, (unsigned)old_t, (unsigned)target); + fflush(stderr); + finalize_tip_hold(); + } + return; + } clear_post_handshake(); if (rnet_rb_extend_target(g_rb, target)) { (void)rnet_rb_resign_slot_range(g_rb, slot, old_t + 1u, target); @@ -2333,16 +2892,28 @@ static void begin_follower(uint32_t epoch, uint32_t mismatch, uint32_t load, uin if (need_rereplay) { /* Reload at prior tip (old_t); mismatch field is episode-original. */ schedule_episode_rereplay(old_t + 1u); - { - uint32_t runway = rnet_rb_get_tip_runway(g_rb); - uint32_t cap = target + runway; - if (cap > g_tip_hold_until) - g_tip_hold_until = cap; - } + } + if (phase_at_entry == nRNetRbPhaseTipHold || need_rereplay) { + uint32_t runway = rnet_rb_get_tip_runway(g_rb); + uint32_t cap = target + runway; + if (cap > g_tip_hold_until) + g_tip_hold_until = cap; + g_tip_hold_quiet_t0_ms = 0ull; /* restart coalesce window */ } } return; } + /* New epoch while TipHold: initiator span-cap committed and reopened — + * yield tip-hold so we can follow (was silently dropping SYNC). */ + if (g_rb && rnet_rb_is_tip_holding(g_rb) && + epoch != rnet_rb_get_epoch_id(g_rb)) { + fprintf(stderr, + "psxrecomp: rb tip-hold yield FOLLOW new epoch=%u " + "(was epoch=%u) — tip-hold commit\n", + (unsigned)epoch, (unsigned)rnet_rb_get_epoch_id(g_rb)); + fflush(stderr); + finalize_tip_hold(); + } if (!g_rb || rnet_rb_is_active(g_rb)) return; (void)rb_in_fmv_defer_rewind_window(); @@ -2387,22 +2958,32 @@ static void begin_follower(uint32_t epoch, uint32_t mismatch, uint32_t load, uin * hash_confirm is a doomed baseline (cpu-only tip snap fork). */ if (g_b.hc) (void)netplay_hc_heal_stale_gap(g_b.hc); + heal_agreed_watermark_if_aged_out(); + /* The initiator advances its agreed along hc-confirmed live play; do the + * same here or its (valid) frontier loads get REFUSED as "unconfirmed". */ + advance_agreed_watermark_from_hc(); { uint32_t frontier = 0u; int have_f = 0; + uint32_t oldest = g_snaps ? netplay_snap_ring_oldest_tick(g_snaps) : 0u; if (g_agreed_valid) { frontier = g_agreed_through; have_f = 1; - } - if (g_b.hc) { + } else if (g_b.hc) { uint32_t rt = netplay_hc_resolved_through(g_b.hc); - if (netplay_hc_confirm_through(g_b.hc, rt) && rt >= frontier) { + if (netplay_hc_confirm_through(g_b.hc, rt)) { frontier = rt; have_f = 1; } } - if (have_f && load > frontier && - !(g_b.hc && netplay_hc_confirm_through(g_b.hc, load))) { + /* hc/agreed below snap ring: do not NACK tip-slack loads (BOOTSTRAP + * should have raised agreed; mirror choose_load fall-through). */ + if (have_f && g_snaps && frontier < oldest) + have_f = 0; + /* Mirror choose_load hard cap: once tip-hold/commit set agreed_through, + * refuse loads above it even if local hash_confirm matched TipHold Live + * (heal raises agreed when that tip aged out of the ring). */ + if (have_f && load > frontier) { fprintf(stderr, "psxrecomp: rb follow REFUSED epoch=%u load=%u — past " "frontier=%u (unconfirmed tip)\n", @@ -2429,7 +3010,8 @@ static void begin_follower(uint32_t epoch, uint32_t mismatch, uint32_t load, uin corr.initiator = 0u; corr.from_peer_notify = 1u; if (g_agreed_valid && - rnet_rb_is_light_tip_candidate(load, target, g_agreed_through)) + rnet_rb_is_light_tip_candidate_ex(load, target, g_agreed_through, + rnet_rb_get_light_tip_max_depth(g_rb))) corr.flags = RNET_RB_CORR_LIGHT_TIP; if (epoch >= g_epoch) g_epoch = epoch + 1u; @@ -2474,8 +3056,67 @@ void psx_netplay_rb_pump(void) if (rnet_rb_is_active(g_rb) && epoch == rnet_rb_get_epoch_id(g_rb) && !rnet_rb_is_from_peer_notify(g_rb)) { char why[96]; + RNetSession *ns = sess(); + uint32_t live_sim = ns ? rnet_session_sim_tick(ns) : 0u; + uint32_t demote = (load > 0u) ? load - 1u : 0u; + /* Capture before abort_episode clears it — decides whether Live + * must rewind (snap already applied) or can stay put. */ + int snap_was_applied = g_episode_snap_applied; snprintf(why, sizeof(why), "peer follow NACK load=%u", (unsigned)load); + /* Peer refused load as past its frontier — usually because we + * tip-held / advanced agreed_through without their POST ack + * (unilateral tip). Plain abort_episode left agreed_through + * at the refused tip, so the next begin reopened load=same + * tip → NACK → abort storm (epochs 7..11 in 2026-08-01 soak). + * + * 2026-08-01 cliff soak: always realigning to demote was wrong + * when the refused load snap was never applied (still in + * SealInputs) — that rewound ~25 ticks of good matched Live + * on only this peer, forked cores at demote+1, and armed + * cooldown so late wire became promote-no-resim poison. Only + * realign when we actually left the pre-episode tip. Always + * demote watermarks + prime HC so live hash_confirm cannot + * immediately re-ADVANCE past the refused tip (soak: NACK + * demote 3840 then `agreed ADVANCE 3840→3856` on the next + * pump from stale HC). */ + if (g_snaps && demote > 0u) { + uint32_t t; + for (t = demote;; --t) { + if (netplay_snap_ring_has(g_snaps, t)) { + demote = t; + break; + } + if (t == 0u) + break; + } + } + if (g_agreed_valid && g_agreed_through >= load) { + g_agreed_through = demote; + g_agreed_span_lo = demote; + } + rnet_rb_demote_resolved_through(g_rb, demote); + if (g_b.hc) + netplay_hc_prime_after(g_b.hc, demote); + if (g_pin_valid && g_pin_tick >= load) + clear_baseline_pin(); + g_episode_baseline_matched = 0; abort_episode(why); + if (snap_was_applied && g_snaps && + netplay_snap_ring_has(g_snaps, demote)) { + fprintf(stderr, + "psxrecomp: rb NACK realign (snap was applied) " + "demote=%u live_was=%u\n", + (unsigned)demote, (unsigned)live_sim); + fflush(stderr); + schedule_live_realign(demote, why); + } else { + fprintf(stderr, + "psxrecomp: rb NACK keep-live demote=%u " + "live=%u (snap never applied — no rewind)\n", + (unsigned)demote, (unsigned)live_sim); + fflush(stderr); + } + arm_rewind_cooldown_ticks(live_sim, RB_ABORT_COOLDOWN_TICKS, why); } continue; } @@ -2500,6 +3141,7 @@ void psx_netplay_rb_pump(void) } while (rnet_session_take_rb_post(s, &epoch, &target, &dig_m, &in_dig, &match)) { + uint32_t tip; if (!rnet_rb_is_active(g_rb) || epoch != rnet_rb_get_epoch_id(g_rb)) continue; /* TipHold cleared the POST handshake — ignore late/rexmit POSTs from @@ -2507,13 +3149,47 @@ void psx_netplay_rb_pump(void) * digest. Replay/Seal/Baseline also must not latch peer_ok early. */ if (rnet_rb_get_phase(g_rb) != nRNetRbPhaseVerify) continue; + tip = rnet_rb_get_target_tick(g_rb); + /* Tip-extend race: peer POST@old tip arrives during Verify@new tip. + * Wire carries target_tick — drop mismatches (was false post diverge). */ + if (!netplay_rb_peer_post_tip_ok(target, tip)) { + static uint32_t s_stale_log_tip; + if (s_stale_log_tip != tip) { + fprintf(stderr, + "psxrecomp: rb post ignore stale tip=%u (episode tip=%u " + "dig_m=%08x)\n", + (unsigned)target, (unsigned)tip, (unsigned)dig_m); + fflush(stderr); + s_stale_log_tip = tip; + } + continue; + } g_peer_post_ok = 1; g_peer_post_digest = dig_m; g_peer_post_av = in_dig; g_peer_post_match = match; if (g_local_post_sent) { + /* RTT sample: elapsed time since *we* sent our POST, now that the + * peer's POST for this same episode has arrived. Both sides enter + * Verify around the same real-world moment (triggered by the same + * mismatch/rewind-request), so this is dominated by one-way + * transit of the peer's packet, not local compute — a genuine + * link-latency measurement. Sanity-bounded: a stale/backdated + * g_verify_wait_ms (e.g. tip-hold reused an old episode's timer) + * would otherwise poison the EMA with a bogus multi-second value. */ + if (g_verify_wait_ms != 0ull) { + uint64_t now = rb_mono_ms(); + if (now >= g_verify_wait_ms) { + uint64_t sample = now - g_verify_wait_ms; + if (sample <= 2000ull) { + g_rb_rtt_ema_ms = g_rb_rtt_ema_ms + ? (uint32_t)((3ull * g_rb_rtt_ema_ms + sample) / 4ull) + : (uint32_t)sample; + } + } + } if (dig_m == g_post_digest && in_dig == g_post_av) - enter_tip_hold(rnet_rb_get_target_tick(g_rb)); + enter_tip_hold(tip); else { char why[128]; if (dig_m != g_post_digest) @@ -2532,8 +3208,25 @@ void psx_netplay_rb_pump(void) { rnet_u32 resolved = 0; - while (rnet_session_take_rb_resolved(s, &resolved)) + while (rnet_session_take_rb_resolved(s, &resolved)) { rnet_rb_set_peer_convergence(g_rb, resolved); + /* Peer already POST-matched and tip-held this tip (or later). If + * we are still stuck in Verify waiting for their POST, trust + * RESOLVED as the missing half of the handshake — otherwise a + * single lost POST leaves us here until verify timeout while + * they advance Live on the same tip (2026-08-01 soak). */ + if (rnet_rb_is_active(g_rb) && + rnet_rb_get_phase(g_rb) == nRNetRbPhaseVerify && + g_local_post_sent && g_post_target > 0u && + resolved >= g_post_target) { + fprintf(stderr, + "psxrecomp: rb verify accept peer RESOLVED tip=%u " + "(local_post=%u — POST may have been lost)\n", + (unsigned)resolved, (unsigned)g_post_target); + fflush(stderr); + enter_tip_hold(g_post_target); + } + } } /* Retransmit follow-NACK while initiator may still be sealing. */ @@ -2616,6 +3309,47 @@ int psx_netplay_rb_tip_holding(void) return g_rb && rnet_rb_is_tip_holding(g_rb); } +uint32_t psx_netplay_rb_episode_target(void) +{ + if (!g_rb || !rnet_rb_is_active(g_rb)) + return 0; + return rnet_rb_get_target_tick(g_rb); +} + +uint32_t psx_netplay_rb_tip_hold_invent_slack(void) +{ + if (!g_rb) + return 0; + return rnet_rb_get_tip_seal_slack(g_rb); +} + +uint32_t psx_netplay_rb_tip_runway(void) +{ + if (!g_rb) + return 0; + return rnet_rb_get_tip_runway(g_rb); +} + +int psx_netplay_rb_ignore_peer_frame_commit(uint32_t tick, uint32_t hash) +{ + uint32_t ld = 0; + if (!g_tip_extend_prime_tick || !g_b.hc) + return 0; + if (!g_rb || !rnet_rb_is_active(g_rb)) + return 0; + if (rnet_rb_get_phase(g_rb) != nRNetRbPhaseReplay && + rnet_rb_get_phase(g_rb) != nRNetRbPhaseVerify) + return 0; + if (tick <= g_tip_extend_prime_tick) + return 0; + /* Ahead of our sealed resim, or TipHold invent that disagrees with it. */ + if (!netplay_hc_local_digest(g_b.hc, tick, &ld)) + return 1; + if (ld != hash) + return 1; + return 0; +} + int psx_netplay_rb_abort_resim_core_mismatch(uint32_t tick, uint32_t local_core, uint32_t peer_core) { @@ -2627,6 +3361,10 @@ int psx_netplay_rb_abort_resim_core_mismatch(uint32_t tick, uint32_t local_core, if (rnet_rb_get_phase(g_rb) != nRNetRbPhaseReplay && rnet_rb_get_phase(g_rb) != nRNetRbPhaseVerify) return 0; + /* TipHold invent still in hc after tip-extend — drain should have dropped + * it; never abort the sealed rereplay on that digest. */ + if (psx_netplay_rb_ignore_peer_frame_commit(tick, peer_core)) + return 0; /* Ignore stale live commits below the episode load (hc_prime_after should * already have cleared them; belt-and-suspenders). */ load = rnet_rb_get_load_tick(g_rb); @@ -2722,8 +3460,9 @@ int psx_netplay_rb_try_admit(void) if (rnet_rb_get_phase(g_rb) == nRNetRbPhaseReplay && g_pending_load_valid && !g_episode_snap_applied) { CPUState *c = cpu(); - if (c && try_apply_pending_load(c)) - arm_rereplay_after_load(g_pending_load_tick); + /* tip-extend apply arms inside try_apply_pending_load. */ + if (c) + (void)try_apply_pending_load(c); return g_needs_advance ? 1 : 0; } @@ -2799,6 +3538,20 @@ uint32_t psx_netplay_rb_episode_count(void) return g_episode_count; } +uint64_t psx_netplay_rb_take_replay_ticks(void) +{ + uint64_t n = g_stat_replay_ticks; + g_stat_replay_ticks = 0; + return n; +} + +/* Live POST-handshake latency EMA (ms), 0 = no episode has round-tripped + * yet this session. See g_rb_rtt_ema_ms above. */ +uint32_t psx_netplay_rb_rtt_estimate_ms(void) +{ + return g_rb_rtt_ema_ms; +} + int psx_netplay_rb_phase(void) { return g_rb ? (int)rnet_rb_get_phase(g_rb) : 0; diff --git a/runtime/src/psx_selfcheck.c b/runtime/src/psx_selfcheck.c index cb06ae98..c6203333 100644 --- a/runtime/src/psx_selfcheck.c +++ b/runtime/src/psx_selfcheck.c @@ -786,7 +786,9 @@ static void sc_do_load(struct CPUState *cpu, const char *via) } psx_cycles_resync_after_restore(cpu); interrupts_resync_after_restore(); - /* Keep the snap's VBlank phase (resync zeroed it for UI savestate safety). */ + /* Keep the snap's VBlank phase (boot_state also restores csv from IRQ + * section; re-apply the latched value so ambient-prime / legacy 8B snaps + * stay bit-identical across replay#1/#2). */ interrupts_set_cycles_since_vblank(s_snap_csv); /* Pin host frame counter so present/GPU hysteresis that keys on * s_frame_count cannot fork resim#1 vs resim#2 (+span otherwise). */ diff --git a/runtime/tests/test_rb_post_tip_filter.c b/runtime/tests/test_rb_post_tip_filter.c new file mode 100644 index 00000000..0afa3d04 --- /dev/null +++ b/runtime/tests/test_rb_post_tip_filter.c @@ -0,0 +1,45 @@ +/* + * test_rb_post_tip_filter.c — tip-extend stale POST must not latch. + * + * Soak class (rb-diag 2026-07-31): both peers POST@1007 (matched), tip-extend + * to 1008, both POST@1008 (matched); initiator aborted local@1008 vs peer@1007. + * + * gcc -std=c11 -Wall -Wextra -Werror -I../include \ + * -o test_rb_post_tip_filter test_rb_post_tip_filter.c && \ + * ./test_rb_post_tip_filter + */ +#include "netplay_rb_post.h" + +#include + +static int failures; +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + printf("FAIL: %s\n", msg); \ + failures++; \ + } else { \ + printf("ok: %s\n", msg); \ + } \ + } while (0) + +int main(void) +{ + const uint32_t tip_old = 1007u; + const uint32_t tip_new = 1008u; + + CHECK(netplay_rb_peer_post_tip_ok(tip_old, tip_old), "POST@T ok while tip=T"); + CHECK(netplay_rb_peer_post_tip_ok(tip_new, tip_new), "POST@T+1 ok while tip=T+1"); + CHECK(!netplay_rb_peer_post_tip_ok(tip_old, tip_new), + "stale POST@T rejected during Verify tip=T+1"); + CHECK(!netplay_rb_peer_post_tip_ok(tip_new, tip_old), + "future POST@T+1 rejected while tip=T"); + CHECK(!netplay_rb_peer_post_tip_ok(0u, tip_new), "zero tip rejected"); + + if (failures == 0) { + printf("test_rb_post_tip_filter: ok\n"); + return 0; + } + printf("test_rb_post_tip_filter: %d failure(s)\n", failures); + return 1; +} diff --git a/tools/bundle_mingw_dlls.sh b/tools/bundle_mingw_dlls.sh new file mode 100755 index 00000000..276744bc --- /dev/null +++ b/tools/bundle_mingw_dlls.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# Bundle imported non-system DLLs next to Windows PE executables. +# +# Title packagers (BPE/MotK setup zips) call this for Windows PEs that still +# import non-system DLLs (typically the MSYS2-built setup host + SDL2). +# Emitters built with llvm-mingw + PSXRECOMP_STATIC_CLI should import none of +# the GCC runtimes; this script only copies DLLs each exe actually imports. +# +# Usage: +# bundle_mingw_dlls.sh [options] --exe [--dest ] [--label ] ... +# +# Options: +# --runtime-bin DIR Preferred DLL search directory (repeatable) +# --search-dir DIR Extra DLL search directory (repeatable) +# --exe PATH PE to inspect (repeatable; --dest/--label apply to it) +# --dest DIR Copy DLLs here (default: dirname of the preceding --exe) +# --label NAME Log label for the preceding --exe +# --require DLL After bundling, DLL must exist in every --dest that +# imported it (repeatable) +# --soft-missing Warn instead of exit 1 when objdump is unavailable +# +# Exit 1 if an imported DLL cannot be found, or a --require check fails. +set -euo pipefail + +SOFT_MISSING=0 +RUNTIME_BINS=() +SEARCH_DIRS=() +REQUIRE=() + +# Parallel arrays for targets. +EXE_PATHS=() +DEST_DIRS=() +LABELS=() + +pending_dest="" +pending_label="" + +usage() { + sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//' + exit 2 +} + +flush_pending_meta() { + # Apply trailing --dest/--label to the last --exe. + local n="${#EXE_PATHS[@]}" + if [[ "$n" -eq 0 ]]; then + return 0 + fi + local i=$((n - 1)) + if [[ -n "${pending_dest}" ]]; then + DEST_DIRS[$i]="${pending_dest}" + pending_dest="" + fi + if [[ -n "${pending_label}" ]]; then + LABELS[$i]="${pending_label}" + pending_label="" + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage ;; + --soft-missing) SOFT_MISSING=1; shift ;; + --runtime-bin) + RUNTIME_BINS+=("${2:?}"); shift 2 ;; + --search-dir) + SEARCH_DIRS+=("${2:?}"); shift 2 ;; + --require) + REQUIRE+=("${2:?}"); shift 2 ;; + --exe) + flush_pending_meta + EXE_PATHS+=("${2:?}") + DEST_DIRS+=("") + LABELS+=("") + shift 2 + ;; + --dest) + pending_dest="${2:?}"; shift 2 ;; + --label) + pending_label="${2:?}"; shift 2 ;; + *) + echo "error: unknown arg: $1" >&2 + usage + ;; + esac +done +flush_pending_meta + +if [[ ${#EXE_PATHS[@]} -eq 0 ]]; then + echo "error: at least one --exe is required" >&2 + usage +fi + +OBJDUMP="" +if command -v x86_64-w64-mingw32-objdump >/dev/null 2>&1; then + OBJDUMP="x86_64-w64-mingw32-objdump" +elif command -v objdump >/dev/null 2>&1; then + OBJDUMP="objdump" +else + if [[ "${SOFT_MISSING}" -eq 1 ]]; then + echo "warning: no objdump; skipping MinGW DLL bundling" >&2 + exit 0 + fi + echo "error: no objdump; cannot bundle MinGW DLLs" >&2 + exit 1 +fi + +SYSTEM_DLL_RE='^(KERNEL32|USER32|GDI32|ADVAPI32|SHELL32|OLE32|OLEAUT32|WS2_32|WINMM|IMM32|SETUPAPI|VERSION|OPENGL32|COMCTL32|COMDLG32|RPCRT4|SHLWAPI|CRYPT32|BCRYPT|IPHLPAPI|NSI|DNSAPI|MSVCRT|UCRTBASE|VCRUNTIME|API-MS-).*\.DLL$' + +PROBE_DLLS=( + SDL2.dll + zlib1.dll + libgcc_s_seh-1.dll + libstdc++-6.dll + libc++.dll + libunwind.dll + libwinpthread-1.dll + libssp-0.dll +) + +exe_imports_dll() { + local exe="$1" + local dll="$2" + "${OBJDUMP}" -p "${exe}" 2>/dev/null | grep -qi "DLL Name:[[:space:]]*${dll}" +} + +bundle_one() { + local exe="$1" + local dest_dir="$2" + local label="$3" + local dll src key cand d + local -a needed=() + local -a unique=() + local -a candidates=() + local -A seen=() + + if [[ ! -f "${exe}" ]]; then + echo "error: cannot bundle DLLs; missing ${exe}" >&2 + exit 1 + fi + if [[ -z "${dest_dir}" ]]; then + dest_dir="$(dirname "${exe}")" + fi + if [[ -z "${label}" ]]; then + label="$(basename "${exe}")" + fi + mkdir -p "${dest_dir}" + + mapfile -t needed < <( + "${OBJDUMP}" -p "${exe}" 2>/dev/null \ + | awk '/DLL Name:/{print $3}' \ + | grep -viE "${SYSTEM_DLL_RE}" \ + | sort -u || true + ) + needed+=("${PROBE_DLLS[@]}") + + for dll in "${needed[@]}"; do + [[ -n "${dll}" ]] || continue + key="$(printf '%s' "${dll}" | tr '[:upper:]' '[:lower:]')" + if [[ -n "${seen[$key]:-}" ]]; then + continue + fi + seen[$key]=1 + unique+=("${dll}") + done + + for dll in "${unique[@]}"; do + if ! exe_imports_dll "${exe}" "${dll}"; then + continue + fi + src="" + candidates=( + "$(dirname "${exe}")/${dll}" + "${dest_dir}/${dll}" + "/mingw64/bin/${dll}" + "/usr/x86_64-w64-mingw32/bin/${dll}" + ) + for d in "${SEARCH_DIRS[@]+"${SEARCH_DIRS[@]}"}"; do + candidates+=("${d}/${dll}") + done + for d in "${RUNTIME_BINS[@]+"${RUNTIME_BINS[@]}"}"; do + candidates+=("${d}/${dll}") + done + for cand in "${candidates[@]}"; do + [[ -n "${cand}" ]] || continue + if [[ -f "${cand}" ]]; then + src="${cand}" + break + fi + done + if [[ -z "${src}" ]]; then + echo "error: required DLL missing for ${label}: ${dll}" >&2 + echo " exe: ${exe}" >&2 + echo " dest: ${dest_dir}" >&2 + if [[ ${#SEARCH_DIRS[@]} -gt 0 ]]; then + echo " search-dirs: ${SEARCH_DIRS[*]}" >&2 + fi + if [[ ${#RUNTIME_BINS[@]} -gt 0 ]]; then + echo " runtime-bins: ${RUNTIME_BINS[*]}" >&2 + fi + exit 1 + fi + dest_file="${dest_dir}/${dll}" + # MSYS2/Windows cp fails when source and dest are the same file. + if [[ -f "${dest_file}" ]] && [[ "${src}" -ef "${dest_file}" ]]; then + echo "bundled ${dll} → ${dest_dir}/ (${label}; already present)" + continue + fi + src_res="$(cd "$(dirname "${src}")" && pwd)/$(basename "${src}")" + dest_res="$(cd "${dest_dir}" && pwd)/${dll}" + if [[ "${src_res}" == "${dest_res}" ]]; then + echo "bundled ${dll} → ${dest_dir}/ (${label}; already present)" + continue + fi + cp -f "${src}" "${dest_dir}/" + echo "bundled ${dll} → ${dest_dir}/ (${label})" + done +} + +for i in "${!EXE_PATHS[@]}"; do + bundle_one "${EXE_PATHS[$i]}" "${DEST_DIRS[$i]}" "${LABELS[$i]}" +done + +for dll in "${REQUIRE[@]}"; do + [[ -n "${dll}" ]] || continue + for i in "${!EXE_PATHS[@]}"; do + exe="${EXE_PATHS[$i]}" + dest="${DEST_DIRS[$i]:-}" + if [[ -z "${dest}" ]]; then + dest="$(dirname "${exe}")" + fi + if ! exe_imports_dll "${exe}" "${dll}"; then + continue + fi + if [[ ! -f "${dest}/${dll}" ]]; then + echo "error: $(basename "${exe}") imported ${dll} but it is missing under ${dest}" >&2 + exit 1 + fi + done +done diff --git a/tools/ci/build_emitters.sh b/tools/ci/build_emitters.sh new file mode 100755 index 00000000..40428567 --- /dev/null +++ b/tools/ci/build_emitters.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Configure and build psxrecomp-game + psxrecomp-bios. +# +# On Windows CI, prefer the portable cmake-clang-v1 (llvm-mingw) pack so +# emitters statically link libc++ and do not need MSYS2 GCC DLLs. +# +# Usage (from game repo root): +# psxrecomp/tools/ci/build_emitters.sh \ +# [--framework psxrecomp] [--build-dir build-recompiler] \ +# [--toolchain-dir DIR] [--jobs N] +# +# Env: +# PSXRECOMP_TOOLCHAIN_DIR | TOOLCHAIN_DIR | BPE_TOOLCHAIN_DIR +set -euo pipefail + +FRAMEWORK="psxrecomp" +BUILD_DIR="build-recompiler" +JOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" +TOOLCHAIN_DIR="${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --framework) FRAMEWORK="${2:?}"; shift 2 ;; + --build-dir) BUILD_DIR="${2:?}"; shift 2 ;; + --toolchain-dir) TOOLCHAIN_DIR="${2:?}"; shift 2 ;; + --jobs) JOBS="${2:?}"; shift 2 ;; + -h|--help) + sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "error: unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +if [[ ! -d "${FRAMEWORK}/recompiler" ]]; then + echo "error: ${FRAMEWORK}/recompiler missing" >&2 + exit 1 +fi + +CMAKE_ARGS=( + -S "${FRAMEWORK}/recompiler" + -B "${BUILD_DIR}" + -G Ninja + -DCMAKE_BUILD_TYPE=Release +) + +# Prefer portable llvm-mingw pack when present (Windows release CI). +if [[ -n "${TOOLCHAIN_DIR}" && -d "${TOOLCHAIN_DIR}/bin" ]]; then + TOOLCHAIN_DIR="$(cd "${TOOLCHAIN_DIR}" && pwd)" + export PATH="${TOOLCHAIN_DIR}/bin:${PATH}" + if [[ -f "${TOOLCHAIN_DIR}/env.sh" ]]; then + # shellcheck disable=SC1091 + source "${TOOLCHAIN_DIR}/env.sh" + fi + + pick_tool() { + local name="$1" + if [[ -f "${TOOLCHAIN_DIR}/bin/${name}.exe" ]]; then + echo "${TOOLCHAIN_DIR}/bin/${name}.exe" + elif [[ -f "${TOOLCHAIN_DIR}/bin/${name}" ]]; then + echo "${TOOLCHAIN_DIR}/bin/${name}" + else + return 1 + fi + } + + if CC_BIN="$(pick_tool clang)"; then + CMAKE_ARGS+=("-DCMAKE_C_COMPILER=${CC_BIN}") + fi + if CXX_BIN="$(pick_tool clang++)"; then + CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=${CXX_BIN}") + fi + if CMAKE_BIN="$(pick_tool cmake)"; then + CMAKE="${CMAKE_BIN}" + else + CMAKE="cmake" + fi + # Static CRT/libc++ so staged emitters need no MSYS2 GCC DLLs. + CMAKE_ARGS+=(-DPSXRECOMP_STATIC_CLI=ON) + echo "build_emitters: using portable toolchain at ${TOOLCHAIN_DIR}" +else + CMAKE="cmake" + if [[ -d /mingw64/bin ]]; then + export PATH="/mingw64/bin:${PATH}" + echo "build_emitters: no portable toolchain; using MSYS2 /mingw64" + fi +fi + +"${CMAKE}" "${CMAKE_ARGS[@]}" +"${CMAKE}" --build "${BUILD_DIR}" --target psxrecomp-game psxrecomp-bios -j"${JOBS}" + +for bin in psxrecomp-bios psxrecomp-game; do + if [[ -f "${BUILD_DIR}/${bin}.exe" ]]; then + chmod +x "${BUILD_DIR}/${bin}.exe" + elif [[ -f "${BUILD_DIR}/${bin}" ]]; then + chmod +x "${BUILD_DIR}/${bin}" + else + echo "error: ${bin} missing after recompiler build" >&2 + exit 1 + fi +done + +echo "emitters ready under ${BUILD_DIR}" diff --git a/tools/ci/clear_generated.sh b/tools/ci/clear_generated.sh new file mode 100755 index 00000000..7dc555cd --- /dev/null +++ b/tools/ci/clear_generated.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Clear generated game/BIOS C so CI can build a BIOS-free setup host. +# +# Usage (from game repo root): +# psxrecomp/tools/ci/clear_generated.sh [--root DIR] +set -euo pipefail + +ROOT="${PWD}" +while [[ $# -gt 0 ]]; do + case "$1" in + --root) ROOT="${2:?}"; shift 2 ;; + -h|--help) + sed -n '2,7p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "error: unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +ROOT="$(cd "${ROOT}" && pwd)" +rm -rf "${ROOT}/generated" "${ROOT}/psxrecomp/generated" +echo "cleared generated/ under ${ROOT} (setup-host mode)" diff --git a/tools/ci/normalize_version.sh b/tools/ci/normalize_version.sh new file mode 100755 index 00000000..3274bf26 --- /dev/null +++ b/tools/ci/normalize_version.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Normalize a release version string to X.Y.Z (strip leading v / whitespace). +# +# Usage: +# normalize_version.sh # prints VERSION=… TAG=… +# normalize_version.sh --write VERSION # also writes VERSION file +# +# When GITHUB_OUTPUT is set, appends version= and tag=. +set -euo pipefail + +WRITE="" +RAW="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --write) WRITE="${2:?}"; shift 2 ;; + -h|--help) + sed -n '2,10p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + RAW="$1" + shift + ;; + esac +done + +if [[ -z "${RAW}" ]]; then + echo "usage: $0 [--write VERSION] " >&2 + exit 2 +fi + +RAW="$(printf '%s' "${RAW}" | tr -d '[:space:]')" +VER="${RAW#v}" +if [[ ! "${VER}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.+-][A-Za-z0-9.+-]*)?$ ]]; then + echo "error: invalid version '${RAW}' (want X.Y.Z or vX.Y.Z)" >&2 + exit 1 +fi +TAG="v${VER}" + +echo "VERSION=${VER}" +echo "TAG=${TAG}" + +if [[ -n "${WRITE}" ]]; then + printf '%s\n' "${VER}" >"${WRITE}" +fi +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "version=${VER}" >>"${GITHUB_OUTPUT}" + echo "tag=${TAG}" >>"${GITHUB_OUTPUT}" +fi diff --git a/tools/ci/record_pins.sh b/tools/ci/record_pins.sh new file mode 100755 index 00000000..4e134a64 --- /dev/null +++ b/tools/ci/record_pins.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Print pinned submodule revisions for CI logs. +# +# Usage (from game repo root): +# psxrecomp/tools/ci/record_pins.sh [--root DIR] +set -euo pipefail + +ROOT="${PWD}" +while [[ $# -gt 0 ]]; do + case "$1" in + --root) ROOT="${2:?}"; shift 2 ;; + -h|--help) + sed -n '2,7p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "error: unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +ROOT="$(cd "${ROOT}" && pwd)" + +pin() { + local name="$1" path="$2" + if [[ -d "${ROOT}/${path}/.git" ]] || [[ -f "${ROOT}/${path}/.git" ]]; then + echo "${name}=$(git -C "${ROOT}/${path}" rev-parse --short HEAD) ($(git -C "${ROOT}/${path}" rev-parse HEAD))" + elif [[ -d "${ROOT}/${path}" ]]; then + echo "${name}=" + else + echo "${name}=" + fi +} + +pin psxrecomp psxrecomp +pin recomp-ui recomp-ui +pin recomp-net psxrecomp/lib/recomp-net diff --git a/tools/fetch_toolchain.sh b/tools/fetch_toolchain.sh new file mode 100755 index 00000000..f04375a2 --- /dev/null +++ b/tools/fetch_toolchain.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Download and unpack a retcomm-toolchains cmake-clang-v1 pack. +# +# Usage: +# fetch_toolchain.sh --artifact \ +# [--dl-dir DIR] [--out-dir DIR] [--repo OWNER/NAME] +# +# Prints: TOOLCHAIN_DIR= +# When GITHUB_ENV is set (Actions), also appends: +# TOOLCHAIN_DIR=... +# PSXRECOMP_TOOLCHAIN_DIR=... +# BPE_TOOLCHAIN_DIR=... (compat alias for existing title workflows) +# +# Env: +# GH_TOKEN / GITHUB_TOKEN — optional; used for gh or Authorization on curl +set -euo pipefail + +ARTIFACT="" +DL_DIR=".cache/toolchain-dl" +OUT_DIR=".cache/toolchain-pack" +REPO="TechnicallyComputers/retcomm-toolchains" + +usage() { + sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//' + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage ;; + --artifact) ARTIFACT="${2:?}"; shift 2 ;; + --dl-dir) DL_DIR="${2:?}"; shift 2 ;; + --out-dir) OUT_DIR="${2:?}"; shift 2 ;; + --repo) REPO="${2:?}"; shift 2 ;; + *) + echo "error: unknown arg: $1" >&2 + usage + ;; + esac +done + +if [[ -z "${ARTIFACT}" ]]; then + echo "error: --artifact is required" >&2 + usage +fi + +case "${ARTIFACT}" in + linux-x64) + PATTERN='cmake-clang-v1*linux*.zip' + ASSET_NAME='cmake-clang-v1-linux-x64.zip' + ;; + windows-x64) + PATTERN='cmake-clang-v1*windows*.zip' + ASSET_NAME='cmake-clang-v1-windows-x64.zip' + ;; + macos-arm64|macos-x64) + PATTERN='cmake-clang-v1*macos*.zip' + ASSET_NAME='cmake-clang-v1-macos-universal.zip' + ;; + *) + echo "error: unknown artifact '${ARTIFACT}' (want linux-x64|windows-x64|macos-arm64|macos-x64)" >&2 + exit 1 + ;; +esac + +TOKEN="${GH_TOKEN:-${GITHUB_TOKEN:-}}" + +rm -rf "${DL_DIR}" "${OUT_DIR}" +mkdir -p "${DL_DIR}" "${OUT_DIR}" + +if command -v gh >/dev/null 2>&1; then + if [[ -n "${TOKEN}" ]]; then + export GH_TOKEN="${TOKEN}" + fi + gh release download -R "${REPO}" -p "${PATTERN}" -D "${DL_DIR}" --clobber +elif command -v curl >/dev/null 2>&1; then + ASSET_URL="https://github.com/${REPO}/releases/latest/download/${ASSET_NAME}" + echo "gh not on PATH; curling ${ASSET_URL}" + curl_args=(-fsSL -o "${DL_DIR}/${ASSET_NAME}") + if [[ -n "${TOKEN}" ]]; then + curl_args+=(-H "Authorization: Bearer ${TOKEN}") + fi + curl "${curl_args[@]}" "${ASSET_URL}" +else + echo "error: neither gh nor curl available to fetch toolchain pack" >&2 + exit 1 +fi + +ZIP="$(find "${DL_DIR}" -maxdepth 1 -type f -name '*.zip' | head -n1)" +if [[ -z "${ZIP}" ]]; then + echo "error: no toolchain zip matching ${PATTERN}" >&2 + ls -la "${DL_DIR}" >&2 || true + exit 1 +fi + +echo "Unpacking toolchain from ${ZIP}" +if command -v unzip >/dev/null 2>&1; then + unzip -q "${ZIP}" -d "${OUT_DIR}" +else + python3 -c ' +import pathlib, zipfile, sys +z = next(pathlib.Path(sys.argv[1]).glob("*.zip")) +zipfile.ZipFile(z).extractall(sys.argv[2]) +' "${DL_DIR}" "${OUT_DIR}" +fi + +# Unwrap single top-level directory if present. +shopt -s nullglob +entries=("${OUT_DIR}"/*) +shopt -u nullglob +if [[ ${#entries[@]} -eq 1 && -d "${entries[0]}" ]]; then + TOOLCHAIN_DIR="$(cd "${entries[0]}" && pwd)" +else + TOOLCHAIN_DIR="$(cd "${OUT_DIR}" && pwd)" +fi + +if [[ ! -d "${TOOLCHAIN_DIR}/bin" ]]; then + echo "error: toolchain pack missing bin/ under ${TOOLCHAIN_DIR}" >&2 + find "${OUT_DIR}" -maxdepth 3 >&2 || true + exit 1 +fi + +echo "TOOLCHAIN_DIR=${TOOLCHAIN_DIR}" +if [[ -n "${GITHUB_ENV:-}" ]]; then + { + echo "TOOLCHAIN_DIR=${TOOLCHAIN_DIR}" + echo "PSXRECOMP_TOOLCHAIN_DIR=${TOOLCHAIN_DIR}" + echo "BPE_TOOLCHAIN_DIR=${TOOLCHAIN_DIR}" + } >>"${GITHUB_ENV}" +fi +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "toolchain-dir=${TOOLCHAIN_DIR}" >>"${GITHUB_OUTPUT}" +fi diff --git a/tools/package_setup_host.sh b/tools/package_setup_host.sh new file mode 100755 index 00000000..ca2ded63 --- /dev/null +++ b/tools/package_setup_host.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# Universal setup-host zip packager for PSXRecomp game repos. +# +# Stages the host exe, title sources, filtered psxrecomp/ + recomp-ui/, then +# finishes with stage_setup_sdk.sh (emitters, OpenBIOS, MinGW DLLs). +# Portable cmake/clang is NOT embedded by default — RetComM / the setup wizard +# download cmake-clang-v1 from retcomm-toolchains (or accept an offline zip). +# +# Usage (from game repo root): +# psxrecomp/tools/package_setup_host.sh \ +# --build-dir build-ci \ +# --artifact linux-x64 \ +# --zip-prefix bpe \ +# --exe-name Bomberman_Party_Edition_Recompiled \ +# --display-name "Bomberman Party Edition Recompiled" \ +# --recompiler-build build-recompiler \ +# [--project-file REL]... [--project-dir REL]... \ +# [--disc-hint "your legally owned disc"] \ +# [--version-env BPE_RELEASE_VERSION] \ +# [--embed-toolchain] # optional: copy PSXRECOMP_TOOLCHAIN_DIR into zip +# +# Env: +# RELEASE_VERSION / / VERSION file +# PSXRECOMP_TOOLCHAIN_DIR | TOOLCHAIN_DIR | BPE_TOOLCHAIN_DIR (only with --embed-toolchain) +# PSXRECOMP_EMBED_TOOLCHAIN=1 same as --embed-toolchain +# PSXRECOMP_RUNTIME_BIN_DIR | BPE_RUNTIME_BIN_DIR (Windows MinGW DLL search) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(pwd)" + +BUILD_DIR="" +ARTIFACT="" +ZIP_PREFIX="" +EXE_NAME="" +DISPLAY_NAME="" +RECOMPILER_BUILD="build-recompiler" +VERSION_ENV="RELEASE_VERSION" +DISC_HINT="your legally owned game disc" +PROJECT_FILES=() +PROJECT_DIRS=() +RUNTIME_BIN_DIR="${PSXRECOMP_RUNTIME_BIN_DIR:-${BPE_RUNTIME_BIN_DIR:-/usr/x86_64-w64-mingw32/bin}}" +EMBED_TOOLCHAIN=0 +if [[ "${PSXRECOMP_EMBED_TOOLCHAIN:-0}" == "1" ]]; then + EMBED_TOOLCHAIN=1 +fi + +usage() { + sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage ;; + --build-dir) BUILD_DIR="${2:?}"; shift 2 ;; + --artifact) ARTIFACT="${2:?}"; shift 2 ;; + --zip-prefix) ZIP_PREFIX="${2:?}"; shift 2 ;; + --exe-name) EXE_NAME="${2:?}"; shift 2 ;; + --display-name) DISPLAY_NAME="${2:?}"; shift 2 ;; + --recompiler-build) RECOMPILER_BUILD="${2:?}"; shift 2 ;; + --version-env) VERSION_ENV="${2:?}"; shift 2 ;; + --disc-hint) DISC_HINT="${2:?}"; shift 2 ;; + --project-file) PROJECT_FILES+=("${2:?}"); shift 2 ;; + --project-dir) PROJECT_DIRS+=("${2:?}"); shift 2 ;; + --runtime-bin) RUNTIME_BIN_DIR="${2:?}"; shift 2 ;; + --root) ROOT="${2:?}"; shift 2 ;; + --embed-toolchain) EMBED_TOOLCHAIN=1; shift ;; + --no-embed-toolchain) EMBED_TOOLCHAIN=0; shift ;; + *) + echo "error: unknown arg: $1" >&2 + usage + ;; + esac +done + +if [[ -z "${BUILD_DIR}" || -z "${ARTIFACT}" || -z "${ZIP_PREFIX}" || -z "${EXE_NAME}" ]]; then + echo "error: --build-dir, --artifact, --zip-prefix, and --exe-name are required" >&2 + usage +fi +if [[ -z "${DISPLAY_NAME}" ]]; then + DISPLAY_NAME="${EXE_NAME}" +fi + +ROOT="$(cd "${ROOT}" && pwd)" +if [[ -d "${ROOT}/${BUILD_DIR}" ]]; then + BUILD_DIR="$(cd "${ROOT}/${BUILD_DIR}" && pwd)" +elif [[ -d "${BUILD_DIR}" ]]; then + BUILD_DIR="$(cd "${BUILD_DIR}" && pwd)" +else + echo "error: build dir not found: ${BUILD_DIR}" >&2 + exit 1 +fi + +# Defaults for a typical title if caller passed none. +# Codegen host sources live in psxrecomp/host/ (copied with the framework tree). +if [[ ${#PROJECT_FILES[@]} -eq 0 && ${#PROJECT_DIRS[@]} -eq 0 ]]; then + PROJECT_FILES=(CMakeLists.txt game.toml VERSION) + for d in seeds launcher_assets; do + if [[ -e "${ROOT}/${d}" ]]; then + PROJECT_DIRS+=("${d}") + fi + done + for f in codegen_setup.c codegen_setup.h DISC.md README.md; do + if [[ -e "${ROOT}/${f}" ]]; then + PROJECT_FILES+=("${f}") + fi + done +fi + +# Resolve version. +VERSION="" +if [[ -n "${VERSION_ENV}" ]]; then + VERSION="${!VERSION_ENV:-}" +fi +if [[ -z "${VERSION}" && -n "${RELEASE_VERSION:-}" ]]; then + VERSION="${RELEASE_VERSION}" +fi +if [[ -z "${VERSION}" && -n "${BPE_RELEASE_VERSION:-}" ]]; then + VERSION="${BPE_RELEASE_VERSION}" +fi +if [[ -z "${VERSION}" && -f "${ROOT}/VERSION" ]]; then + VERSION="$(tr -d '[:space:]' <"${ROOT}/VERSION")" +fi +VERSION="$(printf '%s' "${VERSION}" | tr -d '[:space:]')" +VERSION="${VERSION#v}" +if [[ -z "${VERSION}" ]]; then + echo "error: VERSION empty (set ${VERSION_ENV} / RELEASE_VERSION or write VERSION)" >&2 + exit 1 +fi +printf '%s\n' "${VERSION}" >"${ROOT}/VERSION" + +DIST="${ROOT}/dist" +STAGE="${DIST}/stage-setup-${ARTIFACT}" +ZIP_NAME="${ZIP_PREFIX}-${VERSION}-${ARTIFACT}.zip" + +rm -rf "${STAGE}" +mkdir -p "${STAGE}" "${DIST}" +rm -f "${DIST}/${ZIP_NAME}" + +EXE="" +for cand in \ + "${BUILD_DIR}/${EXE_NAME}" \ + "${BUILD_DIR}/${EXE_NAME}.exe" \ + "${BUILD_DIR}/Release/${EXE_NAME}.exe" +do + if [[ -f "${cand}" ]]; then + EXE="${cand}" + break + fi +done +if [[ -z "${EXE}" ]]; then + echo "error: setup host executable '${EXE_NAME}' not found under ${BUILD_DIR}" >&2 + ls -la "${BUILD_DIR}" >&2 || true + exit 1 +fi + +cp -a "${EXE}" "${STAGE}/" +EXE_BASENAME="$(basename "${EXE}")" +EXE_DIR="$(dirname "${EXE}")" + +if [[ ! -d "${EXE_DIR}/assets/fonts" || ! -d "${EXE_DIR}/assets/img" ]]; then + echo "error: ${EXE_DIR}/assets/{fonts,img} missing — rebuild psx-runtime" >&2 + exit 1 +fi +mkdir -p "${STAGE}/assets" +cp -a "${EXE_DIR}/assets/fonts" "${STAGE}/assets/" +cp -a "${EXE_DIR}/assets/img" "${STAGE}/assets/" +if [[ ! -f "${STAGE}/assets/img/boxart.tga" && -f "${ROOT}/launcher_assets/img/boxart.tga" ]]; then + cp -a "${ROOT}/launcher_assets/img/boxart.tga" "${STAGE}/assets/img/boxart.tga" +fi + +copy_proj() { + local rel="$1" + if [[ -e "${ROOT}/${rel}" ]]; then + mkdir -p "$(dirname "${STAGE}/${rel}")" + cp -a "${ROOT}/${rel}" "${STAGE}/${rel}" + else + echo "error: missing ${rel}" >&2 + exit 1 + fi +} + +for f in "${PROJECT_FILES[@]}"; do + copy_proj "${f}" +done +for d in "${PROJECT_DIRS[@]}"; do + copy_proj "${d}" +done + +copy_tree_filtered() { + local src="$1" dest="$2" + shift 2 + mkdir -p "${dest}" + if command -v rsync >/dev/null 2>&1; then + rsync -a "$@" "${src}/" "${dest}/" + else + cp -a "${src}/." "${dest}/" + rm -rf "${dest}/.git" "${dest}/recompiler/build" "${dest}/generated" 2>/dev/null || true + find "${dest}" -type d -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true + find "${dest}" -type d \( -name 'build' -o -name 'build-*' \) -prune -exec rm -rf {} + 2>/dev/null || true + fi +} + +if [[ ! -d "${ROOT}/psxrecomp" ]]; then + echo "error: ${ROOT}/psxrecomp missing (expected framework submodule)" >&2 + exit 1 +fi +if [[ ! -d "${ROOT}/recomp-ui" ]]; then + echo "error: ${ROOT}/recomp-ui missing (expected UI submodule)" >&2 + exit 1 +fi + +copy_tree_filtered "${ROOT}/psxrecomp" "${STAGE}/psxrecomp" \ + --exclude '.git' \ + --exclude 'recompiler/build' \ + --exclude 'generated' \ + --exclude '__pycache__' \ + --exclude 'build' \ + --exclude 'build-*' + +copy_tree_filtered "${ROOT}/recomp-ui" "${STAGE}/recomp-ui" \ + --exclude '.git' \ + --exclude 'build' \ + --exclude '__pycache__' + +# Never ship game generated C or common disc working trees. +rm -rf "${STAGE}/generated" "${STAGE}/bpe" "${STAGE}/motk" "${STAGE}/disc" + +STAGE_SDK="${SCRIPT_DIR}/stage_setup_sdk.sh" +if [[ ! -f "${STAGE_SDK}" ]]; then + echo "error: missing ${STAGE_SDK}" >&2 + exit 1 +fi +chmod +x "${STAGE_SDK}" 2>/dev/null || true + +stage_args=( + --stage "${STAGE}" + --framework "${ROOT}/psxrecomp" + --search-dir "${EXE_DIR}" + --search-dir "${BUILD_DIR}" + --runtime-bin "${RUNTIME_BIN_DIR}" + --host-exe "${STAGE}/${EXE_BASENAME}" + --recompiler-build "${RECOMPILER_BUILD}" +) +if [[ "${EMBED_TOOLCHAIN}" -eq 1 ]]; then + if [[ -z "${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" ]]; then + echo "error: --embed-toolchain requires PSXRECOMP_TOOLCHAIN_DIR (or TOOLCHAIN_DIR)" >&2 + exit 1 + fi + stage_args+=(--toolchain-dir "${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR}}}") +else + stage_args+=(--allow-no-toolchain) +fi + +bash "${STAGE_SDK}" "${stage_args[@]}" + +cat >"${STAGE}/README-SETUP.txt" </dev/null || find "${STAGE}" -exec touch {} + + +( + cd "${STAGE}" + if command -v zip >/dev/null 2>&1; then + zip -r -q "${DIST}/${ZIP_NAME}" . + else + echo "error: zip not found" >&2 + exit 1 + fi +) + +echo "Wrote ${DIST}/${ZIP_NAME}" +du -h "${DIST}/${ZIP_NAME}" diff --git a/tools/prepare_disc.py b/tools/prepare_disc.py new file mode 100644 index 00000000..92ca89c6 --- /dev/null +++ b/tools/prepare_disc.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +"""Normalize a PS1 disc dump to MODE2/2352 .bin/.cue for psxrecomp games. + +Framework tool — game identity comes from ``game.toml`` (``[game]`` + optional +``[prepare_disc]``) or CLI flags. + +Accepted inputs: + * Redump / raw MODE2/2352 ``.bin`` (+ ``.cue``) + * Cooked 2048-byte/sector ``.iso`` (RomM-style libraries) + * 2448-byte/sector raw dumps (MotK-style): trim to 2352 + +Writes working ``.bin`` + ``.cue``, extracts ``SYSTEM.CNF`` + boot EXE, and +prints ``RESULT_CUE=`` for the first-run wizard / RetComM. + +ISO→2352 sets Mode2 Form1 sync/header/subheader/EDC (ECC zeroed — fine for +software readers). Rebuilt images are not bit-identical to Redump. +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import struct +import sys +from dataclasses import dataclass, field +from pathlib import Path + +DST_SEC = 2352 +SRC_2448 = 2448 +USER = 2048 +USER_OFF = 24 +SYNC = bytes([0x00] + [0xFF] * 10 + [0x00]) + + +def _configure_stdio() -> None: + """Avoid UnicodeEncodeError on Windows consoles (cp1252 / Store Python).""" + for stream in (sys.stdout, sys.stderr): + reconf = getattr(stream, "reconfigure", None) + if callable(reconf): + try: + reconf(errors="replace") + except Exception: # noqa: BLE001 — best-effort stdio harden + pass + + +def _build_edc_table() -> list[int]: + table = [] + for i in range(256): + x = i + for _ in range(8): + x = (x >> 1) ^ 0xD8018001 if (x & 1) else (x >> 1) + table.append(x & 0xFFFFFFFF) + return table + + +_EDC_TABLE = _build_edc_table() + + +def _adjust_edc(buf: bytearray, start: int, length: int) -> None: + x = 0 + end = start + length + for i in range(start, end): + x = _EDC_TABLE[(x ^ buf[i]) & 0xFF] ^ (x >> 8) + struct.pack_into(" bytes: + if len(user) != USER: + raise ValueError("user data must be 2048 bytes") + sector = bytearray(DST_SEC) + sector[0:12] = SYNC + t = lba + 150 + mm, ss, ff = t // (75 * 60), (t // 75) % 60, t % 75 + sector[12] = ((mm // 10) << 4) | (mm % 10) + sector[13] = ((ss // 10) << 4) | (ss % 10) + sector[14] = ((ff // 10) << 4) | (ff % 10) + sector[15] = 0x02 + sub = bytes([0x00, 0x00, submode & 0xFF, 0x00]) + sector[16:20] = sub + sector[20:24] = sub + sector[24 : 24 + USER] = user + _adjust_edc(sector, 0x10, 0x800 + 8) + return bytes(sector) + + +@dataclass +class KnownImage: + size: int = 0 + md5: str = "" + sha1: str = "" + + +@dataclass +class PrepareConfig: + project_root: Path + out_dir: Path + bin_name: str + cue_name: str + boot_exe: str + serial: str = "" + known: list[KnownImage] = field(default_factory=list) + skip_hash_check: bool = False + + +def file_hashes(path: Path) -> tuple[str, str, int]: + h_md5 = hashlib.md5() + h_sha1 = hashlib.sha1() + size = 0 + with open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + h_md5.update(chunk) + h_sha1.update(chunk) + return h_md5.hexdigest(), h_sha1.hexdigest(), size + + +def _parse_array_items(inner: str) -> list[object]: + items: list[object] = [] + if not inner.strip(): + return items + for part in re.split(r",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", inner): + part = part.strip().rstrip(",").strip() + if not part: + continue + if part.startswith('"') and part.endswith('"'): + items.append(part[1:-1]) + else: + try: + items.append(int(part, 0)) + except ValueError: + items.append(part) + return items + + +def _parse_toml_simple(text: str) -> dict[str, dict[str, str]]: + """Minimal TOML reader for flat string/int/bool + string/int arrays. + + Supports multi-line arrays (``key = [ … ]`` spanning lines). + """ + sections: dict[str, dict[str, object]] = {"": {}} + cur = "" + array_key: str | None = None + array_buf: list[str] = [] + for raw in text.splitlines(): + line = raw.split("#", 1)[0].strip() + if array_key is not None: + array_buf.append(line) + joined = " ".join(array_buf) + if "]" in joined: + inner = joined.split("]", 1)[0] + # Drop leading "[" already consumed when array opened + if inner.startswith("["): + inner = inner[1:] + sections[cur][array_key] = _parse_array_items(inner) + array_key = None + array_buf = [] + continue + if not line: + continue + if line.startswith("[") and line.endswith("]"): + cur = line[1:-1].strip() + sections.setdefault(cur, {}) + continue + if "=" not in line: + continue + key, val = line.split("=", 1) + key = key.strip() + val = val.strip() + if val.startswith("["): + if val.endswith("]") and val.count("[") == val.count("]"): + sections[cur][key] = _parse_array_items(val[1:-1]) + else: + array_key = key + array_buf = [val[1:]] # after opening [ + continue + if val.startswith('"') and val.endswith('"'): + sections[cur][key] = val[1:-1] + elif val.lower() in ("true", "false"): + sections[cur][key] = val.lower() == "true" + else: + try: + sections[cur][key] = int(val, 0) + except ValueError: + sections[cur][key] = val + return sections # type: ignore[return-value] + + +def load_config(project_root: Path, config_path: Path | None, out_dir_cli: str | None, + skip_hash: bool) -> PrepareConfig: + project_root = project_root.resolve() + game: dict = {} + prep: dict = {} + if config_path and config_path.is_file(): + sections = _parse_toml_simple(config_path.read_text(encoding="utf-8")) + game = sections.get("game", {}) or {} + prep = sections.get("prepare_disc", {}) or {} + + boot = str(prep.get("boot_exe") or "") + if not boot: + exe = str(game.get("exe") or "") + boot = Path(exe).name if exe else "" + if not boot: + boot = "SLUS_000.00" + + out_rel = str(prep.get("out_dir") or "prepared_disc") + out_dir = Path(out_dir_cli).expanduser() if out_dir_cli else (project_root / out_rel) + + disc = str(game.get("disc") or prep.get("bin_name") or "game.bin") + bin_name = str(prep.get("bin_name") or Path(disc).name) + cue_name = str(prep.get("cue_name") or (Path(bin_name).stem + ".cue")) + + known: list[KnownImage] = [] + sizes = prep.get("known_sizes") or [] + md5s = prep.get("known_md5") or [] + sha1s = prep.get("known_sha1") or [] + if isinstance(sizes, list) or isinstance(md5s, list) or isinstance(sha1s, list): + n = max( + len(sizes) if isinstance(sizes, list) else 0, + len(md5s) if isinstance(md5s, list) else 0, + len(sha1s) if isinstance(sha1s, list) else 0, + ) + for i in range(n): + known.append( + KnownImage( + size=int(sizes[i]) if isinstance(sizes, list) and i < len(sizes) else 0, + md5=str(md5s[i]).lower() + if isinstance(md5s, list) and i < len(md5s) + else "", + sha1=str(sha1s[i]).lower() + if isinstance(sha1s, list) and i < len(sha1s) + else "", + ) + ) + + return PrepareConfig( + project_root=project_root, + out_dir=out_dir.resolve(), + bin_name=bin_name, + cue_name=cue_name, + boot_exe=boot, + serial=str(game.get("id") or ""), + known=known, + skip_hash_check=skip_hash, + ) + + +def resolve_cue_bin(cue_path: Path) -> Path: + text = cue_path.read_text(encoding="utf-8", errors="replace") + m = re.search(r'FILE\s+"([^"]+)"\s+BINARY', text, re.I) + if not m: + m = re.search(r"FILE\s+(\S+)\s+BINARY", text, re.I) + if not m: + raise SystemExit(f"no BINARY FILE in cue: {cue_path}") + cand = Path(m.group(1)) + if not cand.is_absolute(): + cand = cue_path.parent / cand + if not cand.is_file(): + raise SystemExit(f"cue references missing bin: {cand}") + return cand + + +def detect_kind(path: Path, size: int) -> str: + suf = path.suffix.lower() + if suf == ".cue": + return "cue" + if suf == ".iso": + return "iso" + # MotK-style 2448 before generic 2352 (some sizes divisible by both). + if size % SRC_2448 == 0 and size > SRC_2448 and size % DST_SEC != 0: + return "raw2448" + if size % SRC_2448 == 0 and size > SRC_2448: + # Peek: if sync looks like CD at +0, treat as 2448 when user data path works. + with open(path, "rb") as f: + head = f.read(16) + if head[:12] == SYNC: + # Could be 2352 or 2448; check sector stride via second sync. + fpos_2352_ok = False + fpos_2448_ok = False + with open(path, "rb") as f: + f.seek(DST_SEC) + fpos_2352_ok = f.read(12) == SYNC + f.seek(SRC_2448) + fpos_2448_ok = f.read(12) == SYNC + if fpos_2448_ok and not fpos_2352_ok: + return "raw2448" + if size % DST_SEC == 0 and size > DST_SEC: + with open(path, "rb") as f: + sec0 = f.read(12) + f.seek(16 * DST_SEC) + sync16 = f.read(12) + if sec0 == SYNC or sync16 == SYNC: + return "bin2352" + if size % USER == 0 and size > USER: + return "iso" + if size % SRC_2448 == 0 and size > SRC_2448: + return "raw2448" + raise SystemExit( + f"unrecognized disc image {path} (size={size}). " + "Need MODE2/2352 .bin/.cue, cooked 2048 .iso, or 2448-byte/sector dump." + ) + + +def read_user_bin(data: bytes, lba: int) -> bytes: + off = lba * DST_SEC + if off + DST_SEC > len(data): + raise KeyError(lba) + if data[off : off + 12] != SYNC: + raise SystemExit(f"bad sync at LBA {lba}") + return data[off + USER_OFF : off + USER_OFF + USER] + + +def read_user_iso(data: bytes, lba: int) -> bytes: + off = lba * USER + if off + USER > len(data): + raise KeyError(lba) + return data[off : off + USER] + + +def parse_root_entries(root: bytes) -> dict[str, tuple[int, int]]: + entries: dict[str, tuple[int, int]] = {} + i = 0 + while i < len(root): + reclen = root[i] + if reclen == 0: + i = ((i // USER) + 1) * USER + if i >= len(root): + break + continue + if i + reclen > len(root): + break + extent = struct.unpack_from(" tuple[dict[str, tuple[int, int]], dict[str, bytes]]: + pvd = read_user(data, 16) + if pvd[1:6] != b"CD001": + raise SystemExit("PVD not found at LBA 16 (expected CD001)") + root_extent = struct.unpack_from(" 0: + sector = read_user(data, lba) + take = min(USER, rem) + out += sector[:take] + rem -= take + lba += 1 + files[need] = bytes(out) + if files[boot_exe][:8] != b"PS-X EXE": + raise SystemExit(f"{boot_exe} is not a PS-X EXE") + return entries, files + + +def iso_to_bin(iso_data: bytes) -> bytes: + if len(iso_data) % USER != 0: + raise SystemExit(f"ISO size {len(iso_data)} is not a multiple of 2048") + n = len(iso_data) // USER + out = bytearray() + for lba in range(n): + user = iso_data[lba * USER : (lba + 1) * USER] + out += encode_mode2_form1(user, lba, submode=0x08) + return bytes(out) + + +def raw2448_to_bin(src: Path) -> bytes: + out = bytearray() + n = 0 + with open(src, "rb") as inf: + while True: + sec = inf.read(SRC_2448) + if not sec: + break + if len(sec) != SRC_2448: + raise SystemExit(f"truncated 2448 sector {n}: got {len(sec)} bytes") + out += sec[:DST_SEC] + n += 1 + print(f" trimmed {n} sectors 2448 -> 2352") + return bytes(out) + + +def matches_known(cfg: PrepareConfig, size: int, md5: str, sha1: str) -> bool: + if not cfg.known: + return False + md5 = md5.lower() + sha1 = sha1.lower() + for k in cfg.known: + if k.md5 and k.md5.lower() == md5: + if not k.size or k.size == size: + return True + if k.sha1 and k.sha1.lower() == sha1: + if not k.size or k.size == size: + return True + return False + + +def main() -> int: + _configure_stdio() + here = Path(__file__).resolve().parent + # tools/ -> psxrecomp/ -> often game root is parent of psxrecomp + fw_root = here.parent + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("source", nargs="?", default="", help="path to .iso / .cue / .bin") + ap.add_argument( + "--config", + default="", + help="game.toml (reads [game] + [prepare_disc])", + ) + ap.add_argument( + "--project-root", + default="", + help="game project root (default: parent of config, else cwd)", + ) + ap.add_argument("--out-dir", default="", help="override prepare_disc.out_dir") + ap.add_argument("--bin-name", default="", help="override output .bin basename") + ap.add_argument("--cue-name", default="", help="override output .cue basename") + ap.add_argument("--boot-exe", default="", help="override boot EXE name on disc") + ap.add_argument( + "--skip-hash-check", + action="store_true", + help="do not require prepare_disc.known_* digests", + ) + args = ap.parse_args() + + config_path = Path(args.config).expanduser() if args.config else None + if config_path and not config_path.is_file(): + print(f"config not found: {config_path}", file=sys.stderr) + return 1 + + if args.project_root: + project_root = Path(args.project_root).expanduser().resolve() + elif config_path: + project_root = config_path.parent.resolve() + else: + # Invoked from a game tree: prefer cwd if it has game.toml + cwd = Path.cwd() + project_root = cwd if (cwd / "game.toml").is_file() else fw_root.parent + + if not config_path: + cand = project_root / "game.toml" + if cand.is_file(): + config_path = cand + + cfg = load_config(project_root, config_path, args.out_dir or None, args.skip_hash_check) + if args.bin_name: + cfg.bin_name = args.bin_name + if args.cue_name: + cfg.cue_name = args.cue_name + if args.boot_exe: + cfg.boot_exe = args.boot_exe + + if not args.source: + # Convenience: existing working bin under out_dir + cand = cfg.out_dir / cfg.bin_name + if cand.is_file(): + args.source = str(cand) + else: + print("source required (.iso / .cue / .bin)", file=sys.stderr) + return 1 + + src = Path(args.source).expanduser().resolve() + if not src.is_file(): + print(f"source not found: {src}", file=sys.stderr) + return 1 + + cfg.out_dir.mkdir(parents=True, exist_ok=True) + + src_md5, src_sha1, src_size = file_hashes(src) + print(f"source: {src}") + print(f" size {src_size}") + print(f" md5 {src_md5}") + print(f" sha1 {src_sha1}") + if cfg.serial: + print(f" game {cfg.serial} boot={cfg.boot_exe}") + + kind = detect_kind(src, src_size) + if kind == "cue": + src = resolve_cue_bin(src) + src_md5, src_sha1, src_size = file_hashes(src) + print(f"cue -> bin: {src}") + print(f" size {src_size}") + print(f" md5 {src_md5}") + print(f" sha1 {src_sha1}") + kind = detect_kind(src, src_size) + + known_hit = matches_known(cfg, src_size, src_md5, src_sha1) + if cfg.known and not cfg.skip_hash_check: + if known_hit: + print(" known digest OK") + elif kind in ("bin2352", "raw2448"): + print( + "source digests are not in prepare_disc.known_* " + "(pass --skip-hash-check to force)", + file=sys.stderr, + ) + return 1 + else: + print( + "ISO digests are not in prepare_disc.known_*; " + "verifying boot EXE only." + ) + elif not cfg.known and not cfg.skip_hash_check and kind == "bin2352": + print(" no prepare_disc.known_* configured - verifying boot EXE only") + + if kind == "bin2352": + bin_data = src.read_bytes() + entries, files = extract_via(read_user_bin, bin_data, cfg.boot_exe) + elif kind == "iso": + iso_data = src.read_bytes() + entries, files = extract_via(read_user_iso, iso_data, cfg.boot_exe) + print(f" converting {len(iso_data) // USER} sectors ISO -> MODE2/2352...") + bin_data = iso_to_bin(iso_data) + extract_via(read_user_bin, bin_data, cfg.boot_exe) + elif kind == "raw2448": + print(" converting 2448 -> 2352...") + bin_data = raw2448_to_bin(src) + entries, files = extract_via(read_user_bin, bin_data, cfg.boot_exe) + else: + print(f"unsupported kind {kind}", file=sys.stderr) + return 1 + + print(f" root entries: {sorted(entries)[:24]}") + pc0 = struct.unpack_from(" [options] +# +# Options: +# --framework DIR psxrecomp source tree (default: /psxrecomp) +# --sdk-overlay DIR Legacy optional overlay onto stage/psxrecomp +# (prefer shipping CLI/tools inside the submodule) +# --recompiler-build DIR Where psxrecomp-game/bios were built (repeatable) +# --toolchain-dir DIR Pack root with bin/; embedded as stage/toolchain/ +# --allow-no-toolchain Warn instead of failing when toolchain unset +# --host-exe PATH Host PE for Windows DLL bundling (optional) +# --runtime-bin DIR MinGW runtime DLL search dir (repeatable) +# --search-dir DIR Extra DLL search dir (repeatable) +# --require-cli Require psxrecomp_cli.py after overlays (default on) +# +# Env aliases for toolchain dir (first wins): +# PSXRECOMP_TOOLCHAIN_DIR, TOOLCHAIN_DIR, BPE_TOOLCHAIN_DIR +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +FW_TOOLS="${SCRIPT_DIR}" + +STAGE="" +FRAMEWORK="" +SDK_OVERLAY="" +RECOMPILER_BUILDS=() +# Empty until --toolchain-dir or (when not allow-no) env aliases. +TOOLCHAIN_DIR="" +TOOLCHAIN_DIR_SET=0 +ALLOW_NO_TOOLCHAIN=0 +HOST_EXE="" +RUNTIME_BINS=() +SEARCH_DIRS=() +REQUIRE_CLI=1 + +usage() { + sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//' + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage ;; + --stage) STAGE="${2:?}"; shift 2 ;; + --framework) FRAMEWORK="${2:?}"; shift 2 ;; + --sdk-overlay) SDK_OVERLAY="${2:?}"; shift 2 ;; + --recompiler-build) RECOMPILER_BUILDS+=("${2:?}"); shift 2 ;; + --toolchain-dir) TOOLCHAIN_DIR="${2:?}"; TOOLCHAIN_DIR_SET=1; shift 2 ;; + --allow-no-toolchain) ALLOW_NO_TOOLCHAIN=1; shift ;; + --host-exe) HOST_EXE="${2:?}"; shift 2 ;; + --runtime-bin) RUNTIME_BINS+=("${2:?}"); shift 2 ;; + --search-dir) SEARCH_DIRS+=("${2:?}"); shift 2 ;; + --require-cli) REQUIRE_CLI=1; shift ;; + --no-require-cli) REQUIRE_CLI=0; shift ;; + *) + echo "error: unknown arg: $1" >&2 + usage + ;; + esac +done + +if [[ -z "${STAGE}" ]]; then + echo "error: --stage is required" >&2 + usage +fi +STAGE="$(cd "${STAGE}" && pwd)" + +# Env aliases only when embedding is expected (not --allow-no-toolchain alone). +if [[ "${TOOLCHAIN_DIR_SET}" -eq 0 && "${ALLOW_NO_TOOLCHAIN}" -eq 0 ]]; then + TOOLCHAIN_DIR="${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" +fi + +if [[ -z "${FRAMEWORK}" ]]; then + if [[ -d "${PWD}/psxrecomp" ]]; then + FRAMEWORK="$(cd "${PWD}/psxrecomp" && pwd)" + else + FRAMEWORK="$(cd "${SCRIPT_DIR}/.." && pwd)" + fi +else + FRAMEWORK="$(cd "${FRAMEWORK}" && pwd)" +fi + +mkdir -p "${STAGE}/psxrecomp" + +# Optional SDK overlay (CLI / prepare_disc / progress helpers). +if [[ -n "${SDK_OVERLAY}" ]]; then + if [[ ! -d "${SDK_OVERLAY}" ]]; then + echo "error: --sdk-overlay not a directory: ${SDK_OVERLAY}" >&2 + exit 1 + fi + if command -v rsync >/dev/null 2>&1; then + rsync -a "${SDK_OVERLAY}/" "${STAGE}/psxrecomp/" + else + cp -a "${SDK_OVERLAY}/." "${STAGE}/psxrecomp/" + fi + echo "overlaid SDK from ${SDK_OVERLAY}" +fi + +find_tool_bin() { + local name="$1" + local dir cand + local -a roots=() + local r + for r in "${RECOMPILER_BUILDS[@]+"${RECOMPILER_BUILDS[@]}"}"; do + roots+=("$(cd "${r}" && pwd)") + done + roots+=( + "${STAGE}/psxrecomp/recompiler/build" + "${FRAMEWORK}/recompiler/build" + "${PWD}/build-recompiler" + ) + for dir in "${roots[@]}"; do + [[ -d "${dir}" ]] || continue + for cand in \ + "${dir}/${name}" \ + "${dir}/${name}.exe" \ + "${dir}/Release/${name}.exe" + do + if [[ -f "${cand}" ]]; then + echo "${cand}" + return 0 + fi + done + done + return 1 +} + +GAME_BIN="$(find_tool_bin psxrecomp-game || true)" +BIOS_BIN="$(find_tool_bin psxrecomp-bios || true)" +if [[ -z "${GAME_BIN}" ]]; then + echo "error: psxrecomp-game not found (pass --recompiler-build)" >&2 + exit 1 +fi +if [[ -z "${BIOS_BIN}" ]]; then + echo "error: psxrecomp-bios not found (required for OpenBIOS regen)" >&2 + exit 1 +fi + +mkdir -p "${STAGE}/psxrecomp/recompiler/build" +cp -a "${GAME_BIN}" \ + "${STAGE}/psxrecomp/recompiler/build/$(basename "${GAME_BIN}")" +cp -a "${BIOS_BIN}" \ + "${STAGE}/psxrecomp/recompiler/build/$(basename "${BIOS_BIN}")" +chmod +x "${STAGE}/psxrecomp/recompiler/build/$(basename "${GAME_BIN}")" 2>/dev/null || true +chmod +x "${STAGE}/psxrecomp/recompiler/build/$(basename "${BIOS_BIN}")" 2>/dev/null || true +echo "staged emitters from ${GAME_BIN%/*}" + +cat >"${STAGE}/psxrecomp/retcomm-sdk.json" <<'EOF' +{ + "cli": "psxrecomp_cli.py", + "id": "psxrecomp-tools", + "game_bin": "recompiler/build/psxrecomp-game", + "bios_bin": "recompiler/build/psxrecomp-bios" +} +EOF + +for f in OpenBIOS.toml openbios.bin OpenBIOS.LICENSE SCPH1001.toml; do + if [[ ! -f "${STAGE}/psxrecomp/bios/${f}" ]]; then + echo "error: missing psxrecomp/bios/${f} in staged tree" >&2 + exit 1 + fi +done + +if [[ "${REQUIRE_CLI}" -eq 1 && ! -f "${STAGE}/psxrecomp/psxrecomp_cli.py" ]]; then + echo "error: missing psxrecomp/psxrecomp_cli.py in staged tree" >&2 + exit 1 +fi + +# Never ship retail BIOS dumps from the stage. +rm -f "${STAGE}/psxrecomp/bios/SCPH1001.BIN" \ + "${STAGE}/psxrecomp/bios/SCPH1001.bin" 2>/dev/null || true + +if [[ -n "${TOOLCHAIN_DIR}" && -d "${TOOLCHAIN_DIR}" ]]; then + if [[ ! -d "${TOOLCHAIN_DIR}/bin" ]]; then + echo "error: toolchain dir missing bin/: ${TOOLCHAIN_DIR}" >&2 + exit 1 + fi + mkdir -p "${STAGE}/toolchain" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete "${TOOLCHAIN_DIR}/" "${STAGE}/toolchain/" + else + rm -rf "${STAGE}/toolchain" + mkdir -p "${STAGE}/toolchain" + cp -a "${TOOLCHAIN_DIR}/." "${STAGE}/toolchain/" + fi + echo "bundled toolchain from ${TOOLCHAIN_DIR}" +elif [[ "${ALLOW_NO_TOOLCHAIN}" -eq 1 ]]; then + echo "note: no embedded toolchain/ — RetComM/wizard will download cmake-clang-v1" \ + "(or accept an offline zip / system cmake)" >&2 +else + echo "error: toolchain dir required (pass --toolchain-dir or set PSXRECOMP_TOOLCHAIN_DIR)" >&2 + exit 1 +fi + +# Windows MinGW DLL bundling for host + emitters. +BUNDLE="${FW_TOOLS}/bundle_mingw_dlls.sh" +if [[ ! -f "${BUNDLE}" ]]; then + # Staged copy (after sdk overlay) may own the helper. + if [[ -f "${STAGE}/psxrecomp/tools/bundle_mingw_dlls.sh" ]]; then + BUNDLE="${STAGE}/psxrecomp/tools/bundle_mingw_dlls.sh" + fi +fi + +need_dlls=0 +if [[ -n "${HOST_EXE}" && "${HOST_EXE}" == *.exe ]]; then + need_dlls=1 +fi +if [[ -f "${STAGE}/psxrecomp/recompiler/build/psxrecomp-game.exe" ]]; then + need_dlls=1 +fi + +if [[ "${need_dlls}" -eq 1 ]]; then + if [[ ! -f "${BUNDLE}" ]]; then + echo "error: bundle_mingw_dlls.sh not found next to stage_setup_sdk.sh" >&2 + exit 1 + fi + chmod +x "${BUNDLE}" 2>/dev/null || true + + for emitter in psxrecomp-game.exe psxrecomp-bios.exe; do + if [[ ! -f "${STAGE}/psxrecomp/recompiler/build/${emitter}" ]]; then + echo "error: missing ${emitter} in staged tree" >&2 + exit 1 + fi + done + + args=() + for d in "${RUNTIME_BINS[@]+"${RUNTIME_BINS[@]}"}"; do + args+=(--runtime-bin "${d}") + done + for d in "${SEARCH_DIRS[@]+"${SEARCH_DIRS[@]}"}"; do + args+=(--search-dir "${d}") + done + if [[ -n "${HOST_EXE}" && -f "${HOST_EXE}" ]]; then + args+=( + --exe "${HOST_EXE}" + --dest "${STAGE}" + --label "$(basename "${HOST_EXE}")" + ) + fi + args+=( + --exe "${STAGE}/psxrecomp/recompiler/build/psxrecomp-game.exe" + --dest "${STAGE}/psxrecomp/recompiler/build" + --label "psxrecomp-game.exe" + --exe "${STAGE}/psxrecomp/recompiler/build/psxrecomp-bios.exe" + --dest "${STAGE}/psxrecomp/recompiler/build" + --label "psxrecomp-bios.exe" + ) + # Host (MSYS2 GCC) still needs these when imported. llvm-mingw static + # emitters typically import neither — --require is skipped per-exe then. + if [[ -n "${HOST_EXE}" && -f "${HOST_EXE}" ]]; then + args+=( + --require libgcc_s_seh-1.dll + --require libstdc++-6.dll + ) + fi + bash "${BUNDLE}" "${args[@]}" +fi + +echo "stage_setup_sdk: ready under ${STAGE}/psxrecomp" diff --git a/tools/toolchain_pack.py b/tools/toolchain_pack.py new file mode 100644 index 00000000..ff13ee40 --- /dev/null +++ b/tools/toolchain_pack.py @@ -0,0 +1,404 @@ +"""Resolve / download / unpack portable cmake-clang-v1 toolchain packs. + +Used by psxrecomp_cli (ensure-toolchain, rebuild) and mirrors RetComM's +shared-cache layout when possible. +""" + +from __future__ import annotations + +import os +import platform +import shutil +import sys +import tempfile +import urllib.error +import urllib.request +import zipfile +from pathlib import Path +from typing import Optional + +DEFAULT_REPO = "TechnicallyComputers/retcomm-toolchains" +PACK_ID = "cmake-clang-v1" + +_ASSET = { + "linux-x64": "cmake-clang-v1-linux-x64.zip", + "windows-x64": "cmake-clang-v1-windows-x64.zip", + "macos-arm64": "cmake-clang-v1-macos-universal.zip", + "macos-x64": "cmake-clang-v1-macos-universal.zip", +} + + +def sys_platform_is_windows() -> bool: + return sys.platform == "win32" + + +def host_artifact() -> str: + if sys_platform_is_windows(): + return "windows-x64" + if sys.platform == "darwin": + return "macos-arm64" if platform.machine().lower() in ("arm64", "aarch64") else "macos-x64" + return "linux-x64" + + +def cmake_name() -> str: + return "cmake.exe" if sys_platform_is_windows() else "cmake" + + +def bin_looks_usable(bin_dir: Path) -> bool: + return bin_dir.is_dir() and (bin_dir / cmake_name()).is_file() + + +def pack_root_looks_usable(root: Path) -> bool: + return bin_looks_usable(root / "bin") + + +def unwrap_pack_root(path: Path) -> Path: + """If *path* is a single nested directory with bin/, return that child.""" + if pack_root_looks_usable(path): + return path + try: + kids = [p for p in path.iterdir() if p.is_dir() and not p.name.startswith(".")] + except OSError: + return path + if len(kids) == 1 and pack_root_looks_usable(kids[0]): + return kids[0] + return path + + +def resolve_embedded_bin(project_root: Path) -> Optional[Path]: + root = project_root / "toolchain" + if not root.is_dir(): + return None + direct = root / "bin" + if bin_looks_usable(direct): + return direct + try: + kids = [p for p in root.iterdir() if p.is_dir()] + except OSError: + return None + if len(kids) == 1: + nested = kids[0] / "bin" + if bin_looks_usable(nested): + return nested + for kid in kids: + nested = kid / "bin" + if bin_looks_usable(nested): + return nested + return None + + +def env_toolchain_roots() -> list[Path]: + out: list[Path] = [] + for key in ("PSXRECOMP_TOOLCHAIN_DIR", "RETCOMM_TOOLCHAIN_DIR", "TOOLCHAIN_DIR", + "BPE_TOOLCHAIN_DIR"): + raw = os.environ.get(key) + if raw: + out.append(Path(raw).expanduser()) + return out + + +def shared_cache_roots() -> list[Path]: + """Candidate parent dirs that contain / packs (or a flat pack).""" + roots: list[Path] = [] + if sys_platform_is_windows(): + local = os.environ.get("LOCALAPPDATA") + if local: + roots.append(Path(local) / "retcomm" / "toolchains" / PACK_ID) + roots.append(Path(local) / "psxrecomp" / "toolchains" / PACK_ID) + xdg = os.environ.get("XDG_DATA_HOME") + home = Path.home() + if xdg: + roots.append(Path(xdg) / "retcomm" / "toolchains" / PACK_ID) + roots.append(Path(xdg) / "psxrecomp" / "toolchains" / PACK_ID) + else: + roots.append(home / ".local" / "share" / "retcomm" / "toolchains" / PACK_ID) + roots.append(home / ".local" / "share" / "psxrecomp" / "toolchains" / PACK_ID) + # Dedup while preserving order. + seen: set[str] = set() + uniq: list[Path] = [] + for r in roots: + key = str(r) + if key not in seen: + seen.add(key) + uniq.append(r) + return uniq + + +def preferred_install_root() -> Path: + """Where newly downloaded / offline-unpacked packs land.""" + for r in shared_cache_roots(): + # Prefer psxrecomp-owned cache for writes; fall through to first. + if "psxrecomp" in r.parts: + r.mkdir(parents=True, exist_ok=True) + return r + r = shared_cache_roots()[0] + r.mkdir(parents=True, exist_ok=True) + return r + + +STAMP_NAME = ".psxrecomp-bin" + + +def is_windows_store_python() -> bool: + """Microsoft Store Python redirects %LOCALAPPDATA% writes into LocalCache.""" + if not sys_platform_is_windows(): + return False + try: + exe = str(Path(sys.executable).resolve()).lower() + except OSError: + exe = str(sys.executable).lower() + return "windowsapps" in exe or "pythonsoftwarefoundation" in exe + + +def write_toolchain_stamp(project_root: Path, bin_dir: Path) -> None: + """Write project_root/toolchain/.psxrecomp-bin for the C host to read.""" + stamp_dir = project_root / "toolchain" + try: + stamp_dir.mkdir(parents=True, exist_ok=True) + (stamp_dir / STAMP_NAME).write_text( + str(bin_dir.resolve()) + "\n", encoding="utf-8" + ) + except OSError: + pass + + +def materialize_into_project( + project_root: Path, pack_root: Path, log=None +) -> Path: + """Copy a usable pack into project_root/toolchain/ (bin/ at top level). + + The setup host resolves this path first; installing here keeps cmake + visible even when the shared-cache path is long or Store-redirected. + """ + src = unwrap_pack_root(pack_root) + if not pack_root_looks_usable(src): + raise RuntimeError(f"toolchain pack unusable: {pack_root}") + dest = project_root / "toolchain" + # Already have a usable tree with the same cmake — keep it. + existing = resolve_embedded_bin(project_root) + if existing: + try: + if (existing / cmake_name()).resolve() == (src / "bin" / cmake_name()).resolve(): + write_toolchain_stamp(project_root, existing) + return unwrap_pack_root(dest) if pack_root_looks_usable(dest) else src + except OSError: + pass + if dest.exists(): + shutil.rmtree(dest, ignore_errors=True) + shutil.copytree(src, dest) + if log: + log(f"Installed project toolchain at {dest}") + root = unwrap_pack_root(dest) + write_toolchain_stamp(project_root, root / "bin") + return root + + +def find_cached_pack() -> Optional[Path]: + for base in shared_cache_roots(): + if not base.is_dir(): + continue + if pack_root_looks_usable(base): + return unwrap_pack_root(base) + try: + kids = sorted( + [p for p in base.iterdir() if p.is_dir()], + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + except OSError: + continue + for kid in kids: + root = unwrap_pack_root(kid) + if pack_root_looks_usable(root): + return root + return None + + +def resolve_toolchain_bin(project_root: Optional[Path] = None) -> Optional[Path]: + for env_root in env_toolchain_roots(): + root = unwrap_pack_root(env_root) + if pack_root_looks_usable(root): + return root / "bin" + if project_root is not None: + embedded = resolve_embedded_bin(project_root) + if embedded: + return embedded + cached = find_cached_pack() + if cached: + return cached / "bin" + return None + + +def activate_toolchain_bin(bin_dir: Path, log=None) -> None: + prefix = str(bin_dir) + pack_root = bin_dir.parent + cur = os.environ.get("PATH", "") + parts = cur.split(os.pathsep) if cur else [] + if not parts or Path(parts[0]) != bin_dir: + os.environ["PATH"] = prefix + (os.pathsep + cur if cur else "") + # Windows cmake-clang-v1 ships zlib under the pack root; help FindZLIB. + pack_s = str(pack_root) + os.environ["ZLIB_ROOT"] = pack_s + prev_prefix = os.environ.get("CMAKE_PREFIX_PATH", "") + if not prev_prefix: + os.environ["CMAKE_PREFIX_PATH"] = pack_s + elif pack_s not in prev_prefix.split(os.pathsep): + os.environ["CMAKE_PREFIX_PATH"] = pack_s + os.pathsep + prev_prefix + for name, env_key in ( + ("clang", "CC"), + ("clang++", "CXX"), + ("clang.exe", "CC"), + ("clang++.exe", "CXX"), + ): + cand = bin_dir / name + if cand.is_file() and env_key not in os.environ: + os.environ[env_key] = str(cand) + if log: + log(f"Using toolchain: {bin_dir}") + + +def unpack_zip_to(zip_path: Path, dest: Path) -> Path: + """Extract *zip_path* into *dest* (replaced) and return usable pack root.""" + if dest.exists(): + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest) + root = unwrap_pack_root(dest) + if not pack_root_looks_usable(root): + raise RuntimeError(f"toolchain zip missing bin/{cmake_name()}: {zip_path}") + # If unwrap pointed at a child, normalize to dest by moving contents up when + # dest itself is not the pack root. + if root != dest and pack_root_looks_usable(root): + # Leave nested layout — resolve/activate handle it via unwrap. + return root + return root + + +def install_from_zip( + zip_path: Path, + tag: str = "offline", + *, + project_root: Optional[Path] = None, + log=None, +) -> Path: + zip_path = zip_path.expanduser().resolve() + if not zip_path.is_file(): + raise FileNotFoundError(f"toolchain zip not found: {zip_path}") + dest = preferred_install_root() / tag + root = unpack_zip_to(zip_path, dest) + if project_root is not None: + return materialize_into_project(project_root, root, log=log) + return root + + +def download_url(url: str, dest: Path, token: Optional[str] = None) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + req = urllib.request.Request(url, headers={"User-Agent": "psxrecomp-toolchain"}) + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=120) as resp, open(dest, "wb") as out: + shutil.copyfileobj(resp, out) + + +def download_latest_pack( + artifact: Optional[str] = None, + repo: str = DEFAULT_REPO, + log=None, + *, + project_root: Optional[Path] = None, +) -> Path: + art = artifact or host_artifact() + asset = _ASSET.get(art) + if not asset: + raise RuntimeError(f"unknown toolchain artifact: {art}") + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") or "" + url = f"https://github.com/{repo}/releases/latest/download/{asset}" + if log: + log(f"Downloading {asset} from {repo}...") + with tempfile.TemporaryDirectory(prefix="psxrecomp-tc-") as tmp: + zpath = Path(tmp) / asset + try: + download_url(url, zpath, token=token or None) + except urllib.error.HTTPError as e: + raise RuntimeError(f"toolchain download failed ({e.code}): {url}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"toolchain download failed: {e.reason}") from e + dest = preferred_install_root() / "latest" + root = unpack_zip_to(zpath, dest) + if log: + log(f"Installed toolchain pack at {root}") + if project_root is not None: + return materialize_into_project(project_root, root, log=log) + return root + + +def ensure_toolchain( + project_root: Optional[Path] = None, + *, + from_zip: Optional[Path] = None, + download: bool = False, + repo: str = DEFAULT_REPO, + log=None, +) -> Path: + """Return usable toolchain *bin* directory, installing if requested. + + Resolution order: env override → project toolchain/ → shared cache → + optional --from-zip → optional download. + + New installs also land in project_root/toolchain/ when project_root is set + (plus the shared cache), and write toolchain/.psxrecomp-bin for the host. + """ + if is_windows_store_python() and log: + log( + "Warning: Microsoft Store Python redirects AppData writes. " + "Prefer python.org Python if toolchain setup fails to find cmake." + ) + + if from_zip is not None: + root = install_from_zip( + Path(from_zip), project_root=project_root, log=log + ) + bin_dir = unwrap_pack_root(root) / "bin" + if not bin_looks_usable(bin_dir): + bin_dir = root / "bin" + if project_root is not None: + write_toolchain_stamp(project_root, bin_dir) + activate_toolchain_bin(bin_dir, log=log) + return bin_dir + + existing = resolve_toolchain_bin(project_root) + if existing: + # Prefer a project-local copy so the C host always sees the same tree. + if project_root is not None and not resolve_embedded_bin(project_root): + try: + root = materialize_into_project( + project_root, existing.parent, log=log + ) + existing = root / "bin" + except OSError as exc: + if log: + log(f"Could not copy toolchain into project: {exc}") + write_toolchain_stamp(project_root, existing) + elif project_root is not None: + write_toolchain_stamp(project_root, existing) + activate_toolchain_bin(existing, log=log) + return existing + + if download: + root = download_latest_pack( + repo=repo, log=log, project_root=project_root + ) + bin_dir = unwrap_pack_root(root) / "bin" + if not bin_looks_usable(bin_dir): + bin_dir = root / "bin" + if project_root is not None: + write_toolchain_stamp(project_root, bin_dir) + activate_toolchain_bin(bin_dir, log=log) + return bin_dir + + raise RuntimeError( + "No portable toolchain found. Pass --from-zip PATH to a " + "cmake-clang-v1-*.zip, use --download, set PSXRECOMP_TOOLCHAIN_DIR / " + "RETCOMM_TOOLCHAIN_DIR, or install cmake on PATH." + )