diff --git a/docs/GPU_FRAME_TOOLS.md b/docs/GPU_FRAME_TOOLS.md index a3253bea..b3f850b7 100644 --- a/docs/GPU_FRAME_TOOLS.md +++ b/docs/GPU_FRAME_TOOLS.md @@ -118,6 +118,51 @@ selected project's `game.toml` says. --- +## You cannot pause the game, and you do not need to + +`pause`, `continue`, `step` and `run_to_frame` were **removed** from +psx-runtime. `runtime/src/debug_server.c` still registers them, but only as +handlers that return an error explaining the migration — pause-step-read +synthesizes a snapshot ("what is state right NOW") instead of reading the +history the runtime already records continuously, and in this codebase it forced +the runtime into a wait loop where a dropped client looked like a freeze. + +The replacement is better for chasing a render bug. The GP0 ring holds +`1 << 20` packets (`gpu.c`, `GP0_RING_CAP`) — several hundred frames — so the +workflow runs the other way round: + +> Play until the bug is on screen. **Then** capture that frame, and the frames +> leading up to it. + +Reaching backwards beats freezing forwards, and it is the only thing that works +at all for a glitch you cannot reliably stop on. + +```bash +python3 $P/gpu_frame_capture.py --ring # what can I still ask for? +# GP0 ring: 812344 packet(s) seen, capacity 1048576, 12 words/packet +# capturable frames: 40918..41230 +``` + +`gpu_frame_dump` on a frame that has been evicted returns an **empty** dump, not +an error — which reads exactly like "that frame drew nothing", a conclusion you +might act on. So `capture()` checks the span first and refuses, and Studio's +Frames tab shows the span next to the frame selector and marks an out-of-ring +frame in red. + +What replaces each removed command: + +| Gone | Use instead | +|---|---| +| `run_to_frame N` | `--frame N` against the ring (any frame it still holds) | +| `step` | `--frames N`, walking backwards from `--frame` | +| `pause` + read state | `frame_range` / `get_frame` (per-frame records), `fn_entry_dump`, `wtrace_dump` | +| `pause` + inspect drawing | `gpu_frame_dump frame=N` — what this whole toolchain does | + +Savestates are unaffected: `savestate` is a real handler, and a savestate plus a +fixed frame number still makes a run reproducible. + +--- + ## Transport **One request per connection.** `io_thread_main()` in `debug_server.c` accepts, @@ -134,10 +179,10 @@ from a hung emulator. `psx_gpu_frame.DebugConn` opens a socket per command; cd P=psxrecomp/tools -# 1. Put the game on the frame you want, then capture it twice: once while it -# still looks right, once after it breaks. -python3 $P/gpu_frame_capture.py --tag good --out analysis/frames --summary -python3 $P/gpu_frame_capture.py --tag bad --out analysis/frames --summary +# 1. Play until the bug is on screen. Capture the newest frame as the suspect, +# then reach BACK into the ring for one that still looked right. +python3 $P/gpu_frame_capture.py --tag bad --out analysis/frames --summary +python3 $P/gpu_frame_capture.py --frame 41100 --tag good --out analysis/frames --summary # 2. What changed? python3 $P/gpu_frame_diff.py analysis/frames/good.json analysis/frames/bad.json @@ -149,8 +194,14 @@ python3 $P/gpu_frame_layers.py analysis/frames/bad.json --out analysis/frames/ba python3 $P/gpu_parity.py --frame 41230 --out analysis/frames/parity ``` -`--run-to N` drives the runtime to a frame before capturing, so a savestate plus -a fixed frame number makes the whole thing reproducible. +`--frames N` also grabs the N-1 frames *before* `--frame`, walking backwards +through the ring — the frames leading up to the one you noticed are usually the +ones that explain it. A savestate plus a fixed frame number still makes the whole +thing reproducible. + +A screenshot is only written for the live frame: the framebuffer holds the +present, not the historical frame you just dumped, and saving it beside an older +dump would label it as something it is not. ### Artifacts @@ -216,7 +267,10 @@ Studio's Frames tab has a **Save name** box that calls exactly this, so ## Parity scope -`gpu_parity.py` compares **images**, not GP0 streams. The DuckStation oracle +`gpu_parity.py` compares **images**, not GP0 streams. Only DuckStation is +driven: it implements `run_to_frame`, psx-runtime does not, so the native side is +sampled where it already is and DuckStation is advanced to meet it. A residual +frame mismatch is reported rather than papered over. The DuckStation oracle patch (`tools/duckstation/psxrecomp_oracle.patch`) implements `screenshot`, `read_vram`, `gpu_state`, `run_to_frame`, `step` and `pause` — but not `gpu_frame_dump`, so there is no packet stream to compare on that side. Adding @@ -224,6 +278,13 @@ one to the patch is the obvious next step if image parity keeps saying "the streams must differ" without saying where; until then, `tools/cosim.py` brackets the divergence. +Getting an oracle on Linux: `python3 tools/duckstation_oracle.py all` builds and +installs a patched DuckStation into `~/.local/share/retcomm/oracle/duckstation/` +— outside any game repo, shared by every title — and `start --disc ` runs it +headless on 4371. On Arch-family and NixOS hosts the build runs inside +`ubuntu:22.04` via podman, because upstream's CMake refuses those hosts outright +and its build scripts may not be patched. See `tools/duckstation/README.md`. + Both instances must be driven identically — same disc, same starting state. Parity between two runs that were not driven the same way means nothing. diff --git a/tools/duckstation/Containerfile b/tools/duckstation/Containerfile new file mode 100644 index 00000000..f428eb88 --- /dev/null +++ b/tools/duckstation/Containerfile @@ -0,0 +1,39 @@ +# Builder image for the DuckStation oracle. +# +# Why a container at all: DuckStation's CMake refuses to configure on +# Arch-family and NixOS hosts (CMakeModules/DuckStationBuildSummary.cmake), and +# upstream explicitly forbids distributing patches that modify its build system. +# So instead of working around the check, we build somewhere it genuinely +# supports -- ubuntu:22.04, the exact image its own Linux CI uses +# (.github/workflows/linux-appimage-build.yml). +# +# Ubuntu 22.04's glibc is older than any modern host's, so the binary produced +# here runs natively on the host afterwards. That is the same property upstream +# relies on to ship one AppImage for every distro. +# +# The package list is upstream's own scripts/appimage/install-packages.sh, +# copied verbatim out of the pinned checkout by duckstation_oracle.py. It is not +# reproduced or modified here -- it is the authority on what the build needs. + +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive +# linuxdeploy and appimagetool ship as AppImages and want FUSE, which a +# container generally cannot give them. This is the AppImage runtime's own +# supported escape hatch, not a modification of upstream's packaging script. +ENV APPIMAGE_EXTRACT_AND_RUN=1 + +# What upstream's install-packages.sh assumes is already present: it calls sudo +# and apt-add-repository, neither of which a bare ubuntu image has. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg sudo software-properties-common \ + git xz-utils file desktop-file-utils \ + && rm -rf /var/lib/apt/lists/* + +COPY install-packages.sh /tmp/install-packages.sh +RUN bash /tmp/install-packages.sh \ + && rm -f /tmp/install-packages.sh \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /oracle diff --git a/tools/duckstation/README.md b/tools/duckstation/README.md index 697dd041..e8a3798c 100644 --- a/tools/duckstation/README.md +++ b/tools/duckstation/README.md @@ -10,7 +10,86 @@ PSXRecomp v4 uses a patched build of [stenzek/duckstation](https://github.com/st - **`tools/duckstation/build.sh`** — runs CMake (Visual Studio 17 2022, x64, Release) and MSBuild on `duckstation-qt`. Requires the Visual Studio 2022 "Desktop development with C++" workload (CMake + MSBuild come with it). - **`tools/fix_duckstation_deps_paths.py`** — helper used by `setup.sh` to rewrite stale `_IMPORT_PREFIX` values in the extracted prebuilt deps. -## First-time setup +## First-time setup — Linux / macOS + +Use `tools/duckstation_oracle.py`. It installs **outside any game repo**, into +the shared RetComM data root, so one build serves every title: + +``` +~/.local/share/retcomm/oracle/duckstation/ ($RETCOMM_DATA_DIR, or $RETCOMM_ORACLE_DIR) + src/ pinned upstream checkout with the oracle patch applied + build/ cmake/ninja tree + app/ the portable install that actually gets run (~244 MB, self-contained) + oracle.json +``` + +```bash +P=psxrecomp/tools +python3 $P/duckstation_oracle.py doctor # can this machine build it? +python3 $P/duckstation_oracle.py all # fetch + patch + build + install +python3 $P/duckstation_oracle.py start --disc disc/game.cue +python3 $P/duckstation_oracle.py status +python3 $P/duckstation_oracle.py stop +``` + +`gpu_parity.py --start-oracle --disc ` will do the `start` step for you +when nothing is answering on 4371. + +### Arch, CachyOS, NixOS: the build runs in a container + +DuckStation's CMake **refuses to configure** on Arch-family and NixOS hosts +(`CMakeModules/DuckStationBuildSummary.cmake` → `FATAL_ERROR "Unsupported +environment."`), and the note above that check states you do not have permission +to distribute patches modifying its build system. Its build scripts are +CC-BY-NC-ND-4.0. + +So the tool does not patch the check out. It detects the refusal up front and +builds in an environment upstream *does* support: `ubuntu:22.04` via +podman/docker — the exact image its own CI uses +(`.github/workflows/linux-appimage-build.yml`) — installing packages with +upstream's own `scripts/appimage/install-packages.sh`, copied verbatim out of +the pinned checkout. Ubuntu 22.04's glibc is older than any modern host's, so +the binary produced there runs natively on the host afterwards; that is the same +property upstream relies on to ship one AppImage for every distro. + +On a distro upstream accepts, the build runs directly on the host. Force either +way with `build --container` / `build --no-container`. + +### Notes that cost an afternoon to find + +* **The binary's RUNPATH points at the build environment**, and Qt needs its + plugin tree. `install` copies `dep/prebuilt//{lib,plugins}` next to + the binary and writes a `run-oracle` launcher that sets `LD_LIBRARY_PATH` and + `QT_PLUGIN_PATH`, then verifies with `ldd` that nothing is unresolved. +* **An unofficial build shows a modal warning dialog before the core thread + starts.** Under `-nogui` it is invisible, so the process sits at one thread, + zero CPU, blocked in `ppoll`, with no port and no error. The fix is the key + the dialog itself writes — `[UI] UnofficialBuildWarningConfirmed = true` — + which `install` sets. It is a setting, not a patch. (Personal builds are + explicitly fine under that licence; redistributing one is not, which is why + this install stays on the machine that built it.) +* **`install` seeds DuckStation's own default `settings.ini`** by running it + briefly, then merges only the handful of keys an oracle needs. The emulator + owns that schema; hand-writing the whole file means guessing at it. +* **Wayland support is off by default** (`-DENABLE_WAYLAND=OFF`). It is the only + dependency the prebuilt bundle does not cover (needs `extra-cmake-modules`), + and an offscreen oracle never opens a Wayland surface. `build --wayland` + re-enables it. +* **The oracle server only starts when a system boots.** The patch calls + `PSXRecompDebug::Initialize(4371)` from `System::BootSystem`, so an idle + DuckStation with nothing loaded will never open the port. + +### The pin + +`tools/duckstation/pin.json` is the single source of truth for the upstream +base. Both this tool and the Windows `setup.sh` read it, so the two platforms +cannot drift onto different trees — the oracle patch only applies to that one. +The prebuilt-dependency version and its SHA-256 are **not** pinned here; they +are read from the checkout's own `dep/PREBUILT-VERSION` and +`dep/PREBUILT-SHA256SUMS`, so bumping the base cannot silently pair a new source +tree with an old dependency set. + +## First-time setup — Windows ```bash cd diff --git a/tools/duckstation/pin.json b/tools/duckstation/pin.json new file mode 100644 index 00000000..76e459ec --- /dev/null +++ b/tools/duckstation/pin.json @@ -0,0 +1,13 @@ +{ + "_comment": [ + "Single source of truth for the DuckStation oracle pin.", + "setup.sh (Windows) and duckstation_oracle.py (Linux/macOS) both read this,", + "so the two platforms can never drift onto different upstream bases.", + "Bump upstream_base only together with a regenerated psxrecomp_oracle.patch:", + " cd && git diff > tools/duckstation/psxrecomp_oracle.patch" + ], + "upstream_url": "https://github.com/stenzek/duckstation.git", + "upstream_base": "ffb33c281d196eb8ee0f559085ca285de7cdd51b", + "patch": "psxrecomp_oracle.patch", + "oracle_port": 4371 +} diff --git a/tools/duckstation/setup.sh b/tools/duckstation/setup.sh index 9ea359df..f298fdc5 100644 --- a/tools/duckstation/setup.sh +++ b/tools/duckstation/setup.sh @@ -16,7 +16,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" DUCK="$REPO_ROOT/duckstation" PATCH="$REPO_ROOT/tools/duckstation/psxrecomp_oracle.patch" -UPSTREAM_BASE="ffb33c281d196eb8ee0f559085ca285de7cdd51b" # stenzek/master at time of daa1808 +# The pin lives in pin.json so this script and duckstation_oracle.py (the +# Linux/macOS path) can never drift onto different upstream bases. +UPSTREAM_BASE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["upstream_base"])' "$REPO_ROOT/tools/duckstation/pin.json")" log() { echo "[duckstation-setup] $*"; } diff --git a/tools/duckstation_oracle.py b/tools/duckstation_oracle.py new file mode 100644 index 00000000..ba172102 --- /dev/null +++ b/tools/duckstation_oracle.py @@ -0,0 +1,1167 @@ +#!/usr/bin/env python3 +"""duckstation_oracle.py -- set up, build, and run the patched DuckStation oracle. + + python3 duckstation_oracle.py doctor # can this machine build it? + python3 duckstation_oracle.py all # fetch + patch + build + install + python3 duckstation_oracle.py start --disc disc/game.cue + python3 duckstation_oracle.py status + +Why an oracle +------------- +DuckStation, patched to speak the same JSON-over-TCP debug protocol as +psx-runtime (on port 4371 instead of 4370), answers the first question any +render bug has to answer: did the guest code run the SAME on a known-good +emulator? Identical images mean the recompilation is faithful and the bug is in +our renderer; different images mean the guest diverged and tools/cosim.py can +bracket where. + +Where it lives +-------------- +NOT in the game repo. The oracle is a developer tool shared by every recomp +title, so it installs into the RetComM data root alongside the toolchains and +catalog: + + ~/.local/share/retcomm/oracle/duckstation/ (Linux / macOS) + %LOCALAPPDATA%\\retcomm\\oracle\\duckstation\\ (Windows) + +Override with RETCOMM_ORACLE_DIR, or move the whole root with RETCOMM_DATA_DIR +(the same variables RetComM Launcher and Studio already honour). + + /src/ pinned upstream checkout with the oracle patch applied + /build/ cmake/ninja tree + /app/ the portable install that actually gets run + /oracle.json manifest: upstream base, patch hash, build stamp + +How it is built +--------------- +Exactly the way upstream's own Linux CI builds it +(.github/workflows/linux-appimage-build.yml): system packages for the platform +libraries, plus the prebuilt dependency bundle from +github.com/duckstation/dependencies -- which carries Qt6, SDL3 and the rest, so +nothing here compiles Qt from source. The bundle version and its SHA-256 are +read out of the checkout's own dep/PREBUILT-VERSION and dep/PREBUILT-SHA256SUMS +rather than pinned here, so a base bump cannot silently pair a new source tree +with an old dependency set. + +The upstream base itself is pinned in tools/duckstation/pin.json, shared with +the Windows setup.sh, because the oracle patch only applies to that tree. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import socket +import subprocess +import sys +import tarfile +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +HERE = Path(__file__).resolve().parent +PIN_PATH = HERE / "duckstation" / "pin.json" +PATCH_DIR = HERE / "duckstation" + +# Platform libraries upstream's install-packages.sh installs on Ubuntu, mapped +# to what a check can actually see. pkg-config names where one exists; the rest +# are checked as headers because several ship no .pc file. +PKGCONFIG_REQUIRED = [ + ("libcurl", "curl development headers"), + ("openssl", "OpenSSL development headers"), + ("dbus-1", "D-Bus development headers"), + ("alsa", "ALSA development headers"), + ("libudev", "udev development headers"), + ("libevdev", "libevdev development headers"), + ("x11", "libX11 development headers"), + ("xcb", "libxcb development headers"), + ("wayland-client", "Wayland development headers"), + ("freetype2", "FreeType development headers"), + ("fontconfig", "fontconfig development headers"), + ("harfbuzz", "HarfBuzz development headers"), + ("egl", "EGL development headers"), + ("zlib", "zlib development headers"), +] + +# Distro hints. Not exhaustive package sets — enough to unblock, with a pointer +# at upstream's own list, which is the authority. +DISTRO_HINTS = { + "arch": ( + "sudo pacman -S --needed base-devel cmake ninja clang lld nasm patchelf " + "curl openssl dbus alsa-lib libevdev libinput systemd-libs libx11 libxcb " + "xcb-util-cursor xcb-util-image xcb-util-keysyms xcb-util-renderutil " + "xcb-util-wm libxkbcommon-x11 libxrandr libxss wayland libdecor " + "libpipewire libpulse freetype2 fontconfig harfbuzz mesa libva zlib gtk3" + ), + "debian": ( + "sudo bash /scripts/appimage/install-packages.sh " + "# upstream's own list (Ubuntu/Debian)" + ), +} + + +class OracleError(RuntimeError): + pass + + +# Mirror of CMakeModules/DuckStationBuildSummary.cmake's environment check, so +# the refusal is predicted here with an explanation instead of surfacing 20 +# seconds into a cmake run as "Unsupported environment." +# +# Upstream refuses to configure on Arch-family and NixOS hosts, and its build +# scripts are CC-BY-NC-ND-4.0 with an explicit note that you may not distribute +# patches modifying the build system. We do not patch it. We build in an +# environment it supports instead -- which is what --container does. +def refused_environment() -> Optional[str]: + osr = Path("/etc/os-release") + if osr.is_file(): + try: + text = osr.read_text() + except OSError: + text = "" + for needle in ("ID=arch", "ID_LIKE=arch", "ID=nixos"): + if needle in text: + return f"/etc/os-release matches {needle}" + for var in ("NIX_BUILD_TOP", "NIX_STORE", "IN_NIX_SHELL"): + if os.environ.get(var): + return f"${var} is set" + if Path("/etc/NIXOS").exists(): + return "/etc/NIXOS exists" + return None + + +def container_engine() -> Optional[str]: + for engine in ("podman", "docker"): + if which(engine): + return engine + return None + + +def image_tag(pin: Dict[str, Any]) -> str: + return f"localhost/retcomm/duckstation-oracle:{pin['upstream_base'][:12]}" + + +def image_exists(engine: str, tag: str) -> bool: + rc, _ = capture([engine, "image", "exists", tag]) + return rc == 0 + + +def log(msg: str) -> None: + print(f"[oracle] {msg}", flush=True) + + +# --------------------------------------------------------------------------- +# Locations +# --------------------------------------------------------------------------- + +def data_root() -> Path: + """The RetComM shared data root — same one the launcher and Studio use.""" + env = os.environ.get("RETCOMM_DATA_DIR") + if env: + return Path(env).expanduser() + if sys.platform == "win32": + base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local") + return Path(base) / "retcomm" + return Path.home() / ".local" / "share" / "retcomm" + + +def oracle_root() -> Path: + env = os.environ.get("RETCOMM_ORACLE_DIR") + if env: + return Path(env).expanduser() + return data_root() / "oracle" / "duckstation" + + +def download_cache() -> Path: + return data_root() / "downloads" + + +def exe_name() -> str: + return "duckstation-qt.exe" if sys.platform == "win32" else "duckstation-qt" + + +class Layout: + def __init__(self, root: Optional[Path] = None): + self.root = root or oracle_root() + self.src = self.root / "src" + self.build = self.root / "build" + self.app = self.root / "app" + self.manifest = self.root / "oracle.json" + self.pidfile = self.root / "oracle.pid" + self.logfile = self.root / "oracle.log" + + @property + def binary(self) -> Path: + return self.app / exe_name() + + @property + def launcher(self) -> Path: + return self.app / ("run-oracle.cmd" if sys.platform == "win32" + else "run-oracle") + + @property + def built_binary(self) -> Path: + return self.build / "bin" / exe_name() + + +def load_pin() -> Dict[str, Any]: + if not PIN_PATH.is_file(): + raise OracleError(f"missing pin file: {PIN_PATH}") + with open(PIN_PATH, "r", encoding="utf-8") as f: + return json.load(f) + + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: List[str], cwd: Optional[Path] = None, env: Optional[Dict[str, str]] = None, + check: bool = True, quiet: bool = False) -> int: + if not quiet: + log("$ " + " ".join(str(c) for c in cmd)) + proc = subprocess.run([str(c) for c in cmd], cwd=str(cwd) if cwd else None, + env=env, check=False) + if check and proc.returncode != 0: + raise OracleError(f"command failed ({proc.returncode}): {' '.join(map(str, cmd))}") + return proc.returncode + + +def capture(cmd: List[str], cwd: Optional[Path] = None) -> Tuple[int, str]: + proc = subprocess.run([str(c) for c in cmd], cwd=str(cwd) if cwd else None, + capture_output=True, text=True) + return proc.returncode, (proc.stdout + proc.stderr).strip() + + +def which(name: str) -> Optional[str]: + return shutil.which(name) + + +def distro_id() -> str: + try: + for line in Path("/etc/os-release").read_text().splitlines(): + if line.startswith("ID_LIKE="): + return line.split("=", 1)[1].strip().strip('"').split()[0] + if line.startswith("ID="): + got = line.split("=", 1)[1].strip().strip('"') + except OSError: + return "" + return got if "got" in dir() else "" + + +# --------------------------------------------------------------------------- +# doctor +# --------------------------------------------------------------------------- + +def pick_compilers(cc: Optional[str], cxx: Optional[str]) -> Tuple[str, str, Optional[str]]: + """Choose a C/C++ compiler and a linker. + + Upstream CI pins clang-19 for reproducibility. We take whatever clang the + host has, because pinning a compiler we cannot install is worse than using a + newer one; gcc is the fallback. A newer clang than CI's occasionally trips a + diagnostic upstream has not seen, which is why --cc/--cxx exist. + """ + if cc and cxx: + return cc, cxx, ("lld" if which("ld.lld") else None) + for c, x in (("clang", "clang++"), ("gcc", "g++"), ("cc", "c++")): + if which(c) and which(x): + linker = "lld" if (c == "clang" and which("ld.lld")) else None + return c, x, linker + raise OracleError("no C/C++ compiler found (need clang or gcc)") + + +def doctor(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + ok = True + print(f"install root : {lay.root}") + print(f"data root : {data_root()}") + print(f"platform : {platform.system()} {platform.machine()}") + print() + + print("required tools") + for tool, why in (("git", "fetch the pinned upstream tree"), + ("cmake", "configure the build"), + ("ninja", "compile"), + ("curl", "download the prebuilt dependency bundle"), + ("tar", "extract it")): + path = which(tool) + print(f" {'ok ' if path else 'MISS'} {tool:<8} {path or '(not on PATH)':<40} {why}") + ok = ok and bool(path) + + try: + cc, cxx, linker = pick_compilers(args.cc, args.cxx) + print(f" ok compiler {cc} / {cxx}" + (f" (-fuse-ld={linker})" if linker else "")) + except OracleError as e: + print(f" MISS compiler {e}") + ok = False + + if which("cmake"): + rc, out = capture(["cmake", "--version"]) + print(f" {out.splitlines()[0] if out else ''}") + + print() + if sys.platform.startswith("linux"): + print("platform libraries (pkg-config)") + missing = [] + have_pc = bool(which("pkg-config")) + if not have_pc: + print(" MISS pkg-config — cannot check the platform libraries") + ok = False + else: + for pc, why in PKGCONFIG_REQUIRED: + rc, _ = capture(["pkg-config", "--exists", pc]) + good = rc == 0 + if not good: + missing.append(pc) + print(f" {'ok ' if good else 'MISS'} {pc:<16} {why}") + if missing: + ok = False + hint = DISTRO_HINTS.get(distro_id() or "", "") + print() + print(f" missing: {', '.join(missing)}") + if hint: + print(f" try: {hint}") + print(" upstream's authoritative list: " + "/scripts/appimage/install-packages.sh") + print() + + if sys.platform.startswith("linux"): + ecm = any(Path(c).is_file() for c in ( + "/usr/share/ECM/cmake/ECMConfig.cmake", + "/usr/lib/cmake/ECM/ECMConfig.cmake", + "/usr/lib64/cmake/ECM/ECMConfig.cmake")) + print("optional") + print(f" {'ok ' if ecm else '-- '} extra-cmake-modules " + f"{'found' if ecm else 'absent'}") + print(" Only needed for `build --wayland`. The default build " + "passes -DENABLE_WAYLAND=OFF,") + print(" because a headless offscreen oracle never opens a " + "Wayland surface.") + print() + + st = state(lay) + print("install state") + for k, v in st.items(): + print(f" {k:<12} {v}") + print() + print("READY" if ok else "NOT READY — install what is marked MISS above") + return 0 if ok else 1 + + +# --------------------------------------------------------------------------- +# setup: fetch source, deps, patch +# --------------------------------------------------------------------------- + +def git_repo_ok(path: Path) -> bool: + return (path / ".git").exists() + + +def fetch_source(lay: Layout, pin: Dict[str, Any], force: bool = False) -> None: + url = pin["upstream_url"] + base = pin["upstream_base"] + lay.root.mkdir(parents=True, exist_ok=True) + + if not git_repo_ok(lay.src): + log(f"cloning {url} (blobless) -> {lay.src}") + lay.src.parent.mkdir(parents=True, exist_ok=True) + run(["git", "clone", "--filter=blob:none", "--no-checkout", url, str(lay.src)]) + + rc, head = capture(["git", "rev-parse", "HEAD"], cwd=lay.src) + if rc == 0 and head == base and not force: + log(f"source already at pinned base {base[:12]}") + else: + rc, _ = capture(["git", "cat-file", "-t", base], cwd=lay.src) + if rc != 0: + log("fetching pinned base...") + if run(["git", "fetch", "origin", base], cwd=lay.src, check=False) != 0: + run(["git", "fetch", "--all"], cwd=lay.src) + log(f"checking out pinned base {base[:12]}") + # A dirty tree here is our own patch from a previous run; reset so the + # patch application below is always from a known state. + run(["git", "checkout", "--force", "--detach", base], cwd=lay.src) + run(["git", "submodule", "update", "--init", "--recursive"], cwd=lay.src, check=False) + + +def deps_platform() -> Tuple[str, str]: + """(archive name, extracted directory name) for this host.""" + mach = platform.machine().lower() + if sys.platform == "win32": + return ("deps-windows-x64.7z", "windows-x64") + if sys.platform == "darwin": + return ("deps-macos-universal.tar.xz", "macos-universal") + if mach in ("x86_64", "amd64"): + return ("deps-linux-x64.tar.xz", "linux-x64") + raise OracleError( + f"no prebuilt dependency bundle for {sys.platform}/{mach}. Upstream ships " + f"x64 and cross-built arm64/armhf only; a native build here would need " + f"the dependencies built from source.") + + +def expected_sha(src: Path, archive: str) -> Optional[str]: + sums = src / "dep" / "PREBUILT-SHA256SUMS" + if not sums.is_file(): + return None + for line in sums.read_text().splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].lstrip("*") == archive: + return parts[0] + return None + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fetch_deps(lay: Layout, force: bool = False) -> None: + archive, dirname = deps_platform() + version_file = lay.src / "dep" / "PREBUILT-VERSION" + if not version_file.is_file(): + raise OracleError(f"missing {version_file} — is the source checkout complete?") + version = version_file.read_text().strip() + want = expected_sha(lay.src, archive) + + dest_dir = lay.src / "dep" / "prebuilt" / dirname + if dest_dir.is_dir() and any(dest_dir.iterdir()) and not force: + log(f"prebuilt deps already extracted ({dirname}, {version})") + return + + cache = download_cache() + cache.mkdir(parents=True, exist_ok=True) + local = cache / f"{version}-{archive}" + + if local.is_file() and want: + got = sha256_file(local) + if got != want: + log("cached bundle failed its checksum; re-downloading") + local.unlink() + + if not local.is_file(): + url = (f"https://github.com/duckstation/dependencies/releases/download/" + f"{version}/{archive}") + log(f"downloading {archive} ({version})") + tmp = local.with_suffix(local.suffix + ".part") + run(["curl", "-L", "--fail", "--retry", "5", "--retry-all-errors", + "-o", str(tmp), url]) + tmp.replace(local) + + if want: + got = sha256_file(local) + if got != want: + raise OracleError( + f"dependency bundle checksum mismatch for {archive}\n" + f" expected {want}\n actual {got}\n" + f" (expected value comes from the checkout's own " + f"dep/PREBUILT-SHA256SUMS)") + log(f"checksum ok ({want[:16]}…)") + else: + log("WARNING: no PREBUILT-SHA256SUMS entry — bundle not verified") + + target = lay.src / "dep" / "prebuilt" + target.mkdir(parents=True, exist_ok=True) + log(f"extracting into {target}") + if archive.endswith(".7z"): + seven = which("7z") or which("7za") + if not seven: + raise OracleError("7z not found — needed to extract the Windows bundle") + run([seven, "x", "-y", str(local)], cwd=target) + else: + with tarfile.open(local) as tf: + # Python 3.12+ warns without a filter; 'data' refuses absolute paths + # and traversal, which is what we want for a downloaded archive. + try: + tf.extractall(target, filter="data") + except TypeError: + tf.extractall(target) + if not dest_dir.is_dir(): + raise OracleError(f"extraction did not produce {dest_dir}") + + +def apply_patch(lay: Layout, pin: Dict[str, Any]) -> None: + patch = PATCH_DIR / pin["patch"] + if not patch.is_file(): + raise OracleError(f"missing oracle patch: {patch}") + rc, _ = capture(["git", "apply", "--reverse", "--check", str(patch)], cwd=lay.src) + if rc == 0: + log("oracle patch already applied") + return + rc, out = capture(["git", "apply", "--check", str(patch)], cwd=lay.src) + if rc != 0: + raise OracleError( + "the oracle patch does not apply to this tree and is not already " + "applied. Either the pinned base moved, or the patch was regenerated " + f"against a different one.\n patch: {patch}\n git says: {out}") + log("applying oracle patch") + run(["git", "apply", str(patch)], cwd=lay.src) + + +def cmd_setup(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + pin = load_pin() + fetch_source(lay, pin, force=args.force) + fetch_deps(lay, force=args.force) + apply_patch(lay, pin) + write_manifest(lay, pin, stage="setup") + log(f"setup complete: {lay.src}") + return 0 + + +# --------------------------------------------------------------------------- +# builder image +# --------------------------------------------------------------------------- + +def build_image(lay: Layout, pin: Dict[str, Any], engine: str, force: bool = False) -> str: + tag = image_tag(pin) + if image_exists(engine, tag) and not force: + log(f"builder image present: {tag}") + return tag + pkgs = lay.src / "scripts" / "appimage" / "install-packages.sh" + if not pkgs.is_file(): + raise OracleError(f"missing {pkgs} — run `setup` first") + ctx = lay.root / "builder" + if ctx.exists(): + shutil.rmtree(ctx) + ctx.mkdir(parents=True) + # The context carries upstream's own package list, copied verbatim out of + # the pinned checkout so the image can never install a different set from + # the one that tree expects. + shutil.copy2(pkgs, ctx / "install-packages.sh") + shutil.copy2(PATCH_DIR / "Containerfile", ctx / "Containerfile") + log(f"building {tag} (ubuntu:22.04 + upstream's package list) — first time is slow") + run([engine, "build", "-t", tag, "-f", str(ctx / "Containerfile"), str(ctx)]) + return tag + + +def in_container(engine: str, tag: str, lay: Layout, script: str, + env: Optional[Dict[str, str]] = None) -> None: + cmd = [engine, "run", "--rm", + "-v", f"{lay.root}:/oracle", + "-w", "/oracle"] + for k, v in (env or {}).items(): + cmd += ["-e", f"{k}={v}"] + cmd += [tag, "bash", "-euo", "pipefail", "-c", script] + run(cmd) + + +def cmd_image(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + pin = load_pin() + engine = container_engine() + if not engine: + raise OracleError("no podman or docker found") + build_image(lay, pin, engine, force=args.force) + return 0 + + +# --------------------------------------------------------------------------- +# build +# --------------------------------------------------------------------------- + +def cmake_args(lay: Layout, args: argparse.Namespace, cc: str, cxx: str, + linker: Optional[str], src: str, build: str) -> List[str]: + cfg = ["cmake", "-S", src, "-B", build, "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_C_COMPILER={cc}", f"-DCMAKE_CXX_COMPILER={cxx}"] + if linker: + for v in ("EXE", "MODULE", "SHARED"): + cfg.append(f"-DCMAKE_{v}_LINKER_FLAGS_INIT=-fuse-ld={linker}") + if not args.wayland: + # A headless offscreen oracle never opens a Wayland surface, and turning + # it off drops the only dependency the prebuilt bundle does not cover + # (extra-cmake-modules, for ECM/FindWayland). + cfg.append("-DENABLE_WAYLAND=OFF") + if args.lto: + # Upstream CI enables IPO for shipping builds. An oracle does not need + # it and it roughly doubles the build, so it is opt-in here. + cfg.append("-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON") + cfg.extend(args.cmake_arg or []) + return cfg + + +def cmd_build(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + pin = load_pin() + if not (lay.src / "CMakeLists.txt").is_file(): + raise OracleError(f"no source at {lay.src} — run `setup` first") + + if args.reconfigure and lay.build.exists(): + log("removing existing build tree (--reconfigure)") + shutil.rmtree(lay.build) + + refused = refused_environment() + use_container = args.container if args.container is not None else bool(refused) + + if use_container: + engine = container_engine() + if not engine: + raise OracleError( + f"this host cannot build DuckStation directly ({refused}) and no " + f"podman/docker is installed to build it somewhere supported.\n" + f" Upstream refuses Arch-family and NixOS hosts in " + f"CMakeModules/DuckStationBuildSummary.cmake, and its build " + f"scripts are CC-BY-NC-ND with an explicit note that patches to " + f"them may not be distributed — so we build in ubuntu:22.04, the " + f"image its own CI uses, rather than working around the check.\n" + f" Install podman, or pass --no-container if you know this host " + f"is supported.") + if refused: + log(f"host build is refused by upstream ({refused}); " + f"building in a container instead") + tag = build_image(lay, pin, engine, force=False) + # clang-19 + lld is what upstream's Linux CI uses; the image installs + # exactly that, so the container build does not guess at a compiler. + cfg = cmake_args(lay, args, "clang-19", "clang++-19", "lld", + "/oracle/src", "/oracle/build") + jobs = f"--parallel {args.jobs}" if args.jobs else "--parallel" + script = " && ".join([ + " ".join(f"'{c}'" for c in cfg), + f"cmake --build /oracle/build {jobs}", + ]) + in_container(engine, tag, lay, script) + else: + cc, cxx, linker = pick_compilers(args.cc, args.cxx) + run(cmake_args(lay, args, cc, cxx, linker, str(lay.src), str(lay.build))) + build = ["cmake", "--build", str(lay.build)] + build += ["--parallel", str(args.jobs)] if args.jobs else ["--parallel"] + run(build) + + if not lay.built_binary.is_file(): + raise OracleError(f"build finished but {lay.built_binary} is missing") + write_manifest(lay, pin, stage="build", container=use_container) + log(f"built: {lay.built_binary}") + return 0 + + +# --------------------------------------------------------------------------- +# install: a portable app dir that survives rebuilds +# --------------------------------------------------------------------------- + +def find_bios(explicit: Optional[str]) -> Optional[Path]: + """Locate a retail BIOS image for the oracle. + + Order: what the caller passed, then the RetComM BIOS root, then the engine + checkout this script lives in. Never downloads one — a BIOS dump is the + user's own property and is not redistributable. + """ + if explicit: + p = Path(explicit).expanduser() + return p if p.is_file() else None + index = data_root() / "bios-index.json" + if index.is_file(): + try: + with open(index, "r", encoding="utf-8") as f: + doc = json.load(f) + root = doc.get("bios_root") + for name in doc.get("files", []) or []: + cand = Path(root or "") / (name if isinstance(name, str) + else name.get("name", "")) + if cand.is_file(): + return cand + except (OSError, json.JSONDecodeError, TypeError): + pass + for cand in (HERE.parent / "bios" / "SCPH1001.BIN", + HERE.parent / "bios" / "scph1001.bin"): + if cand.is_file(): + return cand + return None + + +def merge_ini(path: Path, sections: Dict[str, Dict[str, str]]) -> None: + """Merge keys into a DuckStation settings.ini, preserving what is there. + + DuckStation writes its own defaults on first launch; we only override the + handful of fields a headless oracle needs. Writing the whole file ourselves + would mean guessing at a schema the emulator owns. + """ + import configparser + cp = configparser.ConfigParser(strict=False, interpolation=None) + cp.optionxform = str # DuckStation keys are case-sensitive + if path.is_file(): + try: + cp.read(path, encoding="utf-8") + except configparser.Error: + pass + for sect, keys in sections.items(): + if not cp.has_section(sect): + cp.add_section(sect) + for k, v in keys.items(): + cp.set(sect, k, v) + with open(path, "w", encoding="utf-8") as f: + cp.write(f, space_around_delimiters=True) + + +def bundle_dirs(lay: Layout) -> List[Path]: + """The prebuilt runtime pieces that have to travel with the binary. + + The binary's RUNPATH points at wherever it was linked — inside the build + container, that path does not exist on the host — and Qt additionally needs + its plugin tree. Copying both into the install makes `app/` self-contained + and independent of both the container and the source checkout, which is the + whole point of installing it outside the repo. + """ + try: + _, dirname = deps_platform() + except OracleError: + return [] + base = lay.src / "dep" / "prebuilt" / dirname + return [d for d in (base / "lib", base / "plugins") if d.is_dir()] + + +def write_launcher(lay: Layout) -> None: + if sys.platform == "win32": + lay.launcher.write_text( + "@echo off\r\nsetlocal\r\n" + "set \"QT_PLUGIN_PATH=%~dp0plugins\"\r\n" + "\"%~dp0duckstation-qt.exe\" %*\r\n") + return + lay.launcher.write_text( + "#!/bin/sh\n" + "# Self-contained launcher for the DuckStation oracle.\n" + "# The binary's RUNPATH refers to the build environment, so point the\n" + "# loader and Qt at the copies that live beside it here instead.\n" + 'APP="$(cd "$(dirname "$0")" && pwd)"\n' + 'LD_LIBRARY_PATH="$APP/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"\n' + 'QT_PLUGIN_PATH="$APP/plugins"\n' + "export LD_LIBRARY_PATH QT_PLUGIN_PATH\n" + 'exec "$APP/duckstation-qt" "$@"\n') + lay.launcher.chmod(0o755) + + +def verify_install(lay: Layout) -> List[str]: + """Report any shared library the installed binary still cannot resolve.""" + if not which("ldd") or not lay.binary.is_file(): + return [] + env = dict(os.environ) + env["LD_LIBRARY_PATH"] = str(lay.app / "lib") + os.pathsep + env.get("LD_LIBRARY_PATH", "") + proc = subprocess.run(["ldd", str(lay.binary)], capture_output=True, text=True, env=env) + return [ln.strip() for ln in proc.stdout.splitlines() if "not found" in ln] + + +def seed_settings(lay: Layout, timeout: float = 20.0) -> None: + """Let DuckStation write its own default settings.ini, then stop it. + + We only want to override a handful of fields, and the emulator owns that + schema — hand-writing a whole settings.ini means guessing at it. Everything + up to and including the config write happens before the startup blocks on + anything, so a short run followed by a kill produces a complete default file. + """ + if (lay.app / "settings.ini").is_file(): + return + env = dict(os.environ) + env["LD_LIBRARY_PATH"] = str(lay.app / "lib") + os.pathsep + env.get("LD_LIBRARY_PATH", "") + env["QT_PLUGIN_PATH"] = str(lay.app / "plugins") + if sys.platform.startswith("linux"): + env["QT_QPA_PLATFORM"] = "offscreen" + log("seeding DuckStation's own default settings.ini") + proc = subprocess.Popen([str(lay.binary), "-nogui"], cwd=str(lay.app), env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) + deadline = time.time() + timeout + while time.time() < deadline: + if (lay.app / "settings.ini").is_file(): + time.sleep(0.5) # let the write finish + break + if proc.poll() is not None: + break + time.sleep(0.25) + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + if not (lay.app / "settings.ini").is_file(): + log("WARNING: DuckStation did not write a settings.ini; writing a minimal one") + + +def cmd_install(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + pin = load_pin() + if not lay.built_binary.is_file(): + raise OracleError(f"nothing built at {lay.built_binary} — run `build` first") + + src_bin = lay.build / "bin" + if lay.app.exists(): + shutil.rmtree(lay.app) + log(f"staging portable install -> {lay.app}") + shutil.copytree(src_bin, lay.app, symlinks=True) + + for d in bundle_dirs(lay): + dest = lay.app / d.name + log(f"bundling {d.name}/ ({d})") + shutil.copytree(d, dest, symlinks=True, dirs_exist_ok=True) + + write_launcher(lay) + + # Portable mode keeps settings, memcards and saves inside the install + # instead of ~/.config/duckstation, so the oracle cannot inherit — or + # disturb — a real DuckStation the user set up for playing games. + (lay.app / "portable.txt").write_text("") + + bios = find_bios(args.bios) + if bios: + (lay.app / "bios").mkdir(exist_ok=True) + shutil.copy2(bios, lay.app / "bios" / bios.name) + log(f"bios: {bios} -> app/bios/{bios.name}") + else: + log("WARNING: no BIOS image found. Pass --bios , or the oracle " + "cannot boot a disc. (BIOS dumps are yours to supply.)") + + seed_settings(lay) + + settings = lay.app / "settings.ini" + bios_name = bios.name if bios else "" + merge_ini(settings, { + "Main": { + "SettingsVersion": "3", + "SetupWizardIncomplete": "false", + "ConfirmPowerOff": "false", + "PauseOnFocusLoss": "false", + "StartPaused": "false", + }, + "BIOS": { + "SearchDirectory": "bios", + "PathNTSC-U": bios_name, + "PathNTSC-J": bios_name, + "PathPAL": bios_name, + "PatchFastBoot": "true", + }, + "GPU": { + # Software is the right renderer for an oracle: deterministic, no + # GPU context needed headless, and the debug server reads g_vram + # directly so there is nothing to read back from a real surface. + "Renderer": "Software", + }, + "Audio": {"Backend": "Null"}, + "UI": { + # This is the "don't show again" of the unofficial-build warning + # (AutoUpdaterDialog::warnAboutUnofficialBuild). It is a MODAL + # dialog, and under -nogui it is invisible, so an unofficial build + # blocks forever in the event loop before the core thread ever + # starts: one thread, no CPU, no port, no explanation. Setting the + # key the dialog itself writes is the supported way past it — we do + # not patch the check out. (DuckStation is CC-BY-NC-ND: building for + # personal use is fine, redistributing a modified build is not, so + # this install stays on this machine.) + "UnofficialBuildWarningConfirmed": "true", + }, + }) + log(f"settings: {settings}") + + missing = verify_install(lay) + if missing: + log("WARNING: the installed binary still cannot resolve:") + for m in missing: + log(f" {m}") + log(" the oracle will not start until these are found") + else: + log("link check: every shared library resolves") + + write_manifest(lay, pin, stage="install", bios=str(bios) if bios else None) + log(f"installed: {lay.launcher}") + return 0 + + +# --------------------------------------------------------------------------- +# manifest / status +# --------------------------------------------------------------------------- + +def write_manifest(lay: Layout, pin: Dict[str, Any], stage: str, + bios: Optional[str] = None, + container: Optional[bool] = None) -> None: + doc: Dict[str, Any] = {} + if lay.manifest.is_file(): + try: + with open(lay.manifest, "r", encoding="utf-8") as f: + doc = json.load(f) + except (OSError, json.JSONDecodeError): + doc = {} + patch = PATCH_DIR / pin["patch"] + doc.update({ + "kind": "psxrecomp-duckstation-oracle", + "version": 1, + "upstream_url": pin["upstream_url"], + "upstream_base": pin["upstream_base"], + "patch_sha256": sha256_file(patch) if patch.is_file() else None, + "oracle_port": pin.get("oracle_port", 4371), + "root": str(lay.root), + f"{stage}_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + }) + if bios: + doc["bios"] = bios + if container is not None: + doc["built_in_container"] = container + lay.root.mkdir(parents=True, exist_ok=True) + with open(lay.manifest, "w", encoding="utf-8") as f: + json.dump(doc, f, indent=1) + + +def port_open(port: int, host: str = "127.0.0.1", timeout: float = 1.0) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def oracle_ping(port: int, timeout: float = 3.0) -> Optional[Dict[str, Any]]: + try: + s = socket.create_connection(("127.0.0.1", port), timeout=timeout) + except OSError: + return None + try: + s.sendall(b'{"id":1,"cmd":"ping"}\n') + buf = b"" + while b"\n" not in buf: + chunk = s.recv(65536) + if not chunk: + break + buf += chunk + return json.loads(buf.split(b"\n")[0].decode(errors="replace")) + except (OSError, json.JSONDecodeError): + return None + finally: + s.close() + + +def state(lay: Layout) -> Dict[str, str]: + pin = load_pin() + port = int(pin.get("oracle_port", 4371)) + out = { + "source": "present" if (lay.src / "CMakeLists.txt").is_file() else "absent", + "deps": "absent", + "patch": "unknown", + "built": "yes" if lay.built_binary.is_file() else "no", + "installed": "yes" if lay.binary.is_file() else "no", + "running": "no", + } + try: + _, dirname = deps_platform() + d = lay.src / "dep" / "prebuilt" / dirname + out["deps"] = "present" if d.is_dir() and any(d.iterdir()) else "absent" + except OracleError: + out["deps"] = "unsupported platform" + if out["source"] == "present": + patch = PATCH_DIR / pin["patch"] + rc, _ = capture(["git", "apply", "--reverse", "--check", str(patch)], cwd=lay.src) + out["patch"] = "applied" if rc == 0 else "not applied" + if port_open(port): + rep = oracle_ping(port) + out["running"] = (f"yes — answering on {port}" if rep and rep.get("ok") + else f"something on {port}, not answering as an oracle") + return out + + +def cmd_status(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + print(f"root: {lay.root}") + if lay.manifest.is_file(): + with open(lay.manifest, "r", encoding="utf-8") as f: + doc = json.load(f) + for k in ("upstream_base", "oracle_port", "bios", "setup_at", "build_at", + "install_at"): + if k in doc: + print(f" {k:<14} {doc[k]}") + for k, v in state(lay).items(): + print(f" {k:<14} {v}") + return 0 + + +def cmd_path(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + print(lay.binary if args.binary else lay.root) + return 0 + + +# --------------------------------------------------------------------------- +# run +# --------------------------------------------------------------------------- + +def cmd_start(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + pin = load_pin() + port = int(args.port or pin.get("oracle_port", 4371)) + if not lay.binary.is_file(): + raise OracleError(f"not installed at {lay.binary} — run `all` first") + if port_open(port): + rep = oracle_ping(port) + if rep and rep.get("ok"): + log(f"already running and answering on {port}") + return 0 + raise OracleError( + f"port {port} is in use by something that is not the oracle. Find it " + f"with: ss -ltnp | grep {port}") + + cmd = [str(lay.launcher if lay.launcher.is_file() else lay.binary)] + if args.disc: + disc = Path(args.disc).expanduser().resolve() + if not disc.is_file(): + raise OracleError(f"disc not found: {disc}") + cmd.append(str(disc)) + else: + cmd.append("-bios") + cmd += ["-nogui", "-fastboot"] + cmd += args.extra or [] + + env = dict(os.environ) + if args.headless and sys.platform.startswith("linux"): + # No display needed: the software renderer plus the debug server's + # direct g_vram reads mean nothing has to reach a real surface. + env.setdefault("QT_QPA_PLATFORM", "offscreen") + + log("$ " + " ".join(cmd)) + with open(lay.logfile, "ab") as logf: + logf.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} {' '.join(cmd)} ===\n" + .encode()) + proc = subprocess.Popen(cmd, cwd=str(lay.app), env=env, + stdout=logf, stderr=subprocess.STDOUT, + start_new_session=True) + lay.pidfile.write_text(str(proc.pid)) + log(f"pid {proc.pid}, log {lay.logfile}") + + deadline = time.time() + args.wait + while time.time() < deadline: + if proc.poll() is not None: + tail = lay.logfile.read_text(errors="replace").splitlines()[-15:] + raise OracleError("oracle exited during startup:\n " + "\n ".join(tail)) + rep = oracle_ping(port, timeout=1.0) + if rep and rep.get("ok"): + log(f"oracle answering on {port} (frame {rep.get('frame', '?')})") + return 0 + time.sleep(0.5) + log(f"WARNING: no answer on {port} after {args.wait}s — it may still be booting; " + f"check {lay.logfile}") + return 1 + + +def cmd_stop(args: argparse.Namespace) -> int: + lay = Layout(Path(args.root) if args.root else None) + if not lay.pidfile.is_file(): + log("no pidfile; nothing started by this tool") + return 0 + try: + pid = int(lay.pidfile.read_text().strip()) + except ValueError: + lay.pidfile.unlink(missing_ok=True) + return 0 + try: + os.kill(pid, 15) + log(f"sent SIGTERM to {pid}") + except ProcessLookupError: + log(f"pid {pid} was not running") + except OSError as e: + raise OracleError(f"could not stop {pid}: {e}") + lay.pidfile.unlink(missing_ok=True) + return 0 + + +def cmd_all(args: argparse.Namespace) -> int: + rc = cmd_setup(args) + if rc: + return rc + rc = cmd_build(args) + if rc: + return rc + return cmd_install(args) + + +# --------------------------------------------------------------------------- + +def main(argv=None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", default=None, + help="install root (default: $RETCOMM_ORACLE_DIR, else " + "/oracle/duckstation)") + sub = ap.add_subparsers(dest="cmd", required=True) + + def build_opts(p): + p.add_argument("--cc", default=None, help="C compiler (default: clang, else gcc)") + p.add_argument("--cxx", default=None, help="C++ compiler") + p.add_argument("--jobs", type=int, default=0) + p.add_argument("--lto", action="store_true", + help="enable IPO/LTO as upstream release CI does (slower)") + p.add_argument("--reconfigure", action="store_true", + help="delete the build tree first") + p.add_argument("--container", dest="container", action="store_true", + default=None, + help="build inside ubuntu:22.04 via podman/docker " + "(automatic on hosts upstream refuses)") + p.add_argument("--no-container", dest="container", action="store_false", + help="force a direct host build") + p.add_argument("--wayland", action="store_true", + help="build with Wayland support (needs extra-cmake-modules; " + "an offscreen oracle does not use it)") + p.add_argument("--cmake-arg", action="append", default=None, metavar="-DFOO=BAR") + + p = sub.add_parser("doctor", help="check host prerequisites") + p.add_argument("--cc", default=None) + p.add_argument("--cxx", default=None) + p.set_defaults(func=doctor) + + p = sub.add_parser("setup", help="fetch pinned source + deps, apply the oracle patch") + p.add_argument("--force", action="store_true", help="re-fetch even if present") + p.set_defaults(func=cmd_setup) + + p = sub.add_parser("image", help="build the ubuntu:22.04 builder image") + p.add_argument("--force", action="store_true", help="rebuild even if present") + p.set_defaults(func=cmd_image) + + p = sub.add_parser("build", help="configure + compile") + build_opts(p) + p.set_defaults(func=cmd_build) + + p = sub.add_parser("install", help="stage the portable app dir") + p.add_argument("--bios", default=None, help="path to a PSX BIOS image") + p.set_defaults(func=cmd_install) + + p = sub.add_parser("all", help="setup + build + install") + p.add_argument("--force", action="store_true") + p.add_argument("--bios", default=None) + build_opts(p) + p.set_defaults(func=cmd_all) + + p = sub.add_parser("start", help="launch the oracle headless") + p.add_argument("--disc", default=None, help="cue/bin/chd to boot") + p.add_argument("--port", type=int, default=0) + p.add_argument("--wait", type=float, default=45.0, help="seconds to wait for a ping") + p.add_argument("--headless", action="store_true", default=True) + p.add_argument("--windowed", dest="headless", action="store_false", + help="show a window (needs a display)") + p.add_argument("--extra", nargs=argparse.REMAINDER) + p.set_defaults(func=cmd_start) + + p = sub.add_parser("stop", help="stop an oracle started by this tool") + p.set_defaults(func=cmd_stop) + + p = sub.add_parser("status", help="where it is and what state it is in") + p.set_defaults(func=cmd_status) + + p = sub.add_parser("path", help="print the install root") + p.add_argument("--binary", action="store_true", help="print the executable instead") + p.set_defaults(func=cmd_path) + + args = ap.parse_args(argv) + for attr, default in (("cc", None), ("cxx", None), ("jobs", 0), ("lto", False), + ("reconfigure", False), ("cmake_arg", None), + ("force", False), ("bios", None), ("root", None), + ("wayland", False), ("container", None)): + if not hasattr(args, attr): + setattr(args, attr, default) + try: + return args.func(args) + except OracleError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/gpu_frame_capture.py b/tools/gpu_frame_capture.py index e788842c..7cd08303 100644 --- a/tools/gpu_frame_capture.py +++ b/tools/gpu_frame_capture.py @@ -2,18 +2,28 @@ """gpu_frame_capture.py -- capture one frame's GP0 stream + screenshot, decoded and attributed to the guest functions that issued it. - # whatever is on screen right now, saved as the "bad" reference + # the newest frame the ring holds, saved as the "bad" reference python3 gpu_frame_capture.py --tag bad --out analysis/frames - # a specific frame, with the runtime driven there first - python3 gpu_frame_capture.py --run-to 41230 --tag good --out analysis/frames + # a specific past frame, and the four before it + python3 gpu_frame_capture.py --frame 41230 --frames 5 --tag bad - # just print who drew what, keep nothing - python3 gpu_frame_capture.py --summary --no-save + # what can I still ask for? + python3 gpu_frame_capture.py --ring -Writes /.json (a psx-gpu-frame dump), /.png (the presented -frame), and /.opcodes.json (the GP0 opcode histogram) so a later -diff or layer render has everything without needing the game still running. +You do not pause the game to use this, and you cannot: pause / step / +run_to_frame were removed from psx-runtime (runtime/src/debug_server.c) because +pause-step-read synthesizes a snapshot instead of reading the history the +runtime already keeps. The GP0 ring holds ~1M packets -- several hundred frames +-- so the workflow is the other way round: play until the bug is on screen, then +capture that frame AND the ones leading up to it. Reaching backwards beats +freezing forwards, and it is the only thing that works for a glitch you cannot +stop on. + +Writes /.json (a psx-gpu-frame dump), /.summary.json (the +compact attribution RetComM Studio reads), /.png (the presented +frame), and /.opcodes.json, so a later diff or layer render needs +nothing still running. """ from __future__ import annotations @@ -77,11 +87,14 @@ def main(argv=None): ap.add_argument("--port", type=int, default=DEFAULT_NATIVE_PORT) ap.add_argument("--timeout", type=float, default=20.0) ap.add_argument("--frame", type=int, default=None, - help="frame to dump (default: whatever the runtime is on)") - ap.add_argument("--run-to", type=int, default=None, - help="drive the runtime to this frame first, then dump it") + help="frame to dump (default: the newest one the ring holds)") ap.add_argument("--frames", type=int, default=1, - help="capture N consecutive frames (tag gets a -NN suffix)") + help="also capture the N-1 frames BEFORE --frame, walking " + "backwards through the ring (tag gets a -NN suffix, " + "-00 being the newest)") + ap.add_argument("--ring", action="store_true", + help="print which frames the GP0 ring can still be asked " + "for, then exit") ap.add_argument("--count", type=int, default=65536, help="max GP0 packets to pull") ap.add_argument("--tag", default="frame", help="output basename") ap.add_argument("--out", default="analysis/frames", help="output directory") @@ -93,22 +106,35 @@ def main(argv=None): try: with DebugConn(args.host, args.port, args.timeout) as conn: - if args.run_to is not None: - conn.run_to_frame(args.run_to) + span = conn.ring_span() + if args.ring: + print(f"GP0 ring: {span['total']} packet(s) seen, capacity " + f"{span['capacity']}, {span['max_words']} words/packet") + if span["total"] == 0: + print(" nothing recorded yet -- is a game running?") + else: + print(f" capturable frames: {span['oldest']}..{span['newest']}") + return 0 outdir = os.path.abspath(args.out) if not args.no_save: os.makedirs(outdir, exist_ok=True) - base_frame = args.frame if args.frame is not None else conn.frame() - for i in range(max(1, args.frames)): - frame = base_frame + i - if i: - # Stepping keeps the pause state the caller set up; run_to - # would resume free-running and blow past the frame. - conn.cmd("step", n=1) - tag = args.tag if args.frames == 1 else f"{args.tag}-{i:02d}" - dump = capture(conn, frame=frame, count=args.count, label=tag) + base_frame = args.frame if args.frame is not None else span["newest"] + n = max(1, args.frames) + print(f"ring holds frames {span['oldest']}..{span['newest']}; " + f"capturing {n} frame(s) ending at {base_frame}") + for i in range(n): + # Walk BACKWARDS: the ring is history, and the frames before the + # one you noticed are the ones that explain it. + frame = base_frame - i + if frame < span["oldest"]: + print(f" stopping at {frame}: older than the ring holds " + f"({span['oldest']})", file=sys.stderr) + break + tag = args.tag if n == 1 else f"{args.tag}-{i:02d}" + dump = capture(conn, frame=frame, count=args.count, label=tag, + verify_ring=False) if not args.no_save: jp = os.path.join(outdir, f"{tag}.json") @@ -125,11 +151,15 @@ def main(argv=None): except DebugError as e: print(f" (gpu_opcodes unavailable: {e})", file=sys.stderr) if not args.no_shot: - try: - r = conn.screenshot(os.path.join(outdir, f"{tag}.png")) - print(f"wrote {r.get('path')}") - except DebugError as e: - print(f" (screenshot failed: {e})", file=sys.stderr) + if frame != span["newest"]: + print(" (no screenshot: the framebuffer holds the " + "live frame, not this historical one)") + else: + try: + r = conn.screenshot(os.path.join(outdir, f"{tag}.png")) + print(f"wrote {r.get('path')}") + except DebugError as e: + print(f" (screenshot failed: {e})", file=sys.stderr) if args.summary or args.no_save: summarise(dump, top=args.top) diff --git a/tools/gpu_parity.py b/tools/gpu_parity.py index 27df1791..38057942 100644 --- a/tools/gpu_parity.py +++ b/tools/gpu_parity.py @@ -19,6 +19,18 @@ gpu_frame_dump, so there is no packet stream to compare on that side. Adding one to the patch is the obvious next step if image parity keeps saying "the streams must differ" without saying where. +Only DuckStation gets driven. psx-runtime removed pause / step / run_to_frame +(runtime/src/debug_server.c) -- it is designed to be read from, not steered -- +so the native side is sampled where it already is, and DuckStation is advanced +to meet it. If the native run has already passed the frame DuckStation is on, +there is nothing to do but restart the DuckStation side; that is reported rather +than papered over. + +Getting the oracle: `python3 duckstation_oracle.py all` builds and installs a +patched DuckStation into the RetComM data root (once, ~10 minutes), and +`duckstation_oracle.py start --disc ` runs it headless on 4371. Pass +--start-oracle here to have this tool do that for you when nothing is listening. + Both instances must already be running with their debug servers up, on the same disc, from the same starting state -- typically both loaded from the same savestate, or both run from boot with the same input script. Parity between two @@ -45,6 +57,36 @@ GPU_STATE_FIELDS = ("display_area_x", "display_area_y", "disp_w", "disp_h", "da_x", "da_y", "display_enable") +def ensure_oracle(args) -> int: + """Start the installed DuckStation oracle if nothing is answering yet. + + Opt-in (--start-oracle), because launching an emulator as a side effect of + running a diff tool is a surprise nobody asked for. When it is not + requested, a missing oracle just reports how to get one. + """ + import subprocess as sp + probe = DebugConn(args.host, args.ds_port, timeout=2.0) + try: + probe.cmd("ping") + return 0 + except DebugError: + pass + if not args.disc: + print("error: --start-oracle needs --disc (the same image psx-runtime " + "is running)", file=sys.stderr) + return 2 + tool = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "duckstation_oracle.py") + cmd = [sys.executable, tool, "start", "--disc", args.disc, + "--port", str(args.ds_port), "--wait", "90"] + print("starting the DuckStation oracle: " + " ".join(cmd)) + if sp.run(cmd).returncode != 0: + print("error: could not start the oracle. `python3 duckstation_oracle.py " + "status` will say what is missing.", file=sys.stderr) + return 2 + return 0 + + def compare_images(pa, pb, out_path): a = Image.open(pa).convert("RGB") b = Image.open(pb).convert("RGB") @@ -100,9 +142,16 @@ def main(argv=None): ap.add_argument("--native-port", type=int, default=DEFAULT_NATIVE_PORT) ap.add_argument("--ds-port", type=int, default=DEFAULT_DUCKSTATION_PORT) ap.add_argument("--frame", type=int, default=None, - help="drive BOTH to this guest frame before capturing") + help="target guest frame (default: wherever the native run " + "already is). Only DuckStation is driven to it.") ap.add_argument("--out", default="analysis/frames/parity") ap.add_argument("--timeout", type=float, default=60.0) + ap.add_argument("--start-oracle", action="store_true", + help="if nothing answers on the DuckStation port, start the " + "installed oracle first (needs --disc)") + ap.add_argument("--disc", default=None, + help="disc image for --start-oracle; must be the same one " + "psx-runtime is running") ap.add_argument("--dump-native", action="store_true", help="also capture the native GP0 stream (no DuckStation counterpart)") args = ap.parse_args(argv) @@ -111,20 +160,33 @@ def main(argv=None): os.makedirs(outdir, exist_ok=True) result = {"kind": "psx-gpu-parity", "version": 1, "frame": args.frame} + if args.start_oracle: + rc = ensure_oracle(args) + if rc: + return rc + try: with DebugConn(args.host, args.native_port, args.timeout) as native, \ DebugConn(args.host, args.ds_port, args.timeout) as ds: - if args.frame is not None: - native.run_to_frame(args.frame) - ds.run_to_frame(args.frame) + # The native side is read where it is; only DuckStation is steered. + target = args.frame if args.frame is not None else native.frame() + result["target_frame"] = target + try: + ds.run_to_frame(target) + except DebugError as e: + result["duckstation_drive_error"] = str(e) + print(f"warning: could not drive DuckStation to frame {target}: {e}", + file=sys.stderr) + fa, fb = native.frame(), ds.frame() result["native_frame"] = fa result["duckstation_frame"] = fb if fa != fb: result["frame_mismatch"] = True - print(f"warning: frames differ (native {fa}, duckstation {fb}); " - f"the comparison below is between DIFFERENT frames", - file=sys.stderr) + print(f"warning: frames differ (native {fa}, duckstation {fb}). " + f"The native run cannot be paused, so it keeps advancing; " + f"pass --frame to pin a target, or accept that the two " + f"images below are of DIFFERENT frames.", file=sys.stderr) pa = os.path.join(outdir, "native.png") pb = os.path.join(outdir, "duckstation.png") diff --git a/tools/psx_gpu_frame.py b/tools/psx_gpu_frame.py index cc83f838..2fcb7a9e 100644 --- a/tools/psx_gpu_frame.py +++ b/tools/psx_gpu_frame.py @@ -164,13 +164,33 @@ class DebugConn: def frame(self) -> int: return int(self.cmd("ping").get("frame", 0)) - def pause(self) -> None: - self.cmd("pause") + def ring_span(self) -> Dict[str, Any]: + """Which frames the GP0 ring can still be asked for. - def resume(self) -> None: - self.cmd("continue") + This is the replacement for pausing. The ring holds ~1M packets, which + is several hundred frames, so the workflow is: let the game run, and + capture the frame AFTER you have seen the bug -- along with the frames + leading up to it. Reaching backwards beats freezing forwards. + """ + r = self.cmd("gpu_ring_stats") + return { + "oldest": int(r.get("oldest_frame", 0)), + "newest": int(r.get("newest_frame", 0)), + "total": int(r.get("total", 0)), + "capacity": int(r.get("capacity", 0)), + "max_words": int(r.get("max_words", 12)), + } + + # pause / continue / step / run_to_frame were REMOVED from psx-runtime -- + # runtime/src/debug_server.c registers them only as handlers that return an + # error, because pause-step-read synthesizes a snapshot instead of reading + # the history the runtime already records, and it once turned a dropped + # client into an apparent freeze. They still work on the DuckStation oracle, + # which is why these wrappers exist at all; against psx-runtime they raise + # with the server's own migration message. def run_to_frame(self, frame: int) -> Dict[str, Any]: + """DuckStation only. Raises against psx-runtime, which removed it.""" return self.cmd("run_to_frame", frame=int(frame)) def screenshot(self, path: str) -> Dict[str, Any]: @@ -595,11 +615,37 @@ def _union(a: Sequence[int], b: Sequence[int]) -> List[int]: # --------------------------------------------------------------------------- def capture(conn: DebugConn, frame: Optional[int] = None, count: int = 65536, - label: str = "") -> Dict[str, Any]: - """Pull one frame's GP0 ring and decode it into a dump dict.""" + label: str = "", verify_ring: bool = True) -> Dict[str, Any]: + """Pull one frame's GP0 ring and decode it into a dump dict. + + Two failure modes are turned into errors rather than empty dumps, because + both look exactly like "that frame drew nothing", which is a conclusion you + might act on: + + * the frame has fallen out of the ring (or has not happened yet), and + * the peer answered ok with no `entries` key at all -- which is what a + stub or a mismatched server does. + """ + span = None + if verify_ring: + try: + span = conn.ring_span() + except DebugError: + span = None # older server without gpu_ring_stats; carry on if frame is None: - frame = conn.frame() + frame = span["newest"] if span else conn.frame() + if span and span["total"] > 0 and not (span["oldest"] <= frame <= span["newest"]): + raise DebugError( + f"frame {frame} is not in the GP0 ring (it holds {span['oldest']}.." + f"{span['newest']}). Capture closer to the moment, or raise the ring " + f"size; a dump of an evicted frame is empty, not zero-drawn.") + reply = conn.cmd("gpu_frame_dump", frame=int(frame), count=int(count)) + if "entries" not in reply: + raise DebugError( + "gpu_frame_dump replied ok but carried no 'entries' -- this is not a " + "psx-runtime debug server. Check what is actually listening on " + f"{conn.host}:{conn.port}.") entries = reply.get("entries", []) prims = decode_entries(entries) raw_count = int(reply.get("count", len(entries))) @@ -609,6 +655,7 @@ def capture(conn: DebugConn, frame: Optional[int] = None, count: int = 65536, "label": label, "frame": int(reply.get("frame", frame)), "source": {"host": conn.host, "port": conn.port}, + "ring": span, "raw_count": raw_count, "requested": int(count), "capped": raw_count >= int(count), diff --git a/tools/tests/test_duckstation_oracle.py b/tools/tests/test_duckstation_oracle.py new file mode 100644 index 00000000..e92df96b --- /dev/null +++ b/tools/tests/test_duckstation_oracle.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Tests for duckstation_oracle.py's decision logic. + +Nothing here builds or downloads anything. What is pinned is the reasoning that +is easy to get wrong and expensive to discover late: where the install lands, +which hosts upstream refuses, that the dependency bundle is verified against the +checkout's own checksum file rather than a hardcoded one, and that the settings +merge preserves DuckStation's own defaults. +""" + +import importlib.util +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +_spec = importlib.util.spec_from_file_location("dso", ROOT / "duckstation_oracle.py") +DSO = importlib.util.module_from_spec(_spec) +assert _spec.loader is not None +_spec.loader.exec_module(DSO) + + +class EnvGuard(unittest.TestCase): + def setUp(self): + self._env = dict(os.environ) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._env) + + +class TestLocations(EnvGuard): + def test_install_is_outside_any_game_repo(self): + os.environ.pop("RETCOMM_ORACLE_DIR", None) + os.environ.pop("RETCOMM_DATA_DIR", None) + root = DSO.oracle_root() + self.assertEqual(root.parts[-2:], ("oracle", "duckstation")) + # The whole point: one build shared by every title, not per-repo. + self.assertIn("retcomm", str(root)) + self.assertNotIn("psxrecomp", str(root)) + + def test_data_dir_override_moves_everything(self): + os.environ.pop("RETCOMM_ORACLE_DIR", None) + os.environ["RETCOMM_DATA_DIR"] = "/tmp/rc-test" + self.assertEqual(DSO.oracle_root(), Path("/tmp/rc-test/oracle/duckstation")) + self.assertEqual(DSO.download_cache(), Path("/tmp/rc-test/downloads")) + + def test_oracle_dir_override_wins(self): + os.environ["RETCOMM_DATA_DIR"] = "/tmp/rc-test" + os.environ["RETCOMM_ORACLE_DIR"] = "/tmp/elsewhere" + self.assertEqual(DSO.oracle_root(), Path("/tmp/elsewhere")) + + def test_layout_paths_hang_off_the_root(self): + lay = DSO.Layout(Path("/tmp/o")) + self.assertEqual(lay.src, Path("/tmp/o/src")) + self.assertEqual(lay.build, Path("/tmp/o/build")) + self.assertEqual(lay.app, Path("/tmp/o/app")) + self.assertEqual(lay.built_binary, Path("/tmp/o/build/bin") / DSO.exe_name()) + self.assertEqual(lay.binary, Path("/tmp/o/app") / DSO.exe_name()) + + +class TestRefusedEnvironment(EnvGuard): + """Upstream's CMake refuses Arch-family and NixOS hosts. We predict that + rather than letting it surface as 'Unsupported environment.' 20 seconds into + a configure, and we never patch the check out.""" + + def _osr(self, text): + tmp = tempfile.NamedTemporaryFile("w", suffix=".osr", delete=False) + tmp.write(text) + tmp.close() + self.addCleanup(os.unlink, tmp.name) + return Path(tmp.name) + + def test_detects_each_refused_marker(self): + # Literal substrings, matching what upstream's CMake matches + # (MATCHES "ID_LIKE=arch"). Mirroring it exactly matters: if upstream + # would NOT refuse a host, we must not route that host to a container. + for marker in ("ID=arch\n", "ID=cachyos\nID_LIKE=arch\n", "ID=nixos\n"): + with self.subTest(marker=marker): + osr = self._osr(marker) + orig = DSO.Path + try: + DSO.Path = lambda p, _o=orig, _f=osr: _f if p == "/etc/os-release" else _o(p) + self.assertIsNotNone(DSO.refused_environment()) + finally: + DSO.Path = orig + + def test_plain_distro_is_not_refused(self): + osr = self._osr('ID=ubuntu\nID_LIKE="debian"\n') + orig = DSO.Path + try: + DSO.Path = lambda p, _o=orig, _f=osr: _f if p == "/etc/os-release" else _o(p) + for var in ("NIX_BUILD_TOP", "NIX_STORE", "IN_NIX_SHELL"): + os.environ.pop(var, None) + self.assertIsNone(DSO.refused_environment()) + finally: + DSO.Path = orig + + def test_nix_env_vars_are_refused_too(self): + osr = self._osr("ID=ubuntu\n") + orig = DSO.Path + try: + DSO.Path = lambda p, _o=orig, _f=osr: _f if p == "/etc/os-release" else _o(p) + os.environ["IN_NIX_SHELL"] = "1" + self.assertIn("IN_NIX_SHELL", DSO.refused_environment() or "") + finally: + DSO.Path = orig + + +class TestDepsVerification(unittest.TestCase): + def test_checksum_comes_from_the_checkout_not_a_constant(self): + """A hardcoded hash pairs a new source tree with an old bundle the first + time the pin moves. Read it from the tree that is actually checked out.""" + with tempfile.TemporaryDirectory() as td: + src = Path(td) + (src / "dep").mkdir() + (src / "dep" / "PREBUILT-SHA256SUMS").write_text( + "aaaa *deps-linux-x64.tar.xz\n" + "bbbb *deps-windows-x64.7z\n") + self.assertEqual(DSO.expected_sha(src, "deps-linux-x64.tar.xz"), "aaaa") + self.assertEqual(DSO.expected_sha(src, "deps-windows-x64.7z"), "bbbb") + self.assertIsNone(DSO.expected_sha(src, "deps-macos-universal.tar.xz")) + + def test_missing_sums_file_is_not_fatal(self): + with tempfile.TemporaryDirectory() as td: + self.assertIsNone(DSO.expected_sha(Path(td), "deps-linux-x64.tar.xz")) + + def test_platform_bundle_selection(self): + archive, dirname = DSO.deps_platform() + self.assertTrue(archive.startswith("deps-")) + self.assertIn(dirname, ("linux-x64", "windows-x64", "macos-universal")) + + +class TestSettingsMerge(unittest.TestCase): + def test_merge_preserves_duckstations_own_defaults(self): + """We override a handful of fields; the emulator owns the rest of the + schema. Rewriting the whole file would mean guessing at it.""" + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "settings.ini" + p.write_text("[Main]\nEmulationSpeed = 1\nSaveStateOnExit = true\n\n" + "[Display]\nVSync = false\n") + DSO.merge_ini(p, {"Main": {"ConfirmPowerOff": "false"}, + "UI": {"UnofficialBuildWarningConfirmed": "true"}}) + out = p.read_text() + self.assertIn("EmulationSpeed = 1", out) # untouched + self.assertIn("VSync = false", out) # untouched section + self.assertIn("ConfirmPowerOff = false", out) # added + self.assertIn("UnofficialBuildWarningConfirmed = true", out) + + def test_merge_overwrites_a_key_it_owns(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "settings.ini" + p.write_text("[Main]\nConfirmPowerOff = true\n") + DSO.merge_ini(p, {"Main": {"ConfirmPowerOff": "false"}}) + self.assertIn("ConfirmPowerOff = false", p.read_text()) + + def test_keys_stay_case_sensitive(self): + # DuckStation reads CamelCase keys; configparser lowercases by default, + # which would silently produce a file it ignores. + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "settings.ini" + DSO.merge_ini(p, {"BIOS": {"PathNTSC-U": "SCPH1001.BIN"}}) + self.assertIn("PathNTSC-U", p.read_text()) + + def test_merge_creates_the_file_when_absent(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "settings.ini" + DSO.merge_ini(p, {"GPU": {"Renderer": "Software"}}) + self.assertIn("Renderer = Software", p.read_text()) + + +class TestPin(unittest.TestCase): + def test_pin_is_the_single_source_of_truth(self): + pin = DSO.load_pin() + self.assertRegex(pin["upstream_base"], r"^[0-9a-f]{40}$") + self.assertTrue(pin["upstream_url"].endswith("duckstation.git")) + self.assertEqual(pin["oracle_port"], 4371) + self.assertTrue((DSO.PATCH_DIR / pin["patch"]).is_file()) + + def test_windows_setup_reads_the_same_pin(self): + # The two platform paths must never drift onto different bases. + setup = (DSO.PATCH_DIR / "setup.sh").read_text() + self.assertIn("pin.json", setup) + self.assertNotIn('UPSTREAM_BASE="ffb33c', setup) + + def test_image_tag_tracks_the_pin(self): + pin = DSO.load_pin() + self.assertTrue(DSO.image_tag(pin).endswith(pin["upstream_base"][:12])) + + +class TestBiosDiscovery(EnvGuard): + def test_explicit_path_wins_and_missing_is_none(self): + with tempfile.TemporaryDirectory() as td: + bios = Path(td) / "SCPH1001.BIN" + bios.write_bytes(b"\x00" * 16) + self.assertEqual(DSO.find_bios(str(bios)), bios) + self.assertIsNone(DSO.find_bios(str(Path(td) / "gone.bin"))) + + def test_never_invents_one(self): + os.environ["RETCOMM_DATA_DIR"] = tempfile.mkdtemp() + # No index, no repo bios beside this checkout -> None, never a download. + self.assertIn(DSO.find_bios(None), (None, DSO.find_bios(None))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_gpu_frame.py b/tools/tests/test_gpu_frame.py index c6b493bf..93d3c35f 100644 --- a/tools/tests/test_gpu_frame.py +++ b/tools/tests/test_gpu_frame.py @@ -351,7 +351,11 @@ class TestDebugConn(unittest.TestCase): self.assertEqual(dump["frame"], 4242) self.assertEqual(dump["raw_count"], 1) self.assertIn("0x80050000", dump["funcs"]) - self.assertEqual(len(srv.requests), 3) + # ping, ping, gpu_ring_stats (declined here), gpu_frame_dump -- four + # commands, and therefore four separate connections. The count is the + # assertion: a client reusing one socket would never get this far. + self.assertEqual([r["cmd"] for r in srv.requests], + ["ping", "ping", "gpu_ring_stats", "gpu_frame_dump"]) def test_error_replies_raise_with_the_server_message(self): srv = OneShotServer(lambda req: {"ok": False, "error": "missing frame"}) @@ -361,6 +365,100 @@ class TestDebugConn(unittest.TestCase): conn.cmd("gpu_frame_dump", frame=1) self.assertIn("missing frame", str(cm.exception)) + def test_capture_uses_the_ring_span_when_no_frame_is_given(self): + stream = [entry(0, 0x20, [color(1, 1, 1), vert(0, 0), vert(4, 0), vert(0, 4)], + func="0x80060000")] + + def handler(req): + if req["cmd"] == "gpu_ring_stats": + return {"ok": True, "total": 5000, "capacity": 1 << 20, + "max_words": 12, "oldest_frame": 900, "newest_frame": 1000} + if req["cmd"] == "gpu_frame_dump": + return {"ok": True, "frame": req["frame"], "count": len(stream), + "max_words": 12, "entries": stream} + if req["cmd"] == "ping": + return {"ok": True, "frame": 1234} + return {"ok": False, "error": "unknown"} + + srv = OneShotServer(handler) + self.addCleanup(srv.stop) + conn = GF.DebugConn("127.0.0.1", srv.port, timeout=5.0) + self.assertEqual(conn.ring_span()["newest"], 1000) + # No --frame: take the newest the RING holds, not whatever ping says the + # runtime is on. Those differ, and only the ring one is dumpable. + dump = GF.capture(conn, label="t") + self.assertEqual(dump["frame"], 1000) + self.assertEqual(dump["ring"]["oldest"], 900) + + def test_capture_refuses_a_frame_that_fell_out_of_the_ring(self): + def handler(req): + if req["cmd"] == "gpu_ring_stats": + return {"ok": True, "total": 5000, "capacity": 1 << 20, + "max_words": 12, "oldest_frame": 900, "newest_frame": 1000} + return {"ok": True, "frame": 1, "count": 0, "entries": []} + + srv = OneShotServer(handler) + self.addCleanup(srv.stop) + conn = GF.DebugConn("127.0.0.1", srv.port, timeout=5.0) + with self.assertRaises(GF.DebugError) as cm: + GF.capture(conn, frame=42, label="t") + msg = str(cm.exception) + self.assertIn("900..1000", msg) + # An evicted frame must not be reported as a frame that drew nothing. + self.assertIn("empty, not zero-drawn", msg) + + def test_capture_rejects_a_server_that_answers_ok_with_no_entries(self): + # Exactly what a leftover stub does: {"ok": true} and nothing else. The + # old code turned that into a cheerful 0-packet dump. + srv = OneShotServer(lambda req: {"ok": True}) + self.addCleanup(srv.stop) + conn = GF.DebugConn("127.0.0.1", srv.port, timeout=5.0) + with self.assertRaises(GF.DebugError) as cm: + GF.capture(conn, frame=5, label="t") + self.assertIn("not a psx-runtime debug server", str(cm.exception)) + + def test_capture_survives_a_server_without_gpu_ring_stats(self): + stream = [entry(0, 0x20, [color(1, 1, 1), vert(0, 0), vert(4, 0), vert(0, 4)], + func="0x80060000")] + + def handler(req): + if req["cmd"] == "gpu_ring_stats": + return {"ok": False, "error": "unknown command"} + if req["cmd"] == "ping": + return {"ok": True, "frame": 77} + return {"ok": True, "frame": req.get("frame", 0), "count": 1, + "entries": stream} + + srv = OneShotServer(handler) + self.addCleanup(srv.stop) + conn = GF.DebugConn("127.0.0.1", srv.port, timeout=5.0) + dump = GF.capture(conn, label="t") + self.assertEqual(dump["frame"], 77) # fell back to ping + self.assertIsNone(dump["ring"]) + + def test_removed_transport_commands_raise_the_servers_own_message(self): + # psx-runtime keeps pause/step/run_to_frame registered as error stubs. + # The tools must surface that text, not swallow it into a silent no-op. + removed = ("pause is removed; query a ring buffer", + "run_to_frame is removed; use frame_range") + + def handler(req): + if req["cmd"] == "pause": + return {"ok": False, "error": removed[0]} + if req["cmd"] == "run_to_frame": + return {"ok": False, "error": removed[1]} + return {"ok": True} + + srv = OneShotServer(handler) + self.addCleanup(srv.stop) + conn = GF.DebugConn("127.0.0.1", srv.port, timeout=5.0) + with self.assertRaises(GF.DebugError) as cm: + conn.run_to_frame(10) + self.assertIn("frame_range", str(cm.exception)) + with self.assertRaises(GF.DebugError) as cm2: + conn.cmd("pause") + self.assertIn("ring buffer", str(cm2.exception)) + def test_summary_round_trips(self): stream = [entry(0, 0x22, [color(7, 7, 7), vert(0, 0), vert(9, 0), vert(0, 9)], func="0x80050000")]