From a2b2c0f8f7e66a7e0992862a483e3ae84da2484b Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Tue, 8 Sep 2026 01:04:19 -0400 Subject: [PATCH 01/56] netplay: Discord sign-in and the device key The runner half. psx_netplay_auth owns the HTTP, a worker thread and the key; main.cpp is eight one-line adapters over it; queue_hello gains an optional session field. The key never crosses the wire. This runtime has no TLS -- wss:// is still an explicit "not in this phase" -- so a permanent bearer credential sent in cleartext would be captured once on a hostile network and reused forever. Instead the server issues a nonce and this proves possession with HMAC-SHA256 keyed by SHA-256 of the key. psx_sha256.c was already here, so the HMAC is twenty lines and there is no new dependency. netplay_secret is written 0600 beside the other runtime config, in the form " ", and redeemed lazily on the first pump so a machine that signed in once comes up signed in with no player action. A stored key that no longer works (revoked, or a different server) is dropped and the client falls back to guest rather than nagging every launch. That file is the point of the exercise: sign in once on a PC and the build you install on a handheld authenticates with no browser -- and it is a password in a file, so it must not go into anything distributed. Guest stays byte-identical. `session` is omitted entirely rather than sent empty, so an unauthenticated hello is the exact frame this client has always sent. The launcher wiring is guarded on RECOMP_LAUNCHER_HAS_ACCOUNT so this builds against a recomp-ui that predates the callbacks too, and the runner and the UI can land in either order. Opening the browser is per platform (ShellExecute / xdg-open / open) since there is no SDL_OpenURL here. The URL comes from our own lobby server but still reaches a shell, so anything that is not a plain http(s) URL free of shell metacharacters is refused rather than trusted. Verified: psx_netplay_auth.c and psx_lobby_client.c both compile clean under -Wall -Wextra, and every psx_account_* symbol main.cpp uses is declared and defined. main.cpp itself could NOT be compiled here -- it needs a generated tree (bios_rom_alias.h) -- so its changes are verified by symbol cross-check and inspection only, not by a build. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015C38nLRmmyXHyJ7P1DiTxg --- runtime/include/psx_netplay_auth.h | 63 +++ runtime/runtime.cmake | 1 + runtime/src/main.cpp | 34 ++ runtime/src/psx_lobby_client.c | 22 +- runtime/src/psx_netplay_auth.c | 619 +++++++++++++++++++++++++++++ 5 files changed, 735 insertions(+), 4 deletions(-) create mode 100644 runtime/include/psx_netplay_auth.h create mode 100644 runtime/src/psx_netplay_auth.c diff --git a/runtime/include/psx_netplay_auth.h b/runtime/include/psx_netplay_auth.h new file mode 100644 index 00000000..c227d9be --- /dev/null +++ b/runtime/include/psx_netplay_auth.h @@ -0,0 +1,63 @@ +/* psx_netplay_auth.h -- Discord sign-in and the device key, for the launcher. + * + * Backs recomp-ui's optional `account_*` netplay callbacks. Everything here is + * optional at runtime: a lobby server whose operator never configured Discord + * answers 503 to a login start, and this reports "not available" so the + * launcher draws no sign-in at all. A player who never signs in is a guest, + * which is what the runtime has always been. + * + * Two credentials, and the difference matters: + * + * netplay_secret Long-lived, per device, written to a file beside the + * config. NEVER sent anywhere. Proved by HMAC over a + * server nonce, because this runtime has no TLS and a + * permanent bearer token in cleartext would be captured + * once and reused forever. + * session Short-lived JWT, minted from that proof, and the only + * thing that goes to the lobby (in `hello`). + * + * The secret file is what makes a browserless device work: sign in once on a + * PC, and the file travels with a build installed on a handheld. It is a + * password in a file -- do not put it in anything you distribute. + */ +#ifndef PSX_NETPLAY_AUTH_H +#define PSX_NETPLAY_AUTH_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Mirrors recomp-ui's RECOMP_LAUNCHER_ACCOUNT_* so this header does not + * depend on the launcher's. */ +enum { + PSX_ACCOUNT_GUEST = 0, + PSX_ACCOUNT_WAITING = 1, + PSX_ACCOUNT_SIGNED_IN = 2, + PSX_ACCOUNT_FAILED = 3 +}; + +/* `ws_url` is the lobby URL the client is configured with; the HTTP endpoints + * live on the same host and port. Safe to call again when the URL changes. */ +void psx_account_init(const char *ws_url); + +/* Call once per frame. Drives the worker handshake and, when a stored key + * exists, redeems it for a session on first use. */ +void psx_account_pump(void); + +int psx_account_available(void); +int psx_account_login_begin(void); +int psx_account_state(void); +const char *psx_account_handle(void); +const char *psx_account_username(void); +const char *psx_account_error(void); +int psx_account_sign_out(void); +int psx_account_set_handle(const char *handle); + +/* The session for `hello`, or "" when this client is a guest. */ +const char *psx_account_session(void); + +#ifdef __cplusplus +} +#endif + +#endif /* PSX_NETPLAY_AUTH_H */ diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 59edc928..f2396a21 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -374,6 +374,7 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/nd_intro_ot.c ${PSXRECOMP_ROOT}/runtime/src/crc32.c ${PSXRECOMP_ROOT}/runtime/src/psx_sha256.c + ${PSXRECOMP_ROOT}/runtime/src/psx_netplay_auth.c ${PSXRECOMP_ROOT}/runtime/src/disc_identity.cpp ${PSXRECOMP_ROOT}/runtime/src/cue_sheet.cpp ${PSXRECOMP_ROOT}/runtime/src/disc_path.cpp diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 95df1665..c593be03 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -56,6 +56,7 @@ extern "C" void psx_event_step_conservative_env_init(void); #include "psx_netplay_rb.h" #include "psx_selfcheck.h" #include "psx_lobby_client.h" +#include "psx_netplay_auth.h" #if defined(PSX_HAS_RECOMP_NET) #include "recomp_net/chat_filter.h" /* chat profanity mask, LAN rooms too */ #endif @@ -9990,6 +9991,19 @@ namespace { } return psx_lobby_send_chat(line); } + /* ---- optional Discord sign-in ------------------------------------- + * Thin adapters over psx_netplay_auth, which owns the HTTP, the worker + * thread and the device key. Nothing here blocks a frame except the + * rename, which is one round trip and wants a verdict for its modal. */ + int ae_np_account_available(void*) { return psx_account_available(); } + int ae_np_account_login_begin(void*) { return psx_account_login_begin(); } + int ae_np_account_state(void*) { return psx_account_state(); } + const char* ae_np_account_handle(void*) { return psx_account_handle(); } + const char* ae_np_account_username(void*) { return psx_account_username(); } + const char* ae_np_account_error(void*) { return psx_account_error(); } + int ae_np_account_sign_out(void*) { return psx_account_sign_out(); } + int ae_np_account_set_handle(void*, const char* h) { return psx_account_set_handle(h); } + int ae_np_chat_count(void*) { ae_np_chat_track_room(); if (g_lnch_hosting_lan || g_lnch_joined_lan) return g_lnch_lan_chat_count; @@ -10195,6 +10209,9 @@ namespace { void ae_np_set_lobby_url(void*, const char* url) { g_lnch_lobby_url = url && url[0] ? url : psx_lobby_default_url(); ae_np_save_identity(nullptr, g_lnch_lobby_url.c_str()); + /* The auth endpoints live on the same host and port as the lobby + * socket, so the sign-in follows whatever server the player points at. */ + psx_account_init(g_lnch_lobby_url.c_str()); } int ae_np_connect(void*) { @@ -10773,6 +10790,10 @@ namespace { void ae_np_pump(void*) { psx_lobby_pump(); + /* Redeems a stored device key on the first pump, so a machine that has + * signed in once comes up signed in with no player action. */ + psx_account_init(g_lnch_lobby_url.c_str()); + psx_account_pump(); ae_np_lan_browse_pump(); ae_np_lan_udp_pump(); /* Lobby UI has no Ready toggle; production WS still requires every @@ -12028,6 +12049,19 @@ namespace { g_lnch_netplay_callbacks.server_chat_send = ae_np_server_chat_send; g_lnch_netplay_callbacks.server_chat_count = ae_np_server_chat_count; g_lnch_netplay_callbacks.server_chat_get = ae_np_server_chat_get; +#if defined(RECOMP_LAUNCHER_HAS_ACCOUNT) + /* Optional Discord sign-in. Guarded on the launcher ABI macro so this + * runtime still builds against a recomp-ui that predates it -- the UI + * and the runner can land in either order. */ + g_lnch_netplay_callbacks.account_available = ae_np_account_available; + g_lnch_netplay_callbacks.account_login_begin = ae_np_account_login_begin; + g_lnch_netplay_callbacks.account_state = ae_np_account_state; + g_lnch_netplay_callbacks.account_handle = ae_np_account_handle; + g_lnch_netplay_callbacks.account_username = ae_np_account_username; + g_lnch_netplay_callbacks.account_error = ae_np_account_error; + g_lnch_netplay_callbacks.account_sign_out = ae_np_account_sign_out; + g_lnch_netplay_callbacks.account_set_handle = ae_np_account_set_handle; +#endif g_lnch_netplay_callbacks.seat_move_self = ae_np_seat_move_self; g_lnch_netplay_callbacks.seat_swap_request = ae_np_seat_swap_request; g_lnch_netplay_callbacks.seat_swap_incoming = ae_np_seat_swap_incoming; diff --git a/runtime/src/psx_lobby_client.c b/runtime/src/psx_lobby_client.c index 129bf023..ea2a2706 100644 --- a/runtime/src/psx_lobby_client.c +++ b/runtime/src/psx_lobby_client.c @@ -153,6 +153,7 @@ void psx_lobby_clear_launch_pending(void) {} #include "recomp_net/lan_beacon.h" #include "recomp_net/rtt_probe.h" #include "recomp_net/chat_filter.h" +#include "psx_netplay_auth.h" /* optional Discord session for `hello` */ #include "host_time.h" #if defined(_WIN32) @@ -1868,12 +1869,25 @@ static void queue_hello(void) { char name_esc[PSX_LOBBY_NAME_LEN * 2 + 8]; char game_esc[PSX_LOBBY_NAME_LEN * 2 + 8]; - char msg[PSX_LOBBY_NAME_LEN * 4 + 64]; + char sess_esc[2048]; + char msg[PSX_LOBBY_NAME_LEN * 4 + 2176]; + const char *sess = psx_account_session(); json_escape(g_lc.display_name, name_esc, sizeof(name_esc)); json_escape(g_lc.filter_game_name, game_esc, sizeof(game_esc)); - snprintf(msg, sizeof(msg), - "{\"op\":\"hello\",\"display_name\":\"%s\",\"game_name\":\"%s\"}", - name_esc, game_esc); + /* The session is OPTIONAL and omitted entirely when this client is a + * guest, which is what keeps an unauthenticated hello byte-identical to + * the one this client has always sent. */ + if (sess && sess[0]) { + json_escape(sess, sess_esc, sizeof(sess_esc)); + snprintf(msg, sizeof(msg), + "{\"op\":\"hello\",\"display_name\":\"%s\",\"game_name\":\"%s\"," + "\"session\":\"%s\"}", + name_esc, game_esc, sess_esc); + } else { + snprintf(msg, sizeof(msg), + "{\"op\":\"hello\",\"display_name\":\"%s\",\"game_name\":\"%s\"}", + name_esc, game_esc); + } queue_send(msg); flush_pending(); } diff --git a/runtime/src/psx_netplay_auth.c b/runtime/src/psx_netplay_auth.c new file mode 100644 index 00000000..a4518d78 --- /dev/null +++ b/runtime/src/psx_netplay_auth.c @@ -0,0 +1,619 @@ +/* psx_netplay_auth.c -- see psx_netplay_auth.h. + * + * All network work happens on one worker thread; the launcher only ever reads + * flags this file publishes. That is the same shape the lobby client already + * uses for its off-thread connect, and it is why a login cannot stall a frame. + */ + +#include "psx_netplay_auth.h" +#include "psx_sha256.h" + +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#include +#include +#define close closesocket +typedef HANDLE auth_thread_t; +#else +#include +#include +#include +#include +#include +#include +#include +typedef pthread_t auth_thread_t; +#endif + +#define SECRET_MAX 96 +#define SESSION_MAX 1024 +#define NAME_MAX_LEN 64 +#define ID_MAX 64 + +/* What the worker is doing. The launcher polls psx_account_state(). */ +enum { JOB_NONE = 0, JOB_LOGIN, JOB_SESSION, JOB_SET_HANDLE, JOB_SIGN_OUT }; + +static struct { + char host[192]; + int port; + + /* Published to the launcher. Written by the worker, read on the UI + * thread; single-writer plus aligned word stores, which is what the + * lobby client already relies on for its connect flags. */ + volatile int state; /* PSX_ACCOUNT_* */ + volatile int available; /* -1 unknown, 0 no logins here, 1 yes */ + volatile int busy; + + char secret[SECRET_MAX]; + char session[SESSION_MAX]; + char player_id[ID_MAX]; + char handle[NAME_MAX_LEN]; + char username[NAME_MAX_LEN]; + char error[192]; + + /* Login in flight. */ + char pair_code[128]; + char pending_handle[NAME_MAX_LEN]; + + int job; + int thread_valid; + auth_thread_t thread; + int initialized; + int tried_stored_key; +} g; + +/* ---- small helpers ------------------------------------------------------ */ + +static void set_err(const char *e) { + snprintf(g.error, sizeof(g.error), "%s", e ? e : ""); +} + +/* Minimal extraction for the flat, server-generated JSON these endpoints + * return. Not a parser: it is the same shape psx_lobby_client.c already uses, + * and the bodies come from our own server. */ +static int json_str(const char *json, const char *key, char *out, size_t cap) { + char pat[64]; + const char *p; + size_t o = 0; + if (!json || !out || cap == 0) return 0; + out[0] = '\0'; + snprintf(pat, sizeof(pat), "\"%s\"", key); + p = strstr(json, pat); + if (!p) return 0; + p = strchr(p + strlen(pat), ':'); + if (!p) return 0; + ++p; + while (*p && isspace((unsigned char)*p)) ++p; + if (*p != '"') return 0; + ++p; + while (*p && *p != '"' && o + 1 < cap) { + if (*p == '\\' && p[1]) { + ++p; + switch (*p) { + case 'n': out[o++] = '\n'; break; + case 't': out[o++] = '\t'; break; + case 'r': out[o++] = '\r'; break; + default: out[o++] = *p; break; + } + ++p; + continue; + } + out[o++] = *p++; + } + out[o] = '\0'; + return o > 0; +} + +/* Escape for a JSON string body. Same rules as the lobby client's escaper: + * quote and backslash escaped, control characters dropped. */ +static void json_esc(const char *in, char *out, size_t cap) { + size_t o = 0; + if (!out || cap == 0) return; + while (in && *in && o + 2 < cap) { + unsigned char c = (unsigned char)*in++; + if (c == '"' || c == '\\') { + if (o + 3 >= cap) break; + out[o++] = '\\'; + out[o++] = (char)c; + } else if (c >= 0x20) { + out[o++] = (char)c; + } + } + out[o] = '\0'; +} + +static void hex_encode(const unsigned char *in, size_t n, char *out) { + static const char H[] = "0123456789abcdef"; + size_t i; + for (i = 0; i < n; ++i) { + out[i * 2] = H[in[i] >> 4]; + out[i * 2 + 1] = H[in[i] & 15]; + } + out[n * 2] = '\0'; +} + +/* HMAC-SHA256, keyed by the hex verifier string, over the nonce. The server + * computes the identical thing (secrets::expected_proof). */ +static void hmac_sha256_hex(const char *key, const char *msg, char out_hex[65]) { + unsigned char k[64], inner[32], outer[32], pad[64]; + unsigned char tmp[96]; + psx_sha256_ctx c; + size_t klen = strlen(key), i; + + memset(k, 0, sizeof(k)); + if (klen > 64) { + psx_sha256_compute((const unsigned char *)key, klen, k); + } else { + memcpy(k, key, klen); + } + + for (i = 0; i < 64; ++i) pad[i] = k[i] ^ 0x36; + psx_sha256_init(&c); + psx_sha256_update(&c, pad, 64); + psx_sha256_update(&c, (const unsigned char *)msg, strlen(msg)); + psx_sha256_final(&c, inner); + + for (i = 0; i < 64; ++i) pad[i] = k[i] ^ 0x5c; + memcpy(tmp, pad, 64); + memcpy(tmp + 64, inner, 32); + psx_sha256_compute(tmp, 96, outer); + hex_encode(outer, 32, out_hex); +} + +/* ---- the secret file ---------------------------------------------------- */ + +static void secret_path(char *out, size_t cap) { + /* Beside the other runtime config (keybinds.ini, config.ini), so a build + * copied to another device carries it the same way. */ + snprintf(out, cap, "netplay_secret"); +} + +static void secret_load(void) { + char path[256]; + FILE *f; + size_t n; + secret_path(path, sizeof(path)); + g.secret[0] = '\0'; + g.player_id[0] = '\0'; + f = fopen(path, "rb"); + if (!f) return; + n = fread(g.secret, 1, sizeof(g.secret) - 1, f); + g.secret[n] = '\0'; + fclose(f); + /* Format: " \n". Tolerate a trailing newline. */ + { + char *sp = strchr(g.secret, ' '); + char *nl; + if (sp) { + size_t idn = (size_t)(sp - g.secret); + if (idn >= sizeof(g.player_id)) idn = sizeof(g.player_id) - 1; + memcpy(g.player_id, g.secret, idn); + g.player_id[idn] = '\0'; + memmove(g.secret, sp + 1, strlen(sp + 1) + 1); + } + nl = strpbrk(g.secret, "\r\n"); + if (nl) *nl = '\0'; + } +} + +static void secret_store(const char *player_id, const char *secret) { + char path[256]; + FILE *f; + secret_path(path, sizeof(path)); + f = fopen(path, "wb"); + if (!f) return; + fprintf(f, "%s %s\n", player_id ? player_id : "", secret ? secret : ""); + fclose(f); +#if !defined(_WIN32) + /* A password in a file: keep it to the owner. */ + (void)chmod(path, S_IRUSR | S_IWUSR); +#endif + snprintf(g.secret, sizeof(g.secret), "%s", secret ? secret : ""); + snprintf(g.player_id, sizeof(g.player_id), "%s", player_id ? player_id : ""); +} + +static void secret_forget(void) { + char path[256]; + secret_path(path, sizeof(path)); + remove(path); + g.secret[0] = '\0'; + g.player_id[0] = '\0'; +} + +/* ---- blocking HTTP POST (worker thread only) ---------------------------- */ + +/* Returns the HTTP status, or <0 on a transport failure. Body is copied into + * `body` (truncated to fit), which is enough for these small JSON replies. */ +static int http_post(const char *path, const char *json, char *body, size_t cap) { + struct addrinfo hints, *res = NULL, *ai; + char portstr[16], req[2048]; + char buf[4096]; + int fd = -1, status = -1; + size_t total = 0; + const char *hdr_end; + + if (body && cap) body[0] = '\0'; + snprintf(portstr, sizeof(portstr), "%d", g.port); + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(g.host, portstr, &hints, &res) != 0 || !res) return -1; + for (ai = res; ai; ai = ai->ai_next) { + fd = (int)socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + if (connect(fd, ai->ai_addr, (int)ai->ai_addrlen) == 0) break; + close(fd); + fd = -1; + } + freeaddrinfo(res); + if (fd < 0) return -2; + + snprintf(req, sizeof(req), + "POST %s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "Content-Type: application/json\r\n" + "Content-Length: %d\r\n" + "Connection: close\r\n\r\n%s", + path, g.host, g.port, (int)strlen(json), json); + if (send(fd, req, (int)strlen(req), 0) < 0) { + close(fd); + return -3; + } + + for (;;) { + int n = (int)recv(fd, buf + total, (int)(sizeof(buf) - 1 - total), 0); + if (n <= 0) break; + total += (size_t)n; + if (total >= sizeof(buf) - 1) break; + } + close(fd); + buf[total] = '\0'; + if (total < 12) return -4; + status = atoi(buf + 9); /* "HTTP/1.1 NNN ..." */ + hdr_end = strstr(buf, "\r\n\r\n"); + if (hdr_end && body && cap) { + snprintf(body, cap, "%s", hdr_end + 4); + } + return status; +} + +/* ---- one challenge/proof round ------------------------------------------ */ + +/* Fills `session` and the display fields. Returns 1 on success. The key never + * leaves this function: only the HMAC does. */ +static int prove_and_get_session(void) { + char body[2048], reqbuf[512], nonce[128], verifier_hex[65], proof[65]; + unsigned char digest[32]; + char id_esc[128]; + int st; + + if (!g.secret[0] || !g.player_id[0]) return 0; + + json_esc(g.player_id, id_esc, sizeof(id_esc)); + snprintf(reqbuf, sizeof(reqbuf), "{\"player_id\":\"%s\"}", id_esc); + st = http_post("/auth/challenge", reqbuf, body, sizeof(body)); + if (st != 200 || !json_str(body, "nonce", nonce, sizeof(nonce))) return 0; + + psx_sha256_compute((const unsigned char *)g.secret, strlen(g.secret), digest); + hex_encode(digest, 32, verifier_hex); + hmac_sha256_hex(verifier_hex, nonce, proof); + + snprintf(reqbuf, sizeof(reqbuf), + "{\"player_id\":\"%s\",\"nonce\":\"%s\",\"proof\":\"%s\"}", + id_esc, nonce, proof); + st = http_post("/auth/session", reqbuf, body, sizeof(body)); + if (st != 200) return 0; + json_str(body, "session", g.session, sizeof(g.session)); + json_str(body, "handle", g.handle, sizeof(g.handle)); + json_str(body, "discord_username", g.username, sizeof(g.username)); + return g.session[0] ? 1 : 0; +} + +/* ---- opening the browser ------------------------------------------------ */ + +/* The one thing the launcher cannot do for us: hand a URL to the desktop. + * There is no SDL_OpenURL in this runtime, so this is per platform. A failure + * is not fatal -- the player can still be told to open the page themselves. */ +static void account_open_url(const char *url) { + if (!url || !url[0]) return; +#if defined(_WIN32) + ShellExecuteA(NULL, "open", url, NULL, NULL, SW_SHOWNORMAL); +#else + { + char cmd[1200]; +#if defined(__APPLE__) + const char *opener = "open"; +#else + const char *opener = "xdg-open"; +#endif + /* The URL comes from our own lobby server, but it still goes through a + * shell, so refuse anything that is not a plain http(s) URL rather + * than trusting the source. */ + const char *q; + if (strncmp(url, "http://", 7) != 0 && strncmp(url, "https://", 8) != 0) return; + for (q = url; *q; ++q) { + if (*q == '\'' || *q == '"' || *q == '`' || *q == '$' || *q == '\\' || + *q == ';' || *q == '|' || *q == '&' || *q == '<' || *q == '>' || + (unsigned char)*q < 0x20) { + fprintf(stderr, "psx_account: refusing to open a suspicious URL\n"); + return; + } + } + snprintf(cmd, sizeof(cmd), "%s '%s' >/dev/null 2>&1 &", opener, url); + if (system(cmd) != 0) { + fprintf(stderr, "psx_account: could not open a browser; visit:\n %s\n", url); + } + } +#endif +} + +/* ---- worker ------------------------------------------------------------- */ + +static void job_login(void) { + char body[2048], url[1024]; + int st, tries; + + st = http_post("/auth/discord/start", "{}", body, sizeof(body)); + if (st == 503) { + g.available = 0; + set_err("This lobby server does not offer Discord sign-in."); + g.state = PSX_ACCOUNT_FAILED; + return; + } + if (st != 200 || !json_str(body, "url", url, sizeof(url)) || + !json_str(body, "code", g.pair_code, sizeof(g.pair_code))) { + set_err("Could not reach the lobby server to start the sign-in."); + g.state = PSX_ACCOUNT_FAILED; + return; + } + g.available = 1; + account_open_url(url); + + /* Poll until the server has an answer. The pairing code expires server + * side after ten minutes; stop a little before that rather than spinning + * forever if the player closed the tab. */ + for (tries = 0; tries < 300; ++tries) { + char req[256]; + char code_esc[160]; + json_esc(g.pair_code, code_esc, sizeof(code_esc)); + snprintf(req, sizeof(req), "{\"code\":\"%s\"}", code_esc); + st = http_post("/auth/discord/poll", req, body, sizeof(body)); + if (st == 200) { + char secret[SECRET_MAX], pid[ID_MAX]; + json_str(body, "session", g.session, sizeof(g.session)); + json_str(body, "handle", g.handle, sizeof(g.handle)); + json_str(body, "discord_username", g.username, sizeof(g.username)); + if (json_str(body, "netplay_secret", secret, sizeof(secret)) && + json_str(body, "player_id", pid, sizeof(pid))) { + /* Returned exactly once. Store it now or it is gone. */ + secret_store(pid, secret); + } + g.state = PSX_ACCOUNT_SIGNED_IN; + return; + } + if (st == 202) { +#if defined(_WIN32) + Sleep(2000); +#else + struct timespec ts; ts.tv_sec = 2; ts.tv_nsec = 0; + nanosleep(&ts, NULL); +#endif + continue; + } + set_err("Discord sign-in did not complete. Try again."); + g.state = PSX_ACCOUNT_FAILED; + return; + } + set_err("Sign-in timed out. Try again."); + g.state = PSX_ACCOUNT_FAILED; +} + +static void job_session(void) { + if (prove_and_get_session()) { + g.state = PSX_ACCOUNT_SIGNED_IN; + } else { + /* A stored key that no longer works (revoked, or a different server): + * drop it and fall back to guest rather than nagging every launch. */ + secret_forget(); + g.session[0] = '\0'; + g.state = PSX_ACCOUNT_GUEST; + } +} + +static void job_set_handle(void) { + char body[1024], req[768], esc[160], id_esc[128], nonce[128]; + char verifier_hex[65], proof[65]; + unsigned char digest[32]; + int st; + + if (!g.secret[0] || !g.player_id[0]) { + set_err("Sign in before changing your name."); + return; + } + json_esc(g.player_id, id_esc, sizeof(id_esc)); + snprintf(req, sizeof(req), "{\"player_id\":\"%s\"}", id_esc); + if (http_post("/auth/challenge", req, body, sizeof(body)) != 200 || + !json_str(body, "nonce", nonce, sizeof(nonce))) { + set_err("Could not reach the lobby server."); + return; + } + psx_sha256_compute((const unsigned char *)g.secret, strlen(g.secret), digest); + hex_encode(digest, 32, verifier_hex); + hmac_sha256_hex(verifier_hex, nonce, proof); + + json_esc(g.pending_handle, esc, sizeof(esc)); + snprintf(req, sizeof(req), + "{\"player_id\":\"%s\",\"nonce\":\"%s\",\"proof\":\"%s\",\"handle\":\"%s\"}", + id_esc, nonce, proof, esc); + st = http_post("/auth/handle", req, body, sizeof(body)); + if (st == 200) { + json_str(body, "handle", g.handle, sizeof(g.handle)); + } else { + set_err("That name can't be used. Please pick another one."); + } +} + +static void job_sign_out(void) { + char body[512], req[512], id_esc[128], nonce[128], verifier_hex[65], proof[65]; + unsigned char digest[32]; + + /* Best effort: tell the server to retire this device's key, then forget it + * locally whatever happened. Signing out must work offline. */ + if (g.secret[0] && g.player_id[0]) { + json_esc(g.player_id, id_esc, sizeof(id_esc)); + snprintf(req, sizeof(req), "{\"player_id\":\"%s\"}", id_esc); + if (http_post("/auth/challenge", req, body, sizeof(body)) == 200 && + json_str(body, "nonce", nonce, sizeof(nonce))) { + psx_sha256_compute((const unsigned char *)g.secret, strlen(g.secret), digest); + hex_encode(digest, 32, verifier_hex); + hmac_sha256_hex(verifier_hex, nonce, proof); + snprintf(req, sizeof(req), + "{\"player_id\":\"%s\",\"nonce\":\"%s\",\"proof\":\"%s\"}", + id_esc, nonce, proof); + (void)http_post("/auth/secret/revoke", req, body, sizeof(body)); + } + } + secret_forget(); + g.session[0] = '\0'; + g.handle[0] = '\0'; + g.username[0] = '\0'; + g.state = PSX_ACCOUNT_GUEST; +} + +#if defined(_WIN32) +static unsigned __stdcall auth_worker(void *arg) { +#else +static void *auth_worker(void *arg) { +#endif + int job = (int)(intptr_t)arg; + switch (job) { + case JOB_LOGIN: job_login(); break; + case JOB_SESSION: job_session(); break; + case JOB_SET_HANDLE: job_set_handle(); break; + case JOB_SIGN_OUT: job_sign_out(); break; + default: break; + } + g.busy = 0; +#if defined(_WIN32) + return 0; +#else + return NULL; +#endif +} + +static int start_job(int job) { + if (g.busy) return -1; + g.busy = 1; + g.job = job; +#if defined(_WIN32) + { + uintptr_t th = _beginthreadex(NULL, 0, auth_worker, (void *)(intptr_t)job, 0, NULL); + if (!th) { g.busy = 0; return -1; } + g.thread = (HANDLE)th; + } +#else + if (pthread_create(&g.thread, NULL, auth_worker, (void *)(intptr_t)job) != 0) { + g.busy = 0; + return -1; + } + pthread_detach(g.thread); +#endif + g.thread_valid = 1; + return 0; +} + +/* ---- public surface ----------------------------------------------------- */ + +void psx_account_init(const char *ws_url) { + const char *p = ws_url; + const char *slash; + const char *colon; + char hostport[192]; + + memset(g.host, 0, sizeof(g.host)); + g.port = 8765; + if (p) { + if (strncmp(p, "ws://", 5) == 0) p += 5; + else if (strncmp(p, "wss://", 6) == 0) p += 6; + slash = strchr(p, '/'); + snprintf(hostport, sizeof(hostport), "%.*s", + slash ? (int)(slash - p) : (int)strlen(p), p); + colon = strrchr(hostport, ':'); + if (colon && strchr(colon, ']') == NULL) { + g.port = atoi(colon + 1); + *(char *)colon = '\0'; + } + snprintf(g.host, sizeof(g.host), "%s", hostport); + } + if (!g.initialized) { + g.available = -1; + g.state = PSX_ACCOUNT_GUEST; + secret_load(); + g.initialized = 1; + } +} + +void psx_account_pump(void) { + if (!g.initialized || g.busy) return; + /* A stored key is redeemed once, lazily, on the first pump: the launcher + * should come up signed in without the player doing anything. */ + if (!g.tried_stored_key && g.secret[0] && g.player_id[0]) { + g.tried_stored_key = 1; + g.state = PSX_ACCOUNT_WAITING; + (void)start_job(JOB_SESSION); + } +} + +int psx_account_available(void) { + /* Unknown until something asks the server. Offering the button is right: + * the first start() settles it, and a 503 turns it off for good. */ + return g.available != 0; +} + +int psx_account_login_begin(void) { + if (g.busy) return -1; + g.error[0] = '\0'; + g.state = PSX_ACCOUNT_WAITING; + return start_job(JOB_LOGIN); +} + +int psx_account_state(void) { return g.state; } +const char *psx_account_handle(void) { return g.handle; } +const char *psx_account_username(void) { return g.username; } +const char *psx_account_error(void) { return g.error; } +const char *psx_account_session(void) { return g.session; } + +int psx_account_sign_out(void) { + if (g.busy) return -1; + return start_job(JOB_SIGN_OUT); +} + +int psx_account_set_handle(const char *handle) { + if (g.busy || !handle || !handle[0]) return -1; + snprintf(g.pending_handle, sizeof(g.pending_handle), "%s", handle); + g.error[0] = '\0'; + if (start_job(JOB_SET_HANDLE) != 0) return -1; + /* The launcher wants a synchronous verdict for its modal, and the request + * is one round trip on a LAN-ish connection: wait briefly rather than + * inventing an async path through the UI for a rename. */ + { + int spins = 0; + while (g.busy && spins++ < 400) { +#if defined(_WIN32) + Sleep(10); +#else + struct timespec ts; ts.tv_sec = 0; ts.tv_nsec = 10 * 1000 * 1000; + nanosleep(&ts, NULL); +#endif + } + } + return g.error[0] ? -1 : 0; +} From cce195f9b43eb44d762816f75d1f617870774584 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Tue, 8 Sep 2026 09:51:49 -0400 Subject: [PATCH 02/56] netplay: open the browser without a shell Sign-in did nothing when clicked: the browser never opened. My own hardening broke it. account_open_url built a `xdg-open ''` command string and screened the URL for shell metacharacters, and an OAuth authorize URL is full of `&` -- client_id&response_type&redirect_uri&scope &state -- so every real URL hit the metacharacter check and was refused. The feature could never have worked on Linux or macOS. Fixed by removing the shell rather than by loosening the screen. The URL goes to execlp as a single argv element, so there is no command line for a metacharacter to escape into and nothing to quote. A double fork leaves the opener reparented to init, so a game loop never has a zombie to reap. Windows was always fine -- ShellExecuteA takes the URL directly. The scheme check stays, and is now the whole of the screening: xdg-open will act on a file:// URL or a local path, and this only ever legitimately receives http(s). Verified rather than reasoned about this time: with a stub opener on PATH, a real Discord authorize URL arrives byte-identical, every `&` intact, and a file:// URL is still refused. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015C38nLRmmyXHyJ7P1DiTxg --- runtime/src/psx_netplay_auth.c | 46 +++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/runtime/src/psx_netplay_auth.c b/runtime/src/psx_netplay_auth.c index a4518d78..d49e2659 100644 --- a/runtime/src/psx_netplay_auth.c +++ b/runtime/src/psx_netplay_auth.c @@ -29,6 +29,7 @@ typedef HANDLE auth_thread_t; #include #include #include +#include #include typedef pthread_t auth_thread_t; #endif @@ -320,36 +321,47 @@ static int prove_and_get_session(void) { /* ---- opening the browser ------------------------------------------------ */ /* The one thing the launcher cannot do for us: hand a URL to the desktop. - * There is no SDL_OpenURL in this runtime, so this is per platform. A failure - * is not fatal -- the player can still be told to open the page themselves. */ + * There is no SDL_OpenURL in this runtime, so this is per platform. + * + * NO SHELL. An earlier version built a `xdg-open ''` command string and + * screened the URL for shell metacharacters -- which refused every real + * authorize URL, because an OAuth query string is full of `&`. Passing the URL + * as a single argv element removes the quoting problem instead of policing it, + * and there is nothing left for a metacharacter to escape into. */ static void account_open_url(const char *url) { if (!url || !url[0]) return; + /* Still scheme-checked: xdg-open will happily act on a file:// URL or a + * local path, and this only ever legitimately receives http(s). */ + if (strncmp(url, "http://", 7) != 0 && strncmp(url, "https://", 8) != 0) { + fprintf(stderr, "%s: refusing to open a non-http(s) URL\n", "psx_account"); + return; + } #if defined(_WIN32) + /* Takes the URL directly; no command line is built. */ ShellExecuteA(NULL, "open", url, NULL, NULL, SW_SHOWNORMAL); #else { - char cmd[1200]; #if defined(__APPLE__) const char *opener = "open"; #else const char *opener = "xdg-open"; #endif - /* The URL comes from our own lobby server, but it still goes through a - * shell, so refuse anything that is not a plain http(s) URL rather - * than trusting the source. */ - const char *q; - if (strncmp(url, "http://", 7) != 0 && strncmp(url, "https://", 8) != 0) return; - for (q = url; *q; ++q) { - if (*q == '\'' || *q == '"' || *q == '`' || *q == '$' || *q == '\\' || - *q == ';' || *q == '|' || *q == '&' || *q == '<' || *q == '>' || - (unsigned char)*q < 0x20) { - fprintf(stderr, "psx_account: refusing to open a suspicious URL\n"); - return; + pid_t pid = fork(); + if (pid == 0) { + /* Double-fork: the opener is reparented to init, so it outlives + * this process and leaves no zombie for a game loop to reap. */ + if (fork() == 0) { + execlp(opener, opener, url, (char *)NULL); + _exit(127); } + _exit(0); } - snprintf(cmd, sizeof(cmd), "%s '%s' >/dev/null 2>&1 &", opener, url); - if (system(cmd) != 0) { - fprintf(stderr, "psx_account: could not open a browser; visit:\n %s\n", url); + if (pid > 0) { + int st = 0; + (void)waitpid(pid, &st, 0); + } else { + fprintf(stderr, "%s: could not open a browser; visit:\n %s\n", + "psx_account", url); } } #endif From 69e31bcd39e531187b995cf086d24eef4c8c083f Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Tue, 8 Sep 2026 09:56:11 -0400 Subject: [PATCH 03/56] netplay: use the shared auth module instead of a private copy psx_netplay_auth.c/h are gone; recomp-net's rnet_auth is the one copy, and the submodule is bumped to pick it up. Callers change name only (psx_account_* -> rnet_account_*). The duplication was a day old and had already cost a bug: the URL screen that refused every OAuth authorize URL had to be fixed in this file and in snesrecomp's copy separately, and the second was caught by remembering rather than by anything structural. Lobby client compiles with the same 12 pre-existing warnings as before and no errors. main.cpp still cannot be compiled here -- it needs a generated tree -- so its rename is verified by symbol cross-check only. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015C38nLRmmyXHyJ7P1DiTxg --- lib/recomp-net | 2 +- runtime/include/psx_netplay_auth.h | 63 --- runtime/runtime.cmake | 1 - runtime/src/main.cpp | 24 +- runtime/src/psx_lobby_client.c | 4 +- runtime/src/psx_netplay_auth.c | 631 ----------------------------- 6 files changed, 15 insertions(+), 710 deletions(-) delete mode 100644 runtime/include/psx_netplay_auth.h delete mode 100644 runtime/src/psx_netplay_auth.c diff --git a/lib/recomp-net b/lib/recomp-net index 30772f6e..85230040 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit 30772f6e78845b937c270e4af869a6b989e91b07 +Subproject commit 85230040c089bf847b9a1dadd1ed4fbc0d395ee6 diff --git a/runtime/include/psx_netplay_auth.h b/runtime/include/psx_netplay_auth.h deleted file mode 100644 index c227d9be..00000000 --- a/runtime/include/psx_netplay_auth.h +++ /dev/null @@ -1,63 +0,0 @@ -/* psx_netplay_auth.h -- Discord sign-in and the device key, for the launcher. - * - * Backs recomp-ui's optional `account_*` netplay callbacks. Everything here is - * optional at runtime: a lobby server whose operator never configured Discord - * answers 503 to a login start, and this reports "not available" so the - * launcher draws no sign-in at all. A player who never signs in is a guest, - * which is what the runtime has always been. - * - * Two credentials, and the difference matters: - * - * netplay_secret Long-lived, per device, written to a file beside the - * config. NEVER sent anywhere. Proved by HMAC over a - * server nonce, because this runtime has no TLS and a - * permanent bearer token in cleartext would be captured - * once and reused forever. - * session Short-lived JWT, minted from that proof, and the only - * thing that goes to the lobby (in `hello`). - * - * The secret file is what makes a browserless device work: sign in once on a - * PC, and the file travels with a build installed on a handheld. It is a - * password in a file -- do not put it in anything you distribute. - */ -#ifndef PSX_NETPLAY_AUTH_H -#define PSX_NETPLAY_AUTH_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* Mirrors recomp-ui's RECOMP_LAUNCHER_ACCOUNT_* so this header does not - * depend on the launcher's. */ -enum { - PSX_ACCOUNT_GUEST = 0, - PSX_ACCOUNT_WAITING = 1, - PSX_ACCOUNT_SIGNED_IN = 2, - PSX_ACCOUNT_FAILED = 3 -}; - -/* `ws_url` is the lobby URL the client is configured with; the HTTP endpoints - * live on the same host and port. Safe to call again when the URL changes. */ -void psx_account_init(const char *ws_url); - -/* Call once per frame. Drives the worker handshake and, when a stored key - * exists, redeems it for a session on first use. */ -void psx_account_pump(void); - -int psx_account_available(void); -int psx_account_login_begin(void); -int psx_account_state(void); -const char *psx_account_handle(void); -const char *psx_account_username(void); -const char *psx_account_error(void); -int psx_account_sign_out(void); -int psx_account_set_handle(const char *handle); - -/* The session for `hello`, or "" when this client is a guest. */ -const char *psx_account_session(void); - -#ifdef __cplusplus -} -#endif - -#endif /* PSX_NETPLAY_AUTH_H */ diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index f2396a21..59edc928 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -374,7 +374,6 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/nd_intro_ot.c ${PSXRECOMP_ROOT}/runtime/src/crc32.c ${PSXRECOMP_ROOT}/runtime/src/psx_sha256.c - ${PSXRECOMP_ROOT}/runtime/src/psx_netplay_auth.c ${PSXRECOMP_ROOT}/runtime/src/disc_identity.cpp ${PSXRECOMP_ROOT}/runtime/src/cue_sheet.cpp ${PSXRECOMP_ROOT}/runtime/src/disc_path.cpp diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index c593be03..837087c5 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -56,7 +56,7 @@ extern "C" void psx_event_step_conservative_env_init(void); #include "psx_netplay_rb.h" #include "psx_selfcheck.h" #include "psx_lobby_client.h" -#include "psx_netplay_auth.h" +#include "recomp_net/auth.h" #if defined(PSX_HAS_RECOMP_NET) #include "recomp_net/chat_filter.h" /* chat profanity mask, LAN rooms too */ #endif @@ -9995,14 +9995,14 @@ namespace { * Thin adapters over psx_netplay_auth, which owns the HTTP, the worker * thread and the device key. Nothing here blocks a frame except the * rename, which is one round trip and wants a verdict for its modal. */ - int ae_np_account_available(void*) { return psx_account_available(); } - int ae_np_account_login_begin(void*) { return psx_account_login_begin(); } - int ae_np_account_state(void*) { return psx_account_state(); } - const char* ae_np_account_handle(void*) { return psx_account_handle(); } - const char* ae_np_account_username(void*) { return psx_account_username(); } - const char* ae_np_account_error(void*) { return psx_account_error(); } - int ae_np_account_sign_out(void*) { return psx_account_sign_out(); } - int ae_np_account_set_handle(void*, const char* h) { return psx_account_set_handle(h); } + int ae_np_account_available(void*) { return rnet_account_available(); } + int ae_np_account_login_begin(void*) { return rnet_account_login_begin(); } + int ae_np_account_state(void*) { return rnet_account_state(); } + const char* ae_np_account_handle(void*) { return rnet_account_handle(); } + const char* ae_np_account_username(void*) { return rnet_account_username(); } + const char* ae_np_account_error(void*) { return rnet_account_error(); } + int ae_np_account_sign_out(void*) { return rnet_account_sign_out(); } + int ae_np_account_set_handle(void*, const char* h) { return rnet_account_set_handle(h); } int ae_np_chat_count(void*) { ae_np_chat_track_room(); @@ -10211,7 +10211,7 @@ namespace { ae_np_save_identity(nullptr, g_lnch_lobby_url.c_str()); /* The auth endpoints live on the same host and port as the lobby * socket, so the sign-in follows whatever server the player points at. */ - psx_account_init(g_lnch_lobby_url.c_str()); + rnet_account_init(g_lnch_lobby_url.c_str()); } int ae_np_connect(void*) { @@ -10792,8 +10792,8 @@ namespace { psx_lobby_pump(); /* Redeems a stored device key on the first pump, so a machine that has * signed in once comes up signed in with no player action. */ - psx_account_init(g_lnch_lobby_url.c_str()); - psx_account_pump(); + rnet_account_init(g_lnch_lobby_url.c_str()); + rnet_account_pump(); ae_np_lan_browse_pump(); ae_np_lan_udp_pump(); /* Lobby UI has no Ready toggle; production WS still requires every diff --git a/runtime/src/psx_lobby_client.c b/runtime/src/psx_lobby_client.c index ea2a2706..73ff7350 100644 --- a/runtime/src/psx_lobby_client.c +++ b/runtime/src/psx_lobby_client.c @@ -153,7 +153,7 @@ void psx_lobby_clear_launch_pending(void) {} #include "recomp_net/lan_beacon.h" #include "recomp_net/rtt_probe.h" #include "recomp_net/chat_filter.h" -#include "psx_netplay_auth.h" /* optional Discord session for `hello` */ +#include "recomp_net/auth.h" /* optional Discord session for `hello` */ #include "host_time.h" #if defined(_WIN32) @@ -1871,7 +1871,7 @@ static void queue_hello(void) char game_esc[PSX_LOBBY_NAME_LEN * 2 + 8]; char sess_esc[2048]; char msg[PSX_LOBBY_NAME_LEN * 4 + 2176]; - const char *sess = psx_account_session(); + const char *sess = rnet_account_session(); json_escape(g_lc.display_name, name_esc, sizeof(name_esc)); json_escape(g_lc.filter_game_name, game_esc, sizeof(game_esc)); /* The session is OPTIONAL and omitted entirely when this client is a diff --git a/runtime/src/psx_netplay_auth.c b/runtime/src/psx_netplay_auth.c deleted file mode 100644 index d49e2659..00000000 --- a/runtime/src/psx_netplay_auth.c +++ /dev/null @@ -1,631 +0,0 @@ -/* psx_netplay_auth.c -- see psx_netplay_auth.h. - * - * All network work happens on one worker thread; the launcher only ever reads - * flags this file publishes. That is the same shape the lobby client already - * uses for its off-thread connect, and it is why a login cannot stall a frame. - */ - -#include "psx_netplay_auth.h" -#include "psx_sha256.h" - -#include -#include -#include -#include -#include -#include - -#if defined(_WIN32) -#include -#include -#include -#include -#define close closesocket -typedef HANDLE auth_thread_t; -#else -#include -#include -#include -#include -#include -#include -#include -#include -typedef pthread_t auth_thread_t; -#endif - -#define SECRET_MAX 96 -#define SESSION_MAX 1024 -#define NAME_MAX_LEN 64 -#define ID_MAX 64 - -/* What the worker is doing. The launcher polls psx_account_state(). */ -enum { JOB_NONE = 0, JOB_LOGIN, JOB_SESSION, JOB_SET_HANDLE, JOB_SIGN_OUT }; - -static struct { - char host[192]; - int port; - - /* Published to the launcher. Written by the worker, read on the UI - * thread; single-writer plus aligned word stores, which is what the - * lobby client already relies on for its connect flags. */ - volatile int state; /* PSX_ACCOUNT_* */ - volatile int available; /* -1 unknown, 0 no logins here, 1 yes */ - volatile int busy; - - char secret[SECRET_MAX]; - char session[SESSION_MAX]; - char player_id[ID_MAX]; - char handle[NAME_MAX_LEN]; - char username[NAME_MAX_LEN]; - char error[192]; - - /* Login in flight. */ - char pair_code[128]; - char pending_handle[NAME_MAX_LEN]; - - int job; - int thread_valid; - auth_thread_t thread; - int initialized; - int tried_stored_key; -} g; - -/* ---- small helpers ------------------------------------------------------ */ - -static void set_err(const char *e) { - snprintf(g.error, sizeof(g.error), "%s", e ? e : ""); -} - -/* Minimal extraction for the flat, server-generated JSON these endpoints - * return. Not a parser: it is the same shape psx_lobby_client.c already uses, - * and the bodies come from our own server. */ -static int json_str(const char *json, const char *key, char *out, size_t cap) { - char pat[64]; - const char *p; - size_t o = 0; - if (!json || !out || cap == 0) return 0; - out[0] = '\0'; - snprintf(pat, sizeof(pat), "\"%s\"", key); - p = strstr(json, pat); - if (!p) return 0; - p = strchr(p + strlen(pat), ':'); - if (!p) return 0; - ++p; - while (*p && isspace((unsigned char)*p)) ++p; - if (*p != '"') return 0; - ++p; - while (*p && *p != '"' && o + 1 < cap) { - if (*p == '\\' && p[1]) { - ++p; - switch (*p) { - case 'n': out[o++] = '\n'; break; - case 't': out[o++] = '\t'; break; - case 'r': out[o++] = '\r'; break; - default: out[o++] = *p; break; - } - ++p; - continue; - } - out[o++] = *p++; - } - out[o] = '\0'; - return o > 0; -} - -/* Escape for a JSON string body. Same rules as the lobby client's escaper: - * quote and backslash escaped, control characters dropped. */ -static void json_esc(const char *in, char *out, size_t cap) { - size_t o = 0; - if (!out || cap == 0) return; - while (in && *in && o + 2 < cap) { - unsigned char c = (unsigned char)*in++; - if (c == '"' || c == '\\') { - if (o + 3 >= cap) break; - out[o++] = '\\'; - out[o++] = (char)c; - } else if (c >= 0x20) { - out[o++] = (char)c; - } - } - out[o] = '\0'; -} - -static void hex_encode(const unsigned char *in, size_t n, char *out) { - static const char H[] = "0123456789abcdef"; - size_t i; - for (i = 0; i < n; ++i) { - out[i * 2] = H[in[i] >> 4]; - out[i * 2 + 1] = H[in[i] & 15]; - } - out[n * 2] = '\0'; -} - -/* HMAC-SHA256, keyed by the hex verifier string, over the nonce. The server - * computes the identical thing (secrets::expected_proof). */ -static void hmac_sha256_hex(const char *key, const char *msg, char out_hex[65]) { - unsigned char k[64], inner[32], outer[32], pad[64]; - unsigned char tmp[96]; - psx_sha256_ctx c; - size_t klen = strlen(key), i; - - memset(k, 0, sizeof(k)); - if (klen > 64) { - psx_sha256_compute((const unsigned char *)key, klen, k); - } else { - memcpy(k, key, klen); - } - - for (i = 0; i < 64; ++i) pad[i] = k[i] ^ 0x36; - psx_sha256_init(&c); - psx_sha256_update(&c, pad, 64); - psx_sha256_update(&c, (const unsigned char *)msg, strlen(msg)); - psx_sha256_final(&c, inner); - - for (i = 0; i < 64; ++i) pad[i] = k[i] ^ 0x5c; - memcpy(tmp, pad, 64); - memcpy(tmp + 64, inner, 32); - psx_sha256_compute(tmp, 96, outer); - hex_encode(outer, 32, out_hex); -} - -/* ---- the secret file ---------------------------------------------------- */ - -static void secret_path(char *out, size_t cap) { - /* Beside the other runtime config (keybinds.ini, config.ini), so a build - * copied to another device carries it the same way. */ - snprintf(out, cap, "netplay_secret"); -} - -static void secret_load(void) { - char path[256]; - FILE *f; - size_t n; - secret_path(path, sizeof(path)); - g.secret[0] = '\0'; - g.player_id[0] = '\0'; - f = fopen(path, "rb"); - if (!f) return; - n = fread(g.secret, 1, sizeof(g.secret) - 1, f); - g.secret[n] = '\0'; - fclose(f); - /* Format: " \n". Tolerate a trailing newline. */ - { - char *sp = strchr(g.secret, ' '); - char *nl; - if (sp) { - size_t idn = (size_t)(sp - g.secret); - if (idn >= sizeof(g.player_id)) idn = sizeof(g.player_id) - 1; - memcpy(g.player_id, g.secret, idn); - g.player_id[idn] = '\0'; - memmove(g.secret, sp + 1, strlen(sp + 1) + 1); - } - nl = strpbrk(g.secret, "\r\n"); - if (nl) *nl = '\0'; - } -} - -static void secret_store(const char *player_id, const char *secret) { - char path[256]; - FILE *f; - secret_path(path, sizeof(path)); - f = fopen(path, "wb"); - if (!f) return; - fprintf(f, "%s %s\n", player_id ? player_id : "", secret ? secret : ""); - fclose(f); -#if !defined(_WIN32) - /* A password in a file: keep it to the owner. */ - (void)chmod(path, S_IRUSR | S_IWUSR); -#endif - snprintf(g.secret, sizeof(g.secret), "%s", secret ? secret : ""); - snprintf(g.player_id, sizeof(g.player_id), "%s", player_id ? player_id : ""); -} - -static void secret_forget(void) { - char path[256]; - secret_path(path, sizeof(path)); - remove(path); - g.secret[0] = '\0'; - g.player_id[0] = '\0'; -} - -/* ---- blocking HTTP POST (worker thread only) ---------------------------- */ - -/* Returns the HTTP status, or <0 on a transport failure. Body is copied into - * `body` (truncated to fit), which is enough for these small JSON replies. */ -static int http_post(const char *path, const char *json, char *body, size_t cap) { - struct addrinfo hints, *res = NULL, *ai; - char portstr[16], req[2048]; - char buf[4096]; - int fd = -1, status = -1; - size_t total = 0; - const char *hdr_end; - - if (body && cap) body[0] = '\0'; - snprintf(portstr, sizeof(portstr), "%d", g.port); - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - if (getaddrinfo(g.host, portstr, &hints, &res) != 0 || !res) return -1; - for (ai = res; ai; ai = ai->ai_next) { - fd = (int)socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) continue; - if (connect(fd, ai->ai_addr, (int)ai->ai_addrlen) == 0) break; - close(fd); - fd = -1; - } - freeaddrinfo(res); - if (fd < 0) return -2; - - snprintf(req, sizeof(req), - "POST %s HTTP/1.1\r\n" - "Host: %s:%d\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %d\r\n" - "Connection: close\r\n\r\n%s", - path, g.host, g.port, (int)strlen(json), json); - if (send(fd, req, (int)strlen(req), 0) < 0) { - close(fd); - return -3; - } - - for (;;) { - int n = (int)recv(fd, buf + total, (int)(sizeof(buf) - 1 - total), 0); - if (n <= 0) break; - total += (size_t)n; - if (total >= sizeof(buf) - 1) break; - } - close(fd); - buf[total] = '\0'; - if (total < 12) return -4; - status = atoi(buf + 9); /* "HTTP/1.1 NNN ..." */ - hdr_end = strstr(buf, "\r\n\r\n"); - if (hdr_end && body && cap) { - snprintf(body, cap, "%s", hdr_end + 4); - } - return status; -} - -/* ---- one challenge/proof round ------------------------------------------ */ - -/* Fills `session` and the display fields. Returns 1 on success. The key never - * leaves this function: only the HMAC does. */ -static int prove_and_get_session(void) { - char body[2048], reqbuf[512], nonce[128], verifier_hex[65], proof[65]; - unsigned char digest[32]; - char id_esc[128]; - int st; - - if (!g.secret[0] || !g.player_id[0]) return 0; - - json_esc(g.player_id, id_esc, sizeof(id_esc)); - snprintf(reqbuf, sizeof(reqbuf), "{\"player_id\":\"%s\"}", id_esc); - st = http_post("/auth/challenge", reqbuf, body, sizeof(body)); - if (st != 200 || !json_str(body, "nonce", nonce, sizeof(nonce))) return 0; - - psx_sha256_compute((const unsigned char *)g.secret, strlen(g.secret), digest); - hex_encode(digest, 32, verifier_hex); - hmac_sha256_hex(verifier_hex, nonce, proof); - - snprintf(reqbuf, sizeof(reqbuf), - "{\"player_id\":\"%s\",\"nonce\":\"%s\",\"proof\":\"%s\"}", - id_esc, nonce, proof); - st = http_post("/auth/session", reqbuf, body, sizeof(body)); - if (st != 200) return 0; - json_str(body, "session", g.session, sizeof(g.session)); - json_str(body, "handle", g.handle, sizeof(g.handle)); - json_str(body, "discord_username", g.username, sizeof(g.username)); - return g.session[0] ? 1 : 0; -} - -/* ---- opening the browser ------------------------------------------------ */ - -/* The one thing the launcher cannot do for us: hand a URL to the desktop. - * There is no SDL_OpenURL in this runtime, so this is per platform. - * - * NO SHELL. An earlier version built a `xdg-open ''` command string and - * screened the URL for shell metacharacters -- which refused every real - * authorize URL, because an OAuth query string is full of `&`. Passing the URL - * as a single argv element removes the quoting problem instead of policing it, - * and there is nothing left for a metacharacter to escape into. */ -static void account_open_url(const char *url) { - if (!url || !url[0]) return; - /* Still scheme-checked: xdg-open will happily act on a file:// URL or a - * local path, and this only ever legitimately receives http(s). */ - if (strncmp(url, "http://", 7) != 0 && strncmp(url, "https://", 8) != 0) { - fprintf(stderr, "%s: refusing to open a non-http(s) URL\n", "psx_account"); - return; - } -#if defined(_WIN32) - /* Takes the URL directly; no command line is built. */ - ShellExecuteA(NULL, "open", url, NULL, NULL, SW_SHOWNORMAL); -#else - { -#if defined(__APPLE__) - const char *opener = "open"; -#else - const char *opener = "xdg-open"; -#endif - pid_t pid = fork(); - if (pid == 0) { - /* Double-fork: the opener is reparented to init, so it outlives - * this process and leaves no zombie for a game loop to reap. */ - if (fork() == 0) { - execlp(opener, opener, url, (char *)NULL); - _exit(127); - } - _exit(0); - } - if (pid > 0) { - int st = 0; - (void)waitpid(pid, &st, 0); - } else { - fprintf(stderr, "%s: could not open a browser; visit:\n %s\n", - "psx_account", url); - } - } -#endif -} - -/* ---- worker ------------------------------------------------------------- */ - -static void job_login(void) { - char body[2048], url[1024]; - int st, tries; - - st = http_post("/auth/discord/start", "{}", body, sizeof(body)); - if (st == 503) { - g.available = 0; - set_err("This lobby server does not offer Discord sign-in."); - g.state = PSX_ACCOUNT_FAILED; - return; - } - if (st != 200 || !json_str(body, "url", url, sizeof(url)) || - !json_str(body, "code", g.pair_code, sizeof(g.pair_code))) { - set_err("Could not reach the lobby server to start the sign-in."); - g.state = PSX_ACCOUNT_FAILED; - return; - } - g.available = 1; - account_open_url(url); - - /* Poll until the server has an answer. The pairing code expires server - * side after ten minutes; stop a little before that rather than spinning - * forever if the player closed the tab. */ - for (tries = 0; tries < 300; ++tries) { - char req[256]; - char code_esc[160]; - json_esc(g.pair_code, code_esc, sizeof(code_esc)); - snprintf(req, sizeof(req), "{\"code\":\"%s\"}", code_esc); - st = http_post("/auth/discord/poll", req, body, sizeof(body)); - if (st == 200) { - char secret[SECRET_MAX], pid[ID_MAX]; - json_str(body, "session", g.session, sizeof(g.session)); - json_str(body, "handle", g.handle, sizeof(g.handle)); - json_str(body, "discord_username", g.username, sizeof(g.username)); - if (json_str(body, "netplay_secret", secret, sizeof(secret)) && - json_str(body, "player_id", pid, sizeof(pid))) { - /* Returned exactly once. Store it now or it is gone. */ - secret_store(pid, secret); - } - g.state = PSX_ACCOUNT_SIGNED_IN; - return; - } - if (st == 202) { -#if defined(_WIN32) - Sleep(2000); -#else - struct timespec ts; ts.tv_sec = 2; ts.tv_nsec = 0; - nanosleep(&ts, NULL); -#endif - continue; - } - set_err("Discord sign-in did not complete. Try again."); - g.state = PSX_ACCOUNT_FAILED; - return; - } - set_err("Sign-in timed out. Try again."); - g.state = PSX_ACCOUNT_FAILED; -} - -static void job_session(void) { - if (prove_and_get_session()) { - g.state = PSX_ACCOUNT_SIGNED_IN; - } else { - /* A stored key that no longer works (revoked, or a different server): - * drop it and fall back to guest rather than nagging every launch. */ - secret_forget(); - g.session[0] = '\0'; - g.state = PSX_ACCOUNT_GUEST; - } -} - -static void job_set_handle(void) { - char body[1024], req[768], esc[160], id_esc[128], nonce[128]; - char verifier_hex[65], proof[65]; - unsigned char digest[32]; - int st; - - if (!g.secret[0] || !g.player_id[0]) { - set_err("Sign in before changing your name."); - return; - } - json_esc(g.player_id, id_esc, sizeof(id_esc)); - snprintf(req, sizeof(req), "{\"player_id\":\"%s\"}", id_esc); - if (http_post("/auth/challenge", req, body, sizeof(body)) != 200 || - !json_str(body, "nonce", nonce, sizeof(nonce))) { - set_err("Could not reach the lobby server."); - return; - } - psx_sha256_compute((const unsigned char *)g.secret, strlen(g.secret), digest); - hex_encode(digest, 32, verifier_hex); - hmac_sha256_hex(verifier_hex, nonce, proof); - - json_esc(g.pending_handle, esc, sizeof(esc)); - snprintf(req, sizeof(req), - "{\"player_id\":\"%s\",\"nonce\":\"%s\",\"proof\":\"%s\",\"handle\":\"%s\"}", - id_esc, nonce, proof, esc); - st = http_post("/auth/handle", req, body, sizeof(body)); - if (st == 200) { - json_str(body, "handle", g.handle, sizeof(g.handle)); - } else { - set_err("That name can't be used. Please pick another one."); - } -} - -static void job_sign_out(void) { - char body[512], req[512], id_esc[128], nonce[128], verifier_hex[65], proof[65]; - unsigned char digest[32]; - - /* Best effort: tell the server to retire this device's key, then forget it - * locally whatever happened. Signing out must work offline. */ - if (g.secret[0] && g.player_id[0]) { - json_esc(g.player_id, id_esc, sizeof(id_esc)); - snprintf(req, sizeof(req), "{\"player_id\":\"%s\"}", id_esc); - if (http_post("/auth/challenge", req, body, sizeof(body)) == 200 && - json_str(body, "nonce", nonce, sizeof(nonce))) { - psx_sha256_compute((const unsigned char *)g.secret, strlen(g.secret), digest); - hex_encode(digest, 32, verifier_hex); - hmac_sha256_hex(verifier_hex, nonce, proof); - snprintf(req, sizeof(req), - "{\"player_id\":\"%s\",\"nonce\":\"%s\",\"proof\":\"%s\"}", - id_esc, nonce, proof); - (void)http_post("/auth/secret/revoke", req, body, sizeof(body)); - } - } - secret_forget(); - g.session[0] = '\0'; - g.handle[0] = '\0'; - g.username[0] = '\0'; - g.state = PSX_ACCOUNT_GUEST; -} - -#if defined(_WIN32) -static unsigned __stdcall auth_worker(void *arg) { -#else -static void *auth_worker(void *arg) { -#endif - int job = (int)(intptr_t)arg; - switch (job) { - case JOB_LOGIN: job_login(); break; - case JOB_SESSION: job_session(); break; - case JOB_SET_HANDLE: job_set_handle(); break; - case JOB_SIGN_OUT: job_sign_out(); break; - default: break; - } - g.busy = 0; -#if defined(_WIN32) - return 0; -#else - return NULL; -#endif -} - -static int start_job(int job) { - if (g.busy) return -1; - g.busy = 1; - g.job = job; -#if defined(_WIN32) - { - uintptr_t th = _beginthreadex(NULL, 0, auth_worker, (void *)(intptr_t)job, 0, NULL); - if (!th) { g.busy = 0; return -1; } - g.thread = (HANDLE)th; - } -#else - if (pthread_create(&g.thread, NULL, auth_worker, (void *)(intptr_t)job) != 0) { - g.busy = 0; - return -1; - } - pthread_detach(g.thread); -#endif - g.thread_valid = 1; - return 0; -} - -/* ---- public surface ----------------------------------------------------- */ - -void psx_account_init(const char *ws_url) { - const char *p = ws_url; - const char *slash; - const char *colon; - char hostport[192]; - - memset(g.host, 0, sizeof(g.host)); - g.port = 8765; - if (p) { - if (strncmp(p, "ws://", 5) == 0) p += 5; - else if (strncmp(p, "wss://", 6) == 0) p += 6; - slash = strchr(p, '/'); - snprintf(hostport, sizeof(hostport), "%.*s", - slash ? (int)(slash - p) : (int)strlen(p), p); - colon = strrchr(hostport, ':'); - if (colon && strchr(colon, ']') == NULL) { - g.port = atoi(colon + 1); - *(char *)colon = '\0'; - } - snprintf(g.host, sizeof(g.host), "%s", hostport); - } - if (!g.initialized) { - g.available = -1; - g.state = PSX_ACCOUNT_GUEST; - secret_load(); - g.initialized = 1; - } -} - -void psx_account_pump(void) { - if (!g.initialized || g.busy) return; - /* A stored key is redeemed once, lazily, on the first pump: the launcher - * should come up signed in without the player doing anything. */ - if (!g.tried_stored_key && g.secret[0] && g.player_id[0]) { - g.tried_stored_key = 1; - g.state = PSX_ACCOUNT_WAITING; - (void)start_job(JOB_SESSION); - } -} - -int psx_account_available(void) { - /* Unknown until something asks the server. Offering the button is right: - * the first start() settles it, and a 503 turns it off for good. */ - return g.available != 0; -} - -int psx_account_login_begin(void) { - if (g.busy) return -1; - g.error[0] = '\0'; - g.state = PSX_ACCOUNT_WAITING; - return start_job(JOB_LOGIN); -} - -int psx_account_state(void) { return g.state; } -const char *psx_account_handle(void) { return g.handle; } -const char *psx_account_username(void) { return g.username; } -const char *psx_account_error(void) { return g.error; } -const char *psx_account_session(void) { return g.session; } - -int psx_account_sign_out(void) { - if (g.busy) return -1; - return start_job(JOB_SIGN_OUT); -} - -int psx_account_set_handle(const char *handle) { - if (g.busy || !handle || !handle[0]) return -1; - snprintf(g.pending_handle, sizeof(g.pending_handle), "%s", handle); - g.error[0] = '\0'; - if (start_job(JOB_SET_HANDLE) != 0) return -1; - /* The launcher wants a synchronous verdict for its modal, and the request - * is one round trip on a LAN-ish connection: wait briefly rather than - * inventing an async path through the UI for a rename. */ - { - int spins = 0; - while (g.busy && spins++ < 400) { -#if defined(_WIN32) - Sleep(10); -#else - struct timespec ts; ts.tv_sec = 0; ts.tv_nsec = 10 * 1000 * 1000; - nanosleep(&ts, NULL); -#endif - } - } - return g.error[0] ? -1 : 0; -} From e19c25d3fbc6bbdd472988e92223899dc3ac2556 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Tue, 8 Sep 2026 09:48:21 -0400 Subject: [PATCH 04/56] yes --- lib/retcomm-rbengine | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/retcomm-rbengine b/lib/retcomm-rbengine index ebd94a47..a7b98507 160000 --- a/lib/retcomm-rbengine +++ b/lib/retcomm-rbengine @@ -1 +1 @@ -Subproject commit ebd94a4729abe2c0615070cef3ffe05b3f9ebf28 +Subproject commit a7b98507a62fe00e5aec3b90c52a4134f3c174bc From 7959acf261ad1b241837a8f62682a0af81ef56c8 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Tue, 8 Sep 2026 10:07:43 -0400 Subject: [PATCH 05/56] netplay: bump recomp-net for the sign-in availability fix A zero-initialised static made rnet_account_available() answer "no logins here" until the netplay pump had run, which is why the first NETPLAY click skipped the sign-in page. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015C38nLRmmyXHyJ7P1DiTxg --- lib/recomp-net | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/recomp-net b/lib/recomp-net index 85230040..e9644544 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit 85230040c089bf847b9a1dadd1ed4fbc0d395ee6 +Subproject commit e964454448afa40afc7e6a11fb26cde4401ceaf1 From f0bbbabe798b2eeed525c1b8fc6c63636b43d88b Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Tue, 8 Sep 2026 10:17:52 -0400 Subject: [PATCH 06/56] netplay: bump recomp-net for cancellable sign-in and clearer failures Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015C38nLRmmyXHyJ7P1DiTxg --- lib/recomp-net | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/recomp-net b/lib/recomp-net index e9644544..b4c58fcf 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit e964454448afa40afc7e6a11fb26cde4401ceaf1 +Subproject commit b4c58fcfcdd3faadb3ba74da9a138ec1955c4540 From cf9648e7ea950a8eb75382930233f8e051b2a413 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Wed, 9 Sep 2026 17:29:48 -0400 Subject: [PATCH 07/56] Rename the product from RetComM to Retro The launcher and studio ship as Retro Launcher and Retro Studio now, from the RetroPortingToolKit org, so the old product name is gone from display text, documentation and release asset filenames. The split is by case, and it is not cosmetic. CamelCase RetComM was always the product; lowercase retcomm and RETCOMM_ are identifiers -- the C++ namespace, the binaries, the data directories, the CMake and environment variables, the Cloudflare service names, the readme markers. Renaming those would orphan existing installs for no benefit, so they are untouched: 1101 lowercase retcomm and 205 RETCOMM_ still stand in the launcher alone. Four things are deliberately still spelled RetComM, and three of them would have been real breakage: - RetComM-Data is CamelCase but it is a directory on disk -- \RetComM-Data\{runtime,config,data} for a portable install. Renaming it strands every portable user's config, toolchains and catalog cache. - TechnicallyComputers/RetComM-Launcher in readme_has_launcher() is the historical half of a both-slugs test. Renaming it makes it a duplicate of the line above and sends every port built before the transfer back to reporting its launcher section missing. - _LEGACY_LAUNCHER_RE matched "^## RetComM Launcher". A sweep renamed it, which would have silently stopped it matching every README already out there; it now accepts both headings. - The portable-stub lookup gained the pre-rename filenames back as fallbacks, because old zips stay on the releases page and a rollback downloads one. Release assets become Retro-Launcher-* / Retro-Studio-*. That is safe for installed clients: pick_launcher_asset() gates on *linux*/*windows*/ *macos* plus extension and arch, not on the product name, and the Windows portable stub is identified by its RCM1 trailer rather than its filename. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013DpaQqVguHFpBuCmdoFYWY --- .github/actions/fetch-toolchain/action.yml | 2 +- .gitmodules | 4 +- README.md | 2 +- THIRD_PARTY_ATTRIBUTION.md | 4 +- docs/GAME_PROJECT_SETUP.md | 22 +++++----- docs/GPU_FRAME_TOOLS.md | 4 +- docs/LOCAL_CODEGEN_SDK.md | 8 ++-- docs/NETPLAY.md | 2 +- docs/ci/HOST_ONLY_RELEASES.md | 10 ++--- docs/ci/README.md | 2 +- docs/ci/templates/setup-release.yml | 4 +- docs/config_schema.md | 4 +- host/psxrecomp_codegen_host.c | 10 ++--- lib/retcomm-rbengine | 2 +- psxrecomp_cli.py | 8 ++-- recompiler/src/analysis_export.h | 2 +- recompiler/src/full_function_emitter.cpp | 2 +- runtime/runtime.cmake | 6 +-- tools/duckstation/README.md | 2 +- tools/duckstation_oracle.py | 10 ++--- tools/fetch_toolchain.sh | 2 +- tools/gpu_frame_capture.py | 2 +- tools/gpu_parity.py | 2 +- tools/new_project_layout/fill_tokens.py | 2 +- tools/new_project_layout/probe_disc.py | 2 +- .../new_project_layout/project_studio/cli.py | 12 +++--- .../project_studio/detect.py | 12 +++--- .../project_studio/gitops.py | 6 +-- .../new_project_layout/project_studio/gui.py | 16 ++++---- .../project_studio/naming.py | 2 +- .../new_project_layout/project_studio/ops.py | 4 +- .../new_project_layout/project_studio/plan.py | 2 +- .../project_studio/readme_metrics.py | 41 +++++++++++++------ tools/new_project_layout/setup_project.ps1 | 8 ++-- tools/new_project_layout/setup_project.sh | 10 ++--- .../new_project_layout/templates/README.md.in | 16 ++++---- tools/new_project_layout/update_disc_set.py | 4 +- tools/package_setup_host.sh | 8 ++-- tools/prepare_disc.py | 2 +- tools/psx_gpu_frame.py | 2 +- tools/stage_setup_sdk.sh | 4 +- tools/tests/test_duckstation_oracle.py | 2 +- tools/toolchain_pack.py | 14 +++---- 43 files changed, 150 insertions(+), 135 deletions(-) diff --git a/.github/actions/fetch-toolchain/action.yml b/.github/actions/fetch-toolchain/action.yml index fc24ab2e..71cd437c 100644 --- a/.github/actions/fetch-toolchain/action.yml +++ b/.github/actions/fetch-toolchain/action.yml @@ -22,7 +22,7 @@ inputs: repo: description: GitHub owner/name for toolchain releases required: false - default: TechnicallyComputers/retcomm-toolchains + default: RetroPortingToolKit/RetroPorting-Toolchains outputs: toolchain-dir: diff --git a/.gitmodules b/.gitmodules index 6adc28bc..953e090d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "lib/recomp-net"] path = lib/recomp-net - url = https://github.com/TechnicallyComputers/recomp-net.git + url = https://github.com/RetroPortingToolKit/recomp-net.git branch = main [submodule "lib/retcomm-rbengine"] path = lib/retcomm-rbengine - url = https://github.com/TechnicallyComputers/retcomm-rbengine.git + url = https://github.com/RetroPortingToolKit/rbengine.git branch = main diff --git a/README.md b/README.md index 5157cb50..54c9d6d4 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ Git for Windows also provides the `bash` the setup script uses. Then pick one of these for the compiler, CMake, and Ninja: - **Bundled toolchain (recommended).** Download `cmake-clang-v1-windows-x64.zip` - from [retcomm-toolchains](https://github.com/TechnicallyComputers/retcomm-toolchains/releases/latest), + from [retcomm-toolchains](https://github.com/RetroPortingToolKit/RetroPorting-Toolchains/releases/latest), unzip it (for example to `C:\retcomm-toolchain`), and in the PowerShell window you will run the setup from: diff --git a/THIRD_PARTY_ATTRIBUTION.md b/THIRD_PARTY_ATTRIBUTION.md index 0ad0cb39..ec09436b 100644 --- a/THIRD_PARTY_ATTRIBUTION.md +++ b/THIRD_PARTY_ATTRIBUTION.md @@ -86,7 +86,7 @@ All reuse keeps the original copyright and dual MIT/Apache-2.0 license. ## retcomm-studio — multi-disc project tooling -[retcomm-studio](https://github.com/TechnicallyComputers/retcomm-studio) by +[retcomm-studio](https://github.com/RetroPortingToolKit/Retro-Studio) by Alex Vanderveen, licensed **MIT** (notice: `LICENSE` in that repository). psxrecomp is PolyForm-NC, so this is permissive vendored into stricter — the MIT notice must ride along and is why this entry exists. @@ -107,5 +107,5 @@ written here because neither repository had it: `probe_disc.py scaffolding a project and destructive on a live one. Keep the pin above accurate when re-syncing. These files are the reason a -standalone setup-wizard install and a RetComM build produce the same +standalone setup-wizard install and a Retro build produce the same multi-disc `game.toml`; if the two drift, so do those two paths. diff --git a/docs/GAME_PROJECT_SETUP.md b/docs/GAME_PROJECT_SETUP.md index c5915f22..2735beba 100644 --- a/docs/GAME_PROJECT_SETUP.md +++ b/docs/GAME_PROJECT_SETUP.md @@ -298,7 +298,7 @@ multi-track `.cue`/`.bin` requirements. Everything game-specific lives at the **root** of *your* title repo. Framework and UI are submodules next to that code — not nested under each other. -The RetComM / Generate & rebuild CLI (`psxrecomp_cli.py`, `tools/prepare_disc.py`, +The Retro / Generate & rebuild CLI (`psxrecomp_cli.py`, `tools/prepare_disc.py`, pack helpers) ships **inside** the `psxrecomp` submodule. There is no separate `psxrecomp-sdk/` tree. @@ -307,12 +307,12 @@ YourGameRecomp/ # your git repo ├── .gitmodules ├── CMakeLists.txt # thin: psxrecomp_add_game_runtime(...) ├── game.toml # probe autofills identity / netplay gates -├── catalog_identity.json # RetComM / catalog digests + track_counts + disc_fp +├── catalog_identity.json # Retro / catalog digests + track_counts + disc_fp ├── framework_pins.txt # optional scaffold snapshot (gitlinks are authoritative) ├── README.md # scaffold stub (legal + quick start) ├── VERSION # release pin (e.g. 0.1.0) ├── assets/ -│ ├── psxrecomp.ico # default Windows app icon (RetComM-themed pad) +│ ├── psxrecomp.ico # default Windows app icon (Retro-themed pad) │ ├── psxrecomp.png # staged beside exe / packaging │ └── psxrecomp.svg # source mark (see psxrecomp/assets/) ├── seeds/ghidra_funcs.txt # probe: boot-EXE JAL seeds (grow over time) @@ -429,10 +429,10 @@ embedded `toolchain/`. The zip includes: - `recomp-ui/` sources (needed to rebuild) - On Windows: MinGW runtime DLLs beside the host and emitters -Players (or [RetComM](https://github.com/TechnicallyComputers/RetComM-Launcher)) -run **Generate once** (wizard or RetComM Build & Install) with a legal disc. -RetComM / the wizard download `cmake-clang-v1` from -[retcomm-toolchains](https://github.com/TechnicallyComputers/retcomm-toolchains) +Players (or [Retro](https://github.com/RetroPortingToolKit/Retro-Launcher)) +run **Generate once** (wizard or Retro Build & Install) with a legal disc. +Retro / the wizard download `cmake-clang-v1` from +[retcomm-toolchains](https://github.com/RetroPortingToolKit/RetroPorting-Toolchains) (or accept an offline zip / `RETCOMM_TOOLCHAIN_DIR`). Pass `--embed-toolchain` to `package_setup_host.sh` only for special offline-first packs. @@ -441,7 +441,7 @@ packs. | Action | Meaning | |--------|---------| -| **Update** (RetComM) | New setup-host zip → refresh source → cmake rebuild. Skips disc→C when `codegen-cache` fingerprints (ROM/BIOS/emitters) still match. | +| **Update** (Retro) | New setup-host zip → refresh source → cmake rebuild. Skips disc→C when `codegen-cache` fingerprints (ROM/BIOS/emitters) still match. | | **Generate & Rebuild** | Force regenerate game C from the disc, then rebuild. Use when emit inputs change or cache is wrong. | Ordinary host/UI releases do **not** require Generate & Rebuild. Details: @@ -559,7 +559,7 @@ Full action reference: `[ci/README.md](ci/README.md)`. ## Bundled release checklist Use this before tagging a setup-host release that matches other titles -(BPE / MotK / RetComM). +(BPE / MotK / Retro). ### Repository @@ -631,7 +631,7 @@ Use this before tagging a setup-host release that matches other titles ``` - [ ] Extract → run host → Generate with a legal disc succeeds end-to-end - [ ] After rebuild, game launches; saves/settings land beside the exe -- [ ] RetComM install/update (if you publish a catalog entry) uses this **same** +- [ ] Retro install/update (if you publish a catalog entry) uses this **same** ``` zip — no separate tools pack required; Update rebuilds via codegen-cache (not raw zip extract over a Play binary) @@ -648,4 +648,4 @@ Use this before tagging a setup-host release that matches other titles ``` no pre-generated game C) ``` -- [ ] Catalog / RetComM entry points at the release assets when ready \ No newline at end of file +- [ ] Catalog / Retro entry points at the release assets when ready \ No newline at end of file diff --git a/docs/GPU_FRAME_TOOLS.md b/docs/GPU_FRAME_TOOLS.md index 19d08bad..8aedbbc0 100644 --- a/docs/GPU_FRAME_TOOLS.md +++ b/docs/GPU_FRAME_TOOLS.md @@ -12,7 +12,7 @@ tools/gpu_parity.py frame-locked image parity vs DuckStation tools/tests/test_gpu_frame.py ``` -RetComM Studio's **Frames** tab is a viewer and launcher over exactly these +Retro Studio's **Frames** tab is a viewer and launcher over exactly these artifacts. It never decodes a GP0 packet itself, so a headless capture and the GUI can never disagree about what a frame contained. @@ -96,7 +96,7 @@ grep -E '^(PSX_DEBUG_TOOLS|CMAKE_BUILD_TYPE):' build-release/CMakeCache.txt `CMAKE_BUILD_TYPE` on an **existing** build dir does *not* flip it. Pass the `-D` explicitly, or start a fresh build dir. -In RetComM Studio this is the **Build tab → Debug tools** row: a selector that +In Retro Studio this is the **Build tab → Debug tools** row: a selector that injects the flag, a **Configure for debugging** button that sets Release + ON in one click, and a status line that reads the configured build dir and says whether the server will be there. The Frames tab shows the same line when it diff --git a/docs/LOCAL_CODEGEN_SDK.md b/docs/LOCAL_CODEGEN_SDK.md index e51c8363..42a67a59 100644 --- a/docs/LOCAL_CODEGEN_SDK.md +++ b/docs/LOCAL_CODEGEN_SDK.md @@ -1,7 +1,7 @@ # Local codegen SDK Headless contract for regenerating an existing psxrecomp game project from a -user-supplied disc. Intended for `recomp-ui` setup flows and RetComM launcher +user-supplied disc. Intended for `recomp-ui` setup flows and Retro launcher automation. This does **not** redistribute disc images. PGO (optional) runs **only on the user’s machine** during local rebuild when @@ -38,14 +38,14 @@ product binary (`psxrecomp_codegen_host_forward_if_built`). Opt out with | `tools/stage_setup_sdk.sh` | Pack: emitters, OpenBIOS checks, optional `toolchain/`, MinGW DLLs | | `tools/bundle_mingw_dlls.sh` | Windows: copy MinGW runtime DLLs next to host + emitters | | Project `toolchain/` | Stamp file `.psxrecomp-bin` pointing at the shared pack `bin/` | -| Shared toolchain cache | `%LOCALAPPDATA%/retcomm/toolchains/cmake-clang-v1/` (Windows) or `~/.local/share/retcomm/toolchains/cmake-clang-v1/` — same tree RetComM uses | +| Shared toolchain cache | `%LOCALAPPDATA%/retcomm/toolchains/cmake-clang-v1/` (Windows) or `~/.local/share/retcomm/toolchains/cmake-clang-v1/` — same tree Retro uses | | `docs/ci/` | Composite actions + [`templates/setup-release.yml`](ci/templates/setup-release.yml) | | `docs/GAME_PROJECT_SETUP.md` | Submodules, CI template usage, bundled-release checklist | | Game sources | `game.toml`, seeds, `CMakeLists.txt` at repo root; `psxrecomp/`, `recomp-ui/` submodules | -RetComM harvests emitters into the SDK cache and **downloads** `cmake-clang-v1` +Retro harvests emitters into the SDK cache and **downloads** `cmake-clang-v1` (no separate tools zip; game zips stay lean). The setup host and CLI install the -portable pack into the **shared RetComM cache** +portable pack into the **shared Retro cache** (`%LOCALAPPDATA%/retcomm/toolchains/cmake-clang-v1//`, or XDG equivalent) with host-native `curl` + `tar`/`unzip` when possible — so Microsoft Store Python AppData redirection cannot hide cmake. Offline zips unpack to the same diff --git a/docs/NETPLAY.md b/docs/NETPLAY.md index 9d8d68d4..8c4e5bbd 100644 --- a/docs/NETPLAY.md +++ b/docs/NETPLAY.md @@ -156,7 +156,7 @@ asymmetric NAT does not strand players on failed ICE attempts. signaling, and fallbacks. With `PSX_NETPLAY=ON`, `runtime.cmake` defaults `RNET_ENABLE_ICE=ON` (libjuice) before adding `recomp-net`. Pass `-DRNET_ENABLE_ICE=OFF` only for LAN-only / no-FetchContent builds. -libjuice is pulled as a **pinned URL tarball** (not `git clone`) so RetComM +libjuice is pulled as a **pinned URL tarball** (not `git clone`) so Retro AppImage / mismatched-libcurl hosts can still configure; offline builds can vendor `lib/recomp-net/third_party/libjuice` or set `-DRNET_LIBJUICE_ROOT`. “Force TURN” in the UI can raise delay floors for relay-heavy paths; it diff --git a/docs/ci/HOST_ONLY_RELEASES.md b/docs/ci/HOST_ONLY_RELEASES.md index 9bea7178..0abecf41 100644 --- a/docs/ci/HOST_ONLY_RELEASES.md +++ b/docs/ci/HOST_ONLY_RELEASES.md @@ -33,17 +33,17 @@ the portable llvm-mingw fetch; host compile stays on MSYS2 g++. Bump the `psxrecomp` gitlink when emitter/CLI behavior must change; leave it pinned for pure host/UI/`recomp-ui` releases so CI stays cmake-time. -## What players / RetComM do +## What players / Retro do | Step | When | |------|------| -| **Generate** (wizard or RetComM Build & Install) | First install, or after ROM/BIOS/emitter fingerprint changes | -| **Update** (RetComM) | New setup-host zip → refresh source → cmake rebuild; **codegen-cache** skips regenerate when fingerprints match | +| **Generate** (wizard or Retro Build & Install) | First install, or after ROM/BIOS/emitter fingerprint changes | +| **Update** (Retro) | New setup-host zip → refresh source → cmake rebuild; **codegen-cache** skips regenerate when fingerprints match | | **Generate & Rebuild** | Force disc→C again (`force_generate`) | Players do **not** need Generate & Rebuild for ordinary host/UI updates after the -first successful generate. RetComM keeps `apps//codegen-cache/` keyed by -ROM/BIOS + emitter fingerprints (see RetComM `docs/BUILD_PACKS.md`). +first successful generate. Retro keeps `apps/<title>/codegen-cache/` keyed by +ROM/BIOS + emitter fingerprints (see Retro `docs/BUILD_PACKS.md`). Raw zip extract is for true prebuilt Play binaries only. Setup-host catalog entries use local generate+cmake (`install_title_auto` / `update_title_auto`). diff --git a/docs/ci/README.md b/docs/ci/README.md index a498403a..54802c99 100644 --- a/docs/ci/README.md +++ b/docs/ci/README.md @@ -37,7 +37,7 @@ patch). Pass an explicit version to override. Shared helper: **Host-only model / player updates:** [`HOST_ONLY_RELEASES.md`](HOST_ONLY_RELEASES.md) -(CI never ships game C; RetComM Update reuses codegen-cache for host/UI bumps; +(CI never ships game C; Retro Update reuses codegen-cache for host/UI bumps; `reuse_cached_emitters` + ccache speed up release jobs when the `psxrecomp` pin is unchanged). diff --git a/docs/ci/templates/setup-release.yml b/docs/ci/templates/setup-release.yml index b6916125..41bedd46 100644 --- a/docs/ci/templates/setup-release.yml +++ b/docs/ci/templates/setup-release.yml @@ -21,7 +21,7 @@ # still loads the ICD dynamically via SDL. # # workflow_dispatch: leave version empty to auto-bump patch (or choose bump). -# Host-focused releases: CI never ships game C. Players Generate once; RetComM +# Host-focused releases: CI never ships game C. Players Generate once; Retro # Update reuses codegen-cache when ROM/BIOS/emitters match (cmake-only). # See docs/ci/HOST_ONLY_RELEASES.md. reuse_cached_emitters skips rebuilding # psxrecomp-game/bios when the psxrecomp SHA cache hits (host/UI bumps). @@ -554,7 +554,7 @@ jobs: export BPE_RUNTIME_BIN_DIR=/mingw64/bin fi export RELEASE_VERSION - # Lean zip (no embedded toolchain/). RetComM / wizard download + # Lean zip (no embedded toolchain/). Retro / wizard download # cmake-clang-v1; pass --embed-toolchain only for offline-first packs. scripts/package_setup_release.sh build-ci "${{ matrix.artifact }}" build-recompiler ls -la dist/ diff --git a/docs/config_schema.md b/docs/config_schema.md index 0454d4e1..c3d6c9e3 100644 --- a/docs/config_schema.md +++ b/docs/config_schema.md @@ -90,9 +90,9 @@ that is where the others go when a title needs them. Offline Play may still launch with a TOC warning; first-run setup Finish and online Create/Join require `netplay_ok` (and online also a clean verify + -non-empty `disc_fp`). Mirror `required_tracks` in the RetComM catalog as +non-empty `disc_fp`). Mirror `required_tracks` in the Retro catalog as `rom_identity.track_counts` so the hub library scan rejects Track-01-only dumps. -Wizard / RetComM / catalog submission accept Redump `.cue` + sibling `.bin` +Wizard / Retro / catalog submission accept Redump `.cue` + sibling `.bin` tracks only — not `.iso`/`.chd` (cannot reliably expand to multi-track). ## Program / game block diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c index b24d28a8..ac9ca0ba 100644 --- a/host/psxrecomp_codegen_host.c +++ b/host/psxrecomp_codegen_host.c @@ -475,7 +475,7 @@ static int python_env_usable(const char* env) { return 1; } -/* Prefer portable pack CPython (RetComM / cmake-clang-v1), then system. */ +/* Prefer portable pack CPython (Retro / cmake-clang-v1), then system. */ static int find_python(char* out, size_t cap) { const char* env = getenv("RETCOMM_PYTHON"); if (python_env_usable(env)) { @@ -998,7 +998,7 @@ static int name_is_releases(const char* name) { #endif } -/* RetComM stages Play under apps/<title>/releases/<tag>/ while the generate +/* Retro stages Play under apps/<title>/releases/<tag>/ while the generate * tree lives at apps/<title>/src/current/. Walking parents of the release dir * never visits that sibling — probe it explicitly. */ static int try_retcomm_src_current(const char* start, char* out, size_t cap) { @@ -1905,7 +1905,7 @@ static int run_cli_posix(char* const argv[], /* ---- Host-native toolchain install (no Store Python AppData redirect) ---- */ -static const char* k_tc_repo = "TechnicallyComputers/retcomm-toolchains"; +static const char* k_tc_repo = "RetroPortingToolKit/RetroPorting-Toolchains"; static const char* toolchain_zip_asset_name(void) { #if defined(_WIN32) @@ -3412,7 +3412,7 @@ static int host_toolchain_update_available(char* local_ver, size_t local_cap, /* Download or offline-install cmake-clang-v1 (wizard page 0 / rebuild fallback). * Prefer host-native curl/tar so Microsoft Store Python cannot redirect the - * unpack into Packages\\...\\LocalCache. Installs into the shared RetComM + * unpack into Packages\\...\\LocalCache. Installs into the shared Retro * cache: %LOCALAPPDATA%/retcomm/toolchains/cmake-clang-v1/… * Broken latest/ stamps are healed, then GitHub /releases/latest is fetched. * @@ -3520,7 +3520,7 @@ static int host_ensure_toolchain(RecompLauncherCPrepareProgressFn on_progress, * disc.cfg is the mounted-image cache and the runtime takes only its first * line; the hot-swap roster is built from game.toml [game] discs. So the * wizard's picks reach the roster only by being written there -- which is - * exactly what the RetComM path does by running probe_disc.py per image and + * exactly what the Retro path does by running probe_disc.py per image and * verify_disc_set.py over the results. * * update_disc_set.py performs the same probe/verify and then edits ONLY the diff --git a/lib/retcomm-rbengine b/lib/retcomm-rbengine index a7b98507..ebd94a47 160000 --- a/lib/retcomm-rbengine +++ b/lib/retcomm-rbengine @@ -1 +1 @@ -Subproject commit a7b98507a62fe00e5aec3b90c52a4134f3c174bc +Subproject commit ebd94a4729abe2c0615070cef3ffe05b3f9ebf28 diff --git a/psxrecomp_cli.py b/psxrecomp_cli.py index 2339242f..d251f4c9 100755 --- a/psxrecomp_cli.py +++ b/psxrecomp_cli.py @@ -593,7 +593,7 @@ def _build_recompiler_targets( ) src = recompiler_source_dir(project_root) - # Prefer recompiler/build (packaging / RetComM harvest layout); also keep + # Prefer recompiler/build (packaging / Retro harvest layout); also keep # project-root build-recompiler if that is where prior binaries lived. build_dir = src / "build" try: @@ -755,7 +755,7 @@ def ensure_framework( ) -> Path: """Ensure project_root/psxrecomp has BIOS profiles + seeds for local generate. - GitHub zipballs omit git submodules, so RetComM source trees often lack + GitHub zipballs omit git submodules, so Retro source trees often lack psxrecomp/bios. Seed from the SDK pack that ships this CLI (ROOT). """ fw = project_root / "psxrecomp" @@ -780,7 +780,7 @@ def ensure_framework( marker = fw / ".gitignore" if not marker.is_file(): marker.write_text( - "# RetComM SDK seed marker (project-root for psxrecomp-bios)\n", + "# Retro SDK seed marker (project-root for psxrecomp-bios)\n", encoding="utf-8", ) return framework_root(project_root) @@ -1765,7 +1765,7 @@ def cmd_pgo_train(args: argparse.Namespace, progress: ProgressReporter) -> int: def cmd_ensure_toolchain(args: argparse.Namespace, progress: ProgressReporter) -> int: - """Resolve / download / unpack cmake-clang-v1 into the shared RetComM cache.""" + """Resolve / download / unpack cmake-clang-v1 into the shared Retro cache.""" project_root = ( Path(args.project_root).expanduser().resolve() if args.project_root diff --git a/recompiler/src/analysis_export.h b/recompiler/src/analysis_export.h index c943ada9..eec14484 100644 --- a/recompiler/src/analysis_export.h +++ b/recompiler/src/analysis_export.h @@ -1,7 +1,7 @@ #pragma once // analysis_export.h — writers that turn an AnalysisDb into artifacts. // -// The JSON files are the stable public interface: RetComM Studio and any other +// The JSON files are the stable public interface: Retro Studio and any other // front end consume those, never the analyzer's in-process types. Everything // else here exists to hand the data to tools people already use rather than to // make them adopt a new one. diff --git a/recompiler/src/full_function_emitter.cpp b/recompiler/src/full_function_emitter.cpp index c15f32d4..e83b446a 100644 --- a/recompiler/src/full_function_emitter.cpp +++ b/recompiler/src/full_function_emitter.cpp @@ -2309,7 +2309,7 @@ EmitStats FullFunctionEmitter::emit( { EmitStats stats; - // Setup / RetComM zips omit generated/; create it before ofstream. + // Setup / Retro zips omit generated/; create it before ofstream. if (!out_dir.empty()) { std::error_code ec; std::filesystem::create_directories(out_dir, ec); diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 59edc928..3c93395e 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -31,7 +31,7 @@ endif() # SOURCE + compiler + flags (content, not mtime), so those recompiles collapse to # near-instant cache hits after any branch op. Completely no-op when ccache is not # on PATH, so builds still work without it. Set once, before any target is added. -# RetComM cmake-clang-v1 packs ship bin/ccache and prepend that dir to PATH; +# Retro cmake-clang-v1 packs ship bin/ccache and prepend that dir to PATH; # also HINT RETCOMM_TOOLCHAIN_DIR for wizards that only set the env override. if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER) set(_psx_ccache_hints "") @@ -103,7 +103,7 @@ set(PSX_SDL_LIBRARIES "") set(PSX_SDL_STATIC_LDFLAGS "") set(PSX_SDL3 OFF) -# Portable cmake-clang-v1 pack roots (wizard / RetComM / CI emitter fetch). +# Portable cmake-clang-v1 pack roots (wizard / Retro / CI emitter fetch). # Collect as HINTS only — never list(PREPEND CMAKE_PREFIX_PATH …): on Windows CI # the pack is llvm-mingw while the setup host links with MSYS2 g++, and a # global prefix puts pack lib/ on the -L path so -static-libstdc++ can pick up @@ -1447,7 +1447,7 @@ function(psxrecomp_add_runtime_target target) # ---- Windows / desktop app icon --------------------------------------- # Prefer an explicit APP_ICON, then the game-repo copy under assets/, then - # the framework default shipped in psxrecomp/assets (RetComM-themed pad). + # the framework default shipped in psxrecomp/assets (Retro-themed pad). if(NOT PSXRT_APP_ICON) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/psxrecomp.ico") set(PSXRT_APP_ICON "${CMAKE_CURRENT_SOURCE_DIR}/assets/psxrecomp.ico") diff --git a/tools/duckstation/README.md b/tools/duckstation/README.md index fa92cb7a..e7ee4a38 100644 --- a/tools/duckstation/README.md +++ b/tools/duckstation/README.md @@ -13,7 +13,7 @@ PSXRecomp v4 uses a patched build of [stenzek/duckstation](https://github.com/st ## First-time setup -- Linux / macOS Use `tools/duckstation_oracle.py`. It installs outside any game repo, into the -shared RetComM data root, so one build serves every title: +shared Retro data root, so one build serves every title: ```text ~/.local/share/retcomm/oracle/duckstation/ ($RETCOMM_DATA_DIR, or $RETCOMM_ORACLE_DIR) diff --git a/tools/duckstation_oracle.py b/tools/duckstation_oracle.py index 1fef9b44..415bc9e7 100644 --- a/tools/duckstation_oracle.py +++ b/tools/duckstation_oracle.py @@ -18,7 +18,7 @@ bracket where. Where it lives -------------- NOT in the game repo. The oracle is a developer tool shared by every recomp -title, so it installs into the RetComM data root alongside the toolchains and +title, so it installs into the Retro data root alongside the toolchains and catalog: ~/.local/share/retcomm/oracle/duckstation/ (Linux / macOS) @@ -26,7 +26,7 @@ catalog: %LOCALAPPDATA%\\retcomm\\oracle\\duckstation\\ (Windows fallback) Override with RETCOMM_ORACLE_DIR, or move the whole root with RETCOMM_DATA_DIR -(the same variables RetComM Launcher and Studio already honour). +(the same variables Retro Launcher and Studio already honour). <root>/src/ pinned upstream checkout with the oracle patch applied <root>/build/ cmake/ninja tree @@ -160,7 +160,7 @@ def log(msg: str) -> None: # --------------------------------------------------------------------------- def data_root() -> Path: - """The RetComM shared data root — same one the launcher and Studio use. + """The Retro shared data root — same one the launcher and Studio use. The precedence must match studio_runner.cpp's retcomm_data_dir() exactly, XDG_DATA_HOME included. If the two disagree, the GUI reports an oracle @@ -656,7 +656,7 @@ def cmd_build(args: argparse.Namespace) -> int: def find_bios(explicit: Optional[str]) -> Optional[Path]: """Locate a retail BIOS image for the oracle. - Order: what the caller passed, then the RetComM BIOS root, then the engine + Order: what the caller passed, then the Retro BIOS root, then the engine checkout this script lives in. Never downloads one — a BIOS dump is the user's own property and is not redistributable. """ @@ -1209,7 +1209,7 @@ def main(argv=None) -> int: p = sub.add_parser("status", help="where it is and what state it is in") p.add_argument("--json", action="store_true", - help="machine-readable, for RetComM Studio") + help="machine-readable, for Retro Studio") p.set_defaults(func=cmd_status) p = sub.add_parser("path", help="print the install root") diff --git a/tools/fetch_toolchain.sh b/tools/fetch_toolchain.sh index f04375a2..f42cc52b 100755 --- a/tools/fetch_toolchain.sh +++ b/tools/fetch_toolchain.sh @@ -18,7 +18,7 @@ set -euo pipefail ARTIFACT="" DL_DIR=".cache/toolchain-dl" OUT_DIR=".cache/toolchain-pack" -REPO="TechnicallyComputers/retcomm-toolchains" +REPO="RetroPortingToolKit/RetroPorting-Toolchains" usage() { sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//' diff --git a/tools/gpu_frame_capture.py b/tools/gpu_frame_capture.py index 7cd08303..1818eefb 100644 --- a/tools/gpu_frame_capture.py +++ b/tools/gpu_frame_capture.py @@ -21,7 +21,7 @@ freezing forwards, and it is the only thing that works for a glitch you cannot stop on. Writes <out>/<tag>.json (a psx-gpu-frame dump), <out>/<tag>.summary.json (the -compact attribution RetComM Studio reads), <out>/<tag>.png (the presented +compact attribution Retro Studio reads), <out>/<tag>.png (the presented frame), and <out>/<tag>.opcodes.json, so a later diff or layer render needs nothing still running. """ diff --git a/tools/gpu_parity.py b/tools/gpu_parity.py index 38057942..39abead4 100644 --- a/tools/gpu_parity.py +++ b/tools/gpu_parity.py @@ -27,7 +27,7 @@ there is nothing to do but restart the DuckStation side; that is reported rather than papered over. Getting the oracle: `python3 duckstation_oracle.py all` builds and installs a -patched DuckStation into the RetComM data root (once, ~10 minutes), and +patched DuckStation into the Retro data root (once, ~10 minutes), and `duckstation_oracle.py start --disc <cue>` runs it headless on 4371. Pass --start-oracle here to have this tool do that for you when nothing is listening. diff --git a/tools/new_project_layout/fill_tokens.py b/tools/new_project_layout/fill_tokens.py index 43e6c048..76c0a5d0 100755 --- a/tools/new_project_layout/fill_tokens.py +++ b/tools/new_project_layout/fill_tokens.py @@ -42,7 +42,7 @@ def install_dir_name(name: str) -> str: """Local checkout / launcher ``apps/`` folder — same slug as the GitHub repo name. New-project wizard must create this folder (not the display name with spaces) - so Studio catalog matching and RetComM ``install_dir_name`` stay consistent. + so Studio catalog matching and Retro ``install_dir_name`` stay consistent. """ return sanitize_github_name(name) diff --git a/tools/new_project_layout/probe_disc.py b/tools/new_project_layout/probe_disc.py index 959074b4..6cd2a04a 100755 --- a/tools/new_project_layout/probe_disc.py +++ b/tools/new_project_layout/probe_disc.py @@ -651,7 +651,7 @@ def main() -> int: ap.add_argument( "--write-catalog", default="", - help="write catalog_identity.json for RetComM / submission autofill", + help="write catalog_identity.json for Retro / submission autofill", ) ap.add_argument( "--write-seeds", diff --git a/tools/new_project_layout/project_studio/cli.py b/tools/new_project_layout/project_studio/cli.py index 934296ee..15281004 100644 --- a/tools/new_project_layout/project_studio/cli.py +++ b/tools/new_project_layout/project_studio/cli.py @@ -168,13 +168,13 @@ def cmd_gui(args: argparse.Namespace) -> int: # tools/new_project_layout/project_studio/cli.py → repo root repo = here.parents[3] if len(here.parents) > 3 else here.parents[2] for rel in ( - Path("build") / "RetComM-Studio", + Path("build") / "Retro-Studio", Path("build") / "retcomm-studio", - Path("build-release") / "RetComM-Studio", - Path("out") / "bin" / "RetComM-Studio", + Path("build-release") / "Retro-Studio", + Path("out") / "bin" / "Retro-Studio", ): candidates.append(repo / rel) - which = shutil.which("RetComM-Studio") or shutil.which("retcomm-studio") + which = shutil.which("Retro-Studio") or shutil.which("retcomm-studio") if which: candidates.append(Path(which)) @@ -186,10 +186,10 @@ def cmd_gui(args: argparse.Namespace) -> int: return int(subprocess.call(cmd)) print( - "RetComM Studio GUI is Dear ImGui (native).\n" + "Retro Studio GUI is Dear ImGui (native).\n" "Build it first:\n" " cmake -S . -B build && cmake --build build\n" - " ./build/RetComM-Studio\n" + " ./build/Retro-Studio\n" "Or set RETCOMM_STUDIO_BIN to the executable path.", file=sys.stderr, ) diff --git a/tools/new_project_layout/project_studio/detect.py b/tools/new_project_layout/project_studio/detect.py index 55cada0b..e4d4063d 100644 --- a/tools/new_project_layout/project_studio/detect.py +++ b/tools/new_project_layout/project_studio/detect.py @@ -740,7 +740,7 @@ def audit_project(root: Path) -> AuditReport: title="Disc / catalog identity", status=CheckStatus.WARN, severity=Severity.RECOMMENDED, - detail="No catalog_identity.json / [prepare_disc] — needed for RetComM gates.", + detail="No catalog_identity.json / [prepare_disc] — needed for Retro gates.", fix_op="probe_disc_refresh", ) ) @@ -785,7 +785,7 @@ def audit_project(root: Path) -> AuditReport: ) ) - # Default RetComM-themed app icon (Windows .ico + PNG) + # Default Retro-themed app icon (Windows .ico + PNG) app_ico = root / "assets" / "psxrecomp.ico" if app_ico.is_file(): checks.append( @@ -804,7 +804,7 @@ def audit_project(root: Path) -> AuditReport: title="assets/psxrecomp app icon", status=CheckStatus.WARN, severity=Severity.RECOMMENDED, - detail="Missing assets/psxrecomp.ico (RetComM-themed default pad icon).", + detail="Missing assets/psxrecomp.ico (Retro-themed default pad icon).", fix_op="ensure_app_icon", ) ) @@ -895,7 +895,7 @@ def audit_project(root: Path) -> AuditReport: ) ) - # README download badges + boxart + RetComM Launcher (idempotent migrate op) + # README download badges + boxart + Retro Launcher (idempotent migrate op) readme_path = root / "README.md" readme_text = _read(readme_path) if readme_path.is_file() else "" missing_readme: list[str] = [] @@ -907,7 +907,7 @@ def audit_project(root: Path) -> AuditReport: if not readme_has_boxart(readme_text): missing_readme.append("libretro boxart") if not readme_has_launcher(readme_text): - missing_readme.append("RetComM Launcher section") + missing_readme.append("Retro Launcher section") if not readme_has_raid(readme_text): missing_readme.append("R.A.I.D. Discord footer") if not (root / ".github" / "raid-discord.png").is_file(): @@ -932,7 +932,7 @@ def audit_project(root: Path) -> AuditReport: title="README download metrics / launcher / RAID / boxart", status=CheckStatus.PASS, severity=Severity.RECOMMENDED, - detail="Download badges, boxart, RetComM Launcher, and R.A.I.D. footer present.", + detail="Download badges, boxart, Retro Launcher, and R.A.I.D. footer present.", ) ) diff --git a/tools/new_project_layout/project_studio/gitops.py b/tools/new_project_layout/project_studio/gitops.py index 821ddef9..8e31f29b 100644 --- a/tools/new_project_layout/project_studio/gitops.py +++ b/tools/new_project_layout/project_studio/gitops.py @@ -16,9 +16,9 @@ from pathlib import Path from typing import Any DEFAULT_PSXRECOMP_URL = "https://github.com/mstan/psxrecomp.git" -DEFAULT_RECOMP_UI_URL = "https://github.com/mstan/recomp-ui.git" -DEFAULT_RECOMP_NET_URL = "https://github.com/TechnicallyComputers/recomp-net.git" -DEFAULT_RBENGINE_URL = "https://github.com/TechnicallyComputers/retcomm-rbengine.git" +DEFAULT_RECOMP_UI_URL = "https://github.com/RetroPortingToolKit/recomp-ui.git" +DEFAULT_RECOMP_NET_URL = "https://github.com/RetroPortingToolKit/recomp-net.git" +DEFAULT_RBENGINE_URL = "https://github.com/RetroPortingToolKit/rbengine.git" DEFAULT_BRANCH = "master" DEFAULT_NESTED_BRANCH = "main" KNOWN_SUBMODULES = ("psxrecomp", "recomp-ui") diff --git a/tools/new_project_layout/project_studio/gui.py b/tools/new_project_layout/project_studio/gui.py index 58029c24..ce3d266e 100644 --- a/tools/new_project_layout/project_studio/gui.py +++ b/tools/new_project_layout/project_studio/gui.py @@ -294,7 +294,7 @@ def _ensure_gui_deps() -> str | None: if _ctk_importable(): return None return ( - "Frozen RetComM Studio build is missing customtkinter.\n" + "Frozen Retro Studio build is missing customtkinter.\n" "Reinstall from a release package, or run from source." ) @@ -376,7 +376,7 @@ class ProjectStudioApp: def __init__(self, ctk, *, initial_root: Path | None = None) -> None: self.ctk = ctk self.root = ctk.CTk() - self.root.title("RetComM Studio") + self.root.title("Retro Studio") self.root.geometry("1100x820") self.root.minsize(900, 640) # Maximize after the first map so WMs honor zoomed/fullscreen hints. @@ -518,7 +518,7 @@ class ProjectStudioApp: header.pack(fill="x", padx=16, pady=(16, 8)) ctk.CTkLabel( header, - text="RetComM Studio", + text="Retro Studio", font=ctk.CTkFont(size=22, weight="bold"), ).pack(side="left") ctk.CTkLabel( @@ -1123,7 +1123,7 @@ class ProjectStudioApp: self._prompt_apply_updates(result) elif not silent_if_current: messagebox.showinfo( - "RetComM Studio", + "Retro Studio", result.message or "Everything is up to date.", parent=self.root, ) @@ -1146,10 +1146,10 @@ class ProjectStudioApp: "Updates are available:\n\n" + "\n".join(lines) + "\n\nApply now?\n" - "(Toolchain installs into the shared RetComM cache used by the " + "(Toolchain installs into the shared Retro cache used by the " "launcher and game apps.)" ) - if not messagebox.askyesno("RetComM Studio", body, parent=self.root): + if not messagebox.askyesno("Retro Studio", body, parent=self.root): return update_studio = bool(result.studio.available) @@ -1177,14 +1177,14 @@ class ProjectStudioApp: self._update_status_var.set("Update done" if not should_exit else "Restarting…") if should_exit: messagebox.showinfo( - "RetComM Studio", + "Retro Studio", summary + "\n\nStudio will exit to finish the update.", parent=self.root, ) self.root.after(300, self.root.destroy) else: messagebox.showinfo( - "RetComM Studio", summary, parent=self.root + "Retro Studio", summary, parent=self.root ) self.root.after(0, done) diff --git a/tools/new_project_layout/project_studio/naming.py b/tools/new_project_layout/project_studio/naming.py index a5e2a0e5..82c9c638 100644 --- a/tools/new_project_layout/project_studio/naming.py +++ b/tools/new_project_layout/project_studio/naming.py @@ -297,7 +297,7 @@ def build_token_map( "EXE_BASENAME": exe_basename(title), "ZIP_PREFIX": zp, "GITHUB_OWNER": sanitize_github_name( - (github_owner or "").strip() or "TechnicallyComputers" + (github_owner or "").strip() or "RetroPortingToolKit" ), "GITHUB_REPO": sanitize_github_name((github_repo or "").strip() or name), "DISC_HINT": f"your legally owned {game} disc", diff --git a/tools/new_project_layout/project_studio/ops.py b/tools/new_project_layout/project_studio/ops.py index fe2cd43e..ed4c00a6 100644 --- a/tools/new_project_layout/project_studio/ops.py +++ b/tools/new_project_layout/project_studio/ops.py @@ -1259,7 +1259,7 @@ def _ensure_boxart_png(root: Path, options: MigrateOptions) -> list[str]: def op_patch_readme_metrics(root: Path, options: MigrateOptions) -> ApplyResult: - """Upsert download badges, libretro boxart, RetComM Launcher, and R.A.I.D. footer.""" + """Upsert download badges, libretro boxart, Retro Launcher, and R.A.I.D. footer.""" from .readme_metrics import ( apply_github_about, boxart_png_present, @@ -1313,7 +1313,7 @@ def op_patch_readme_metrics(root: Path, options: MigrateOptions) -> ApplyResult: parts: list[str] = [] if changed: parts.append( - f"Patched README badges, boxart, RetComM Launcher, and R.A.I.D. footer ({owner}/{repo})" + f"Patched README badges, boxart, Retro Launcher, and R.A.I.D. footer ({owner}/{repo})" ) if fetched: parts.append("fetched libretro boxart") diff --git a/tools/new_project_layout/project_studio/plan.py b/tools/new_project_layout/project_studio/plan.py index 6c46e5e4..000da746 100644 --- a/tools/new_project_layout/project_studio/plan.py +++ b/tools/new_project_layout/project_studio/plan.py @@ -54,7 +54,7 @@ _OP_TITLES = { "annotate_legacy_packaging": "Annotate legacy prebuilt packaging", "probe_disc_refresh": "Refresh disc identity via probe_disc.py", "record_framework_pins": "Write framework_pins.txt", - "patch_readme_metrics": "Patch README badges, RetComM Launcher, and R.A.I.D. footer", + "patch_readme_metrics": "Patch README badges, Retro Launcher, and R.A.I.D. footer", } diff --git a/tools/new_project_layout/project_studio/readme_metrics.py b/tools/new_project_layout/project_studio/readme_metrics.py index bf771a50..09c7a38c 100644 --- a/tools/new_project_layout/project_studio/readme_metrics.py +++ b/tools/new_project_layout/project_studio/readme_metrics.py @@ -1,4 +1,4 @@ -"""README download-badge + RetComM Launcher blocks for scaffold and migrate.""" +"""README download-badge + Retro Launcher blocks for scaffold and migrate.""" from __future__ import annotations @@ -40,8 +40,11 @@ _RAID_RE = re.compile( re.escape(RAID_BEGIN) + r".*?" + re.escape(RAID_END) + r"\n?", re.S, ) +# Both headings: this section was "## RetComM Launcher" until the rename, and +# every README generated before it still says so. Matching only the new one +# would leave the old block in place and append a second one beside it. _LEGACY_LAUNCHER_RE = re.compile( - r"^## RetComM Launcher\n.*?(?=^## |\Z)", + r"^## (?:Retro|RetComM) Launcher\n.*?(?=^## |\Z)", re.M | re.S, ) _LEGACY_RAID_RE = re.compile( @@ -53,8 +56,8 @@ _REMOTE_RE = re.compile( re.I, ) -DEFAULT_GITHUB_OWNER = "TechnicallyComputers" -LAUNCHER_REPO = "https://github.com/TechnicallyComputers/RetComM-Launcher" +DEFAULT_GITHUB_OWNER = "RetroPortingToolKit" +LAUNCHER_REPO = "https://github.com/RetroPortingToolKit/Retro-Launcher" def parse_github_remote(url: str) -> tuple[str, str] | None: @@ -185,33 +188,35 @@ def render_boxart_block(game_name: str) -> str: def render_launcher_block() -> str: - shots = ( - "https://raw.githubusercontent.com/TechnicallyComputers/" - "RetComM-Launcher/main/docs/screenshots" - ) + # Derived from LAUNCHER_REPO rather than spelled out again: this was a + # second copy of the slug, split across two string literals, and the org + # rename swept past it — every regenerated README kept the dead owner. + shots = LAUNCHER_REPO.replace( + "https://github.com/", "https://raw.githubusercontent.com/" + ) + "/main/docs/screenshots" return "\n".join( [ LAUNCHER_BEGIN, - "## RetComM Launcher", + "## Retro Launcher", "", "You can run this title **standalone** (release zip + the built-in recomp-ui", "Generate & Build flow), or manage installs, updates, ROM/BIOS wiring, and queued", "builds more intuitively with", - f"**[RetComM Launcher]({LAUNCHER_REPO})** —", + f"**[Retro Launcher]({LAUNCHER_REPO})** —", "the Retro Compilation Manager hub for self-compiling recomps.", "", f"[Downloads]({LAUNCHER_REPO}/releases) ·", f"[Full README & features]({LAUNCHER_REPO}#readme)", "", "<p align=\"center\">", - f' <img src="{shots}/hub-and-game-launcher.png" alt="RetComM hub with a background build, next to a title’s recomp-ui launcher" width="720">', + f' <img src="{shots}/hub-and-game-launcher.png" alt="Retro hub with a background build, next to a title’s recomp-ui launcher" width="720">', "</p>", "", "<p align=\"center\">", f' <img src="{shots}/queue-and-background-build.png" alt="Background cmake build with titles queued" width="720">', "</p>", "", - "RetComM checks for updates, rebuilds with existing build data when possible,", + "Retro checks for updates, rebuilds with existing build data when possible,", "shares the portable toolchain used by per-title launchers, and automates", "BIOS/ROM/save plumbing so you are not stuck repeating each game’s wizard by hand.", LAUNCHER_END, @@ -250,7 +255,17 @@ def boxart_png_present(root: Path) -> bool: def readme_has_launcher(text: str) -> bool: - return "TechnicallyComputers/RetComM-Launcher" in text + # Both slugs: every port generated before the RetroPortingToolKit transfer + # carries the old one, and matching only the new one would report each of + # them as missing its launcher section — and append a second copy to any + # README that has neither the markers nor a "## Retro Launcher" heading. + return any( + slug in text + for slug in ( + "RetroPortingToolKit/Retro-Launcher", + "TechnicallyComputers/RetComM-Launcher", + ) + ) def readme_has_raid(text: str) -> bool: diff --git a/tools/new_project_layout/setup_project.ps1 b/tools/new_project_layout/setup_project.ps1 index 051cf695..c0b1d87e 100644 --- a/tools/new_project_layout/setup_project.ps1 +++ b/tools/new_project_layout/setup_project.ps1 @@ -146,9 +146,9 @@ if (-not $ZipPrefix) { if (-not $GithubOwner) { if ($interactive) { - $GithubOwner = Prompt-Line "GitHub owner / org (README download badges)" "TechnicallyComputers" + $GithubOwner = Prompt-Line "GitHub owner / org (README download badges)" "RetroPortingToolKit" } else { - $GithubOwner = "TechnicallyComputers" + $GithubOwner = "RetroPortingToolKit" } } $derivedRepo = (python -c "from fill_tokens import sanitize_github_name; import sys; print(sanitize_github_name(sys.argv[1]))" $Name).Trim() @@ -293,7 +293,7 @@ set(PSX_RECOMP_UI OFF CACHE BOOL if ($useNetplay) { # PSX_NETPLAY defaults RNET_ENABLE_ICE=ON; recomp-net FetchContents - # libjuice via pinned URL (not git) so RetComM AppImage builds configure. + # libjuice via pinned URL (not git) so Retro AppImage builds configure. Set-Content -Encoding UTF8 -Path $NetplayBlockFile -Value @" if(EXISTS "`${PSXRECOMP_ROOT}/lib/recomp-net/CMakeLists.txt") set(PSX_NETPLAY ON CACHE BOOL @@ -430,7 +430,7 @@ if ($useRecompUi) { } git submodule update --init --recursive -# RetComM-themed default app icon (Windows .ico + PNG for packaging). +# Retro-themed default app icon (Windows .ico + PNG for packaging). $iconSrc = Join-Path $Root "psxrecomp\assets" $iconDst = Join-Path $Root "assets" if (Test-Path $iconSrc) { diff --git a/tools/new_project_layout/setup_project.sh b/tools/new_project_layout/setup_project.sh index 8011a311..5c4d7efa 100755 --- a/tools/new_project_layout/setup_project.sh +++ b/tools/new_project_layout/setup_project.sh @@ -9,7 +9,7 @@ # --stage-disc Copy full cue+bins into repo disc/ (large; optional) # --no-stage-disc Default: probe in place, extract boot EXE only # --psxrecomp-ref / --recomp-ui-ref / --recomp-net-ref / URLs -# --github-owner <org> README download-badge owner (default TechnicallyComputers) +# --github-owner <org> README download-badge owner (default RetroPortingToolKit) # --github-repo <name> README download-badge repo (default project name) # # Everything else is prompted on a TTY (or passed via flags / --yes defaults): @@ -272,10 +272,10 @@ fi if [ -z "$GITHUB_OWNER" ]; then if [ "$YES_MODE" -eq 1 ] || ! is_tty; then - GITHUB_OWNER=TechnicallyComputers + GITHUB_OWNER=RetroPortingToolKit else prompt_line "GitHub owner / org (README download badges)" GITHUB_OWNER \ - "TechnicallyComputers" + "RetroPortingToolKit" fi fi if [ -z "$GITHUB_REPO" ]; then @@ -512,7 +512,7 @@ fi if [ "$ENABLE_NETPLAY" -eq 1 ]; then # PSX_NETPLAY defaults RNET_ENABLE_ICE=ON; recomp-net FetchContents - # libjuice via pinned URL (not git) so RetComM AppImage builds configure. + # libjuice via pinned URL (not git) so Retro AppImage builds configure. printf '%s\n' \ 'if(EXISTS "${PSXRECOMP_ROOT}/lib/recomp-net/CMakeLists.txt") set(PSX_NETPLAY ON CACHE BOOL @@ -651,7 +651,7 @@ if [ "$ENABLE_RECOMP_UI" -eq 1 ]; then fi git submodule update --init --recursive -# RetComM-themed default app icon (Windows .ico + PNG for packaging). +# Retro-themed default app icon (Windows .ico + PNG for packaging). if [ -d psxrecomp/assets ]; then mkdir -p "$ROOT/assets" for _icon in psxrecomp.svg psxrecomp.png psxrecomp.ico; do diff --git a/tools/new_project_layout/templates/README.md.in b/tools/new_project_layout/templates/README.md.in index 3f31de03..cafc39da 100644 --- a/tools/new_project_layout/templates/README.md.in +++ b/tools/new_project_layout/templates/README.md.in @@ -23,26 +23,26 @@ Scaffolded with the New Project Layout. See `psxrecomp/docs/GAME_PROJECT_SETUP.md` for the full flow. <!-- retcomm-readme-launcher --> -## RetComM Launcher +## Retro Launcher You can run this title **standalone** (release zip + the built-in recomp-ui Generate & Build flow), or manage installs, updates, ROM/BIOS wiring, and queued builds more intuitively with -**[RetComM Launcher](https://github.com/TechnicallyComputers/RetComM-Launcher)** — +**[Retro Launcher](https://github.com/RetroPortingToolKit/Retro-Launcher)** — the Retro Compilation Manager hub for self-compiling recomps. -[Downloads](https://github.com/TechnicallyComputers/RetComM-Launcher/releases) · -[Full README & features](https://github.com/TechnicallyComputers/RetComM-Launcher#readme) +[Downloads](https://github.com/RetroPortingToolKit/Retro-Launcher/releases) · +[Full README & features](https://github.com/RetroPortingToolKit/Retro-Launcher#readme) <p align="center"> - <img src="https://raw.githubusercontent.com/TechnicallyComputers/RetComM-Launcher/main/docs/screenshots/hub-and-game-launcher.png" alt="RetComM hub with a background build, next to a title’s recomp-ui launcher" width="720"> + <img src="https://raw.githubusercontent.com/RetroPortingToolKit/Retro-Launcher/main/docs/screenshots/hub-and-game-launcher.png" alt="Retro hub with a background build, next to a title’s recomp-ui launcher" width="720"> </p> <p align="center"> - <img src="https://raw.githubusercontent.com/TechnicallyComputers/RetComM-Launcher/main/docs/screenshots/queue-and-background-build.png" alt="Background cmake build with titles queued" width="720"> + <img src="https://raw.githubusercontent.com/RetroPortingToolKit/Retro-Launcher/main/docs/screenshots/queue-and-background-build.png" alt="Background cmake build with titles queued" width="720"> </p> -RetComM checks for updates, rebuilds with existing build data when possible, +Retro checks for updates, rebuilds with existing build data when possible, shares the portable toolchain used by per-title launchers, and automates BIOS/ROM/save plumbing so you are not stuck repeating each game’s wizard by hand. <!-- /retcomm-readme-launcher --> @@ -53,7 +53,7 @@ You must own the original game. Disc images under `disc/` are gitignored and must never be committed. Retail BIOS dumps are not redistributed; OpenBIOS is used for Generate unless you supply your own SCPH locally. -Default app icon: `assets/psxrecomp.ico` (and `.png` / `.svg`) — RetComM-themed controller mark from `psxrecomp/assets/`. Windows builds embed it via `APP_ICON`. +Default app icon: `assets/psxrecomp.ico` (and `.png` / `.svg`) — Retro-themed controller mark from `psxrecomp/assets/`. Windows builds embed it via `APP_ICON`. Optional box art under `launcher_assets/img/` may come from [libretro-thumbnails](https://github.com/libretro-thumbnails/libretro-thumbnails) diff --git a/tools/new_project_layout/update_disc_set.py b/tools/new_project_layout/update_disc_set.py index d674f26a..92156233 100755 --- a/tools/new_project_layout/update_disc_set.py +++ b/tools/new_project_layout/update_disc_set.py @@ -17,7 +17,7 @@ player located and update only the keys that describe the set: Everything else in the file is preserved byte-for-byte, comments included. This is the step that makes the standalone wizard reach the same end state as -the RetComM path, which runs probe_disc.py per disc and verify_disc_set.py over +the Retro path, which runs probe_disc.py per disc and verify_disc_set.py over the results. The runtime's hot-swap roster is built from [game] discs -- not from disc.cfg, which is only the mounted-image cache -- so without this the wizard can record every disc a player owns and none of them reach the roster. @@ -212,7 +212,7 @@ def main() -> int: print(f"error: missing disc image {c}", file=sys.stderr) return 1 - # Probe each disc, writing the same per-disc artifacts the RetComM path + # Probe each disc, writing the same per-disc artifacts the Retro path # leaves behind (disc_probe.json, disc_probe.2.json, ...). probes, probe_jsons = [], [] for i, cue in enumerate(cues, start=1): diff --git a/tools/package_setup_host.sh b/tools/package_setup_host.sh index 19dbd8c3..7482e9fa 100755 --- a/tools/package_setup_host.sh +++ b/tools/package_setup_host.sh @@ -3,7 +3,7 @@ # # Stages the host exe, title sources, filtered psxrecomp/ + recomp-ui/, then # finishes with stage_setup_sdk.sh (emitters, OpenBIOS, MinGW DLLs). -# Portable cmake/clang is NOT embedded by default — RetComM / the setup wizard +# Portable cmake/clang is NOT embedded by default — Retro / the setup wizard # download cmake-clang-v1 from retcomm-toolchains (or accept an offline zip). # # Usage (from game repo root): @@ -262,7 +262,7 @@ ZIP_NAME="${ZIP_PREFIX}-${VERSION}-${ARTIFACT}.zip" rm -f "${DIST}/${ZIP_NAME}" cp -a "${EXE}" "${STAGE}/" -# Ship the stamp beside the exe so installers / RetComM can prefer it over VERSION. +# Ship the stamp beside the exe so installers / Retro can prefer it over VERSION. if [[ -f "$(dirname "${EXE}")/psx_game_version.txt" ]]; then cp -a "$(dirname "${EXE}")/psx_game_version.txt" "${STAGE}/psx_game_version.txt" else @@ -498,11 +498,11 @@ Standalone: 3. Provide ${DISC_HINT} (and optional retail SCPH-1001 BIOS; otherwise OpenBIOS is regenerated locally). 4. Follow the Generate & rebuild wizard. On first rebuild the host downloads - cmake-clang-v1 from TechnicallyComputers/retcomm-toolchains (or you can + cmake-clang-v1 from RetroPortingToolKit/RetroPorting-Toolchains (or you can pick a local cmake-clang-v1-*.zip for offline builds). System cmake/ninja also works if already on PATH. -RetComM uses this same zip: it harvests emitters into a shared SDK cache, +Retro uses this same zip: it harvests emitters into a shared SDK cache, downloads the toolchain pack (or uses RETCOMM_TOOLCHAIN_DIR), and preserves saves/user config across updates. EOF diff --git a/tools/prepare_disc.py b/tools/prepare_disc.py index ea1e62fe..40a7aa72 100644 --- a/tools/prepare_disc.py +++ b/tools/prepare_disc.py @@ -13,7 +13,7 @@ Accepted inputs: Writes working ``.bin`` + ``.cue`` (or copies a multi-track Redump set), extracts ``SYSTEM.CNF`` + boot EXE, and prints ``RESULT_CUE=<abs path>`` -for the first-run wizard / RetComM. +for the first-run wizard / Retro. ISO→2352 sets Mode2 Form1 sync/header/subheader/EDC (ECC zeroed — fine for software readers). Rebuilt images are not bit-identical to Redump. diff --git a/tools/psx_gpu_frame.py b/tools/psx_gpu_frame.py index 2fcb7a9e..cb40219f 100644 --- a/tools/psx_gpu_frame.py +++ b/tools/psx_gpu_frame.py @@ -674,7 +674,7 @@ def summarise_dump(dump: Dict[str, Any], dump_name: str = "") -> Dict[str, Any]: A busy frame's full dump runs to tens of megabytes. Nothing that only wants to know *who drew what* should have to parse that, so the summary is a - separate artifact and RetComM Studio reads this rather than the dump. + separate artifact and Retro Studio reads this rather than the dump. """ ops: Dict[str, int] = {} modes: Dict[str, int] = {} diff --git a/tools/stage_setup_sdk.sh b/tools/stage_setup_sdk.sh index 4dd91b85..2a739dee 100755 --- a/tools/stage_setup_sdk.sh +++ b/tools/stage_setup_sdk.sh @@ -3,7 +3,7 @@ # into an existing setup-host release stage directory. # # Title packagers copy the host exe, game sources, and framework tree first, -# then call this to finish the RetComM/wizard-complete zip layout. +# then call this to finish the Retro/wizard-complete zip layout. # # Usage: # stage_setup_sdk.sh --stage <stage-dir> [options] @@ -235,7 +235,7 @@ if [[ -n "${TOOLCHAIN_DIR}" && -d "${TOOLCHAIN_DIR}" ]]; then fi echo "bundled toolchain from ${TOOLCHAIN_DIR}" elif [[ "${ALLOW_NO_TOOLCHAIN}" -eq 1 ]]; then - echo "note: no embedded toolchain/ — RetComM/wizard will download cmake-clang-v1" \ + echo "note: no embedded toolchain/ — Retro/wizard will download cmake-clang-v1" \ "(or accept an offline zip / system cmake)" >&2 else echo "error: toolchain dir required (pass --toolchain-dir or set PSXRECOMP_TOOLCHAIN_DIR)" >&2 diff --git a/tools/tests/test_duckstation_oracle.py b/tools/tests/test_duckstation_oracle.py index 67e73618..a1435e46 100644 --- a/tools/tests/test_duckstation_oracle.py +++ b/tools/tests/test_duckstation_oracle.py @@ -193,7 +193,7 @@ class TestSettingsMerge(unittest.TestCase): class TestStatusDoc(EnvGuard): - """The status document is the whole contract with RetComM Studio.""" + """The status document is the whole contract with Retro Studio.""" def test_absent_install_reports_absent(self): os.environ["RETCOMM_ORACLE_DIR"] = tempfile.mkdtemp() diff --git a/tools/toolchain_pack.py b/tools/toolchain_pack.py index 93d6780a..c2938cab 100644 --- a/tools/toolchain_pack.py +++ b/tools/toolchain_pack.py @@ -1,6 +1,6 @@ """Resolve / download / unpack portable cmake-clang-v1 toolchain packs. -Shared cache matches RetComM / retcomm-toolchains install.sh: +Shared cache matches Retro / retcomm-toolchains install.sh: Windows: %LOCALAPPDATA%/retcomm/toolchains/cmake-clang-v1/<tag>/ Linux/macOS: $XDG_DATA_HOME/retcomm/… and ~/.local/share/retcomm/… @@ -24,7 +24,7 @@ import zipfile from pathlib import Path from typing import Optional -DEFAULT_REPO = "TechnicallyComputers/retcomm-toolchains" +DEFAULT_REPO = "RetroPortingToolKit/RetroPorting-Toolchains" PACK_ID = "cmake-clang-v1" _ASSET = { @@ -260,7 +260,7 @@ def shared_cache_roots() -> list[Path]: Layout matches retcomm-toolchains install.sh: …/retcomm/toolchains/cmake-clang-v1/<tag>/ plus latest → current - RetComM (`retcomm`) is preferred; legacy `psxrecomp` remains a read/migrate + Retro (`retcomm`) is preferred; legacy `psxrecomp` remains a read/migrate fallback. Honors RETCOMM_TOOLCHAIN_CACHE / RETCOMM_DATA_HOME. """ roots: list[Path] = [] @@ -281,7 +281,7 @@ def shared_cache_roots() -> list[Path]: def preferred_install_root() -> Path: - """Where newly downloaded / offline-unpacked packs land (RetComM shared).""" + """Where newly downloaded / offline-unpacked packs land (Retro shared).""" cache = (os.environ.get("RETCOMM_TOOLCHAIN_CACHE") or "").strip() if cache: r = Path(cache).expanduser() @@ -747,7 +747,7 @@ def _register_user_path_unix(cache_root: Path, latest: Path, log=None) -> None: ) block = ( f"{_MARKER_BEGIN}\n" - f"# Managed by psxrecomp / RetComM — remove via uninstall.sh\n" + f"# Managed by psxrecomp / Retro — remove via uninstall.sh\n" f'if [ -f "{hook}" ]; then\n' f" # shellcheck disable=SC1091\n" f' . "{hook}"\n' @@ -775,7 +775,7 @@ def _register_user_path_unix(cache_root: Path, latest: Path, log=None) -> None: def register_toolchain_user_env(pack_root: Path, log=None) -> Path: """Refresh ``latest`` + idempotently add ``latest/bin`` to the user login PATH. - Mirrors retcomm-toolchains zip ``install.sh`` / ``install.ps1`` so RetComM + Mirrors retcomm-toolchains zip ``install.sh`` / ``install.ps1`` so Retro setup hosts and the standalone wizard leave cmake/clang on PATH for shell development after placing the pack in the shared cache. """ @@ -953,7 +953,7 @@ def ensure_toolchain( Resolution order: env override → project stamp/toolchain/ → shared retcomm cache (legacy psxrecomp migrated) → optional --from-zip → download. - Installs always land under the shared RetComM cache; the project only gets + Installs always land under the shared Retro cache; the project only gets a stamp file pointing at that pack. """ need = min_version or default_min_version() From 84bc48df5b9541465478fa0f99ed8d0b958a2820 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen <alex@technicallycomputers.ca> Date: Sat, 12 Sep 2026 19:14:50 -0400 Subject: [PATCH 08/56] sync --- lib/recomp-net | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/recomp-net b/lib/recomp-net index bfa20b40..859ba28b 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit bfa20b403a6d71d01685e8e0a8100d63026716bc +Subproject commit 859ba28b93f7374f2ac1eb70d2171594c5cd4f4b From fddc8b8516c14334d9c74816a361963bcc7650e4 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen <alex@technicallycomputers.ca> Date: Sat, 12 Sep 2026 19:29:57 -0400 Subject: [PATCH 09/56] netplay: report chat lines for moderation, and pin recomp-net for it PsxLobbyChatMsg gains the server's `mid` for a line, taken off `chat` and `server_chat`, and psx_lobby_report_chat hands a list of them to rnet_chat_report_build. The frame carries ids and never the text, so a report cannot fabricate what somebody said. Same eight lines as the SNES copy; what a report contains lives in recomp-net. lib/recomp-net -> 859ba28 (support chat reporting), which is also where rnet_account_set_secret_path comes from. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FPXfGGhBBBxQPcEzaMsauF --- lib/recomp-net | 2 +- runtime/include/psx_lobby_client.h | 23 ++++++++++++ runtime/src/psx_lobby_client.c | 60 ++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/lib/recomp-net b/lib/recomp-net index b4c58fcf..859ba28b 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit b4c58fcfcdd3faadb3ba74da9a138ec1955c4540 +Subproject commit 859ba28b93f7374f2ac1eb70d2171594c5cd4f4b diff --git a/runtime/include/psx_lobby_client.h b/runtime/include/psx_lobby_client.h index 9242daa1..b70521a3 100644 --- a/runtime/include/psx_lobby_client.h +++ b/runtime/include/psx_lobby_client.h @@ -128,6 +128,15 @@ typedef struct PsxLobbyChatMsg { char player_id[PSX_LOBBY_ID_LEN]; char from[PSX_LOBBY_NAME_LEN]; char text[PSX_LOBBY_CHAT_TEXT_LEN]; + /* The SERVER's id for this line, and what psx_lobby_report_chat names. + * Empty for a system line, a locally generated one, and anything from a + * server too old to assign one -- none of which can be reported, because + * there is no referent both sides agree on. + * + * A report carries this and NOT the text: the server writes down what it + * relayed under this id. Sending the words would let a client fabricate a + * message and have somebody sanctioned for it. */ + char mid[40]; int is_local; int is_system; uint32_t seq; @@ -400,6 +409,20 @@ void psx_lobby_seat_swap_clear(void); /* Lobby chat. send: 0 when queued (the line appears via the server echo). * count/get read the ring, oldest first; cleared on create/join/leave. */ +/* Report chat lines for moderation. Same contract on every console -- the + * frame is built by recomp-net (recomp_net/chat_report.h), which is also where + * the RNET_REPORT_* categories live; this only says where we are and sends it. + * + * `mids` are PsxLobbyChatMsg.mid values, several at once because harassment is + * usually a burst rather than a line. Entries with an empty id are skipped. + * The text is never sent: see PsxLobbyChatMsg.mid. + * + * 0 when handed to the server, which is not the same as accepted -- it refuses + * an expired line, a guest's line, your own, and anything over the rate + * limit. */ +int psx_lobby_report_chat(const char *const *mids, int mid_count, + const char *reason, const char *note); + int psx_lobby_send_chat(const char *text); /* Server chat: per-game, outside any room (op server_chat). Its own ring, diff --git a/runtime/src/psx_lobby_client.c b/runtime/src/psx_lobby_client.c index 73ff7350..e0615747 100644 --- a/runtime/src/psx_lobby_client.c +++ b/runtime/src/psx_lobby_client.c @@ -1,4 +1,5 @@ #include "psx_lobby_client.h" +#include "recomp_net/chat_report.h" #include <ctype.h> #include <errno.h> @@ -122,6 +123,9 @@ const PsxLobbyMemcardOffer *psx_lobby_memcard_offer(void) return &z; } int psx_lobby_send_chat(const char *text) { (void)text; return -1; } +int psx_lobby_report_chat(const char *const *m, int c, const char *r, + const char *n) +{ (void)m; (void)c; (void)r; (void)n; return -1; } int psx_lobby_seat_move_self(int to_slot) { (void)to_slot; return -1; } int psx_lobby_seat_swap_request(int target_slot) { (void)target_slot; return -1; } int psx_lobby_seat_swap_incoming(char *who, size_t who_cap, int *from_slot) @@ -1803,7 +1807,7 @@ static void drain_ws_pending(void) } static void chat_push(const char *player_id, const char *from, const char *text, - int is_system) + const char *mid, int is_system) { PsxLobbyChatMsg *m; int idx; @@ -1820,6 +1824,8 @@ static void chat_push(const char *player_id, const char *from, const char *text, snprintf(m->player_id, sizeof(m->player_id), "%s", player_id ? player_id : ""); snprintf(m->from, sizeof(m->from), "%s", from ? from : ""); snprintf(m->text, sizeof(m->text), "%s", text); + snprintf(m->mid, sizeof(m->mid), "%s", + (mid && !is_system) ? mid : ""); /* Masked on arrival, whatever relayed it: the server already did this, * an older server did not, and the rule is that nothing unmasked is * ever shown. */ @@ -1830,7 +1836,8 @@ static void chat_push(const char *player_id, const char *from, const char *text, m->seq = ++g_lc.chat_seq; } -static void schat_push(const char *player_id, const char *from, const char *text) +static void schat_push(const char *player_id, const char *from, const char *text, + const char *mid) { PsxLobbyChatMsg *m; int idx; @@ -1847,6 +1854,7 @@ static void schat_push(const char *player_id, const char *from, const char *text snprintf(m->player_id, sizeof(m->player_id), "%s", player_id ? player_id : ""); snprintf(m->from, sizeof(m->from), "%s", from ? from : ""); snprintf(m->text, sizeof(m->text), "%s", text); + snprintf(m->mid, sizeof(m->mid), "%s", mid ? mid : ""); (void)rnet_chat_filter_apply(m->text, sizeof(m->text)); m->is_local = (g_lc.player_id[0] && player_id && strcmp(player_id, g_lc.player_id) == 0) ? 1 : 0; @@ -2299,26 +2307,32 @@ static void handle_server_json(const char *json) char text[PSX_LOBBY_CHAT_TEXT_LEN]; char from_id[PSX_LOBBY_ID_LEN]; char from[PSX_LOBBY_NAME_LEN]; + char mid[40]; text[0] = '\0'; from_id[0] = '\0'; from[0] = '\0'; + mid[0] = '\0'; json_get_str(json, "text", text, sizeof(text)); json_get_str(json, "from_player_id", from_id, sizeof(from_id)); json_get_str(json, "from", from, sizeof(from)); - schat_push(from_id, from, text); + json_get_str(json, "mid", mid, sizeof(mid)); + schat_push(from_id, from, text, mid); return; } if (strcmp(op, "chat") == 0) { char text[PSX_LOBBY_CHAT_TEXT_LEN]; char from_id[PSX_LOBBY_ID_LEN]; char from[PSX_LOBBY_NAME_LEN]; + char mid[40]; text[0] = '\0'; from_id[0] = '\0'; from[0] = '\0'; + mid[0] = '\0'; json_get_str(json, "text", text, sizeof(text)); json_get_str(json, "from_player_id", from_id, sizeof(from_id)); json_get_str(json, "from", from, sizeof(from)); - chat_push(from_id, from, text, json_get_bool(json, "system", 0)); + json_get_str(json, "mid", mid, sizeof(mid)); + chat_push(from_id, from, text, mid, json_get_bool(json, "system", 0)); return; } if (strcmp(op, "signal") == 0) { @@ -3707,6 +3721,44 @@ void psx_lobby_seat_swap_clear(void) if (g_lc.swap_out != 1) g_lc.swap_out = 0; } +int psx_lobby_report_chat(const char *const *mids, int mid_count, + const char *reason, const char *note) +{ + /* Thin on purpose, and identical in shape to the SNES copy. What a report + * CONTAINS lives in recomp-net (recomp_net/chat_report.h) so there is one + * implementation rather than one per console; this says where we are and + * hands the frame to the socket. */ + RNetChatReportMeta meta; + char msg[2048]; + size_t n; + + if (!psx_lobby_connected()) + return -1; + + memset(&meta, 0, sizeof(meta)); + meta.game = g_lc.filter_game_name; + meta.game_version = psx_lobby_game_version(); + /* Metadata only. Nothing downstream may name a file or a directory after + * it -- one moderation queue spans every title, and splitting the evidence + * by console would fragment it along a line that has nothing to do with + * moderation. */ + meta.platform = "psx"; + /* The host this client connected to. PSX keeps the parsed host rather + * than the whole URL, which is the part that identifies a deployment and + * the part a moderator needs. */ + meta.server = g_lc.host; + meta.lobby = g_lc.join.lobby_id[0] ? g_lc.join.lobby_id : ""; + meta.scope = g_lc.in_lobby ? "lobby" : "server"; + + n = rnet_chat_report_build(msg, sizeof(msg), mids, mid_count, + reason, note, &meta); + if (n == 0) + return -1; + queue_send(msg); + flush_pending(); + return 0; +} + int psx_lobby_send_chat(const char *text) { char esc[PSX_LOBBY_CHAT_TEXT_LEN * 2 + 8]; From b9ca60077882b35bfae6e11cbfebe6d01fd8e19d Mon Sep 17 00:00:00 2001 From: Alex Vanderveen <alex@technicallycomputers.ca> Date: Sat, 12 Sep 2026 19:39:09 -0400 Subject: [PATCH 10/56] netplay: block list, account keys, list scope and automatch for the launcher Brings the PSX runtime level with what recomp-ui's netplay pages now ask a host for, and with the SNES runner that already answers them (snes_lobby_client.c / snes_host_lobby.c): - Chat lines, seat rows and players-online rows carry the server's opaque account key, so the launcher's ignore/block list survives a rename. - psx_lobby_set_blocks pushes that list to the server as `set_blocks`, since refusing to pair or seat the two together is the server's job; recomp-ui re-sends it on every reconnect. - list_scope_set: a LAN player no longer sees online rooms they have no connection for, and an online player no longer sees their own LAN rows. - Automatch: rulesets discovery, the relay latency probe (14-byte LE frame, type 200 -> 201, nonce-matched), queue with the disc fingerprint, the accept gate with a locally counted deadline, requeue/cancel/cooldown, and claiming automatch's `error` codes only while a ticket is in flight. Leaving an automatch room after the match: it is the server's room, has no host to rematch with, and holding the seat blocks the next ticket. - Account: init the auth client once per URL rather than every pump (the login worker reads the host), anchor the device key to the executable directory, and publish the signed-in handle as the lobby display name. mods_enabled is asserted false on PSX: every netplay launch clears the mod plan or refuses to start, and the caps a match runs are the ruleset's. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FPXfGGhBBBxQPcEzaMsauF --- runtime/include/psx_lobby_client.h | 135 ++++++ runtime/src/main.cpp | 217 ++++++++- runtime/src/psx_lobby_client.c | 755 ++++++++++++++++++++++++++++- 3 files changed, 1097 insertions(+), 10 deletions(-) diff --git a/runtime/include/psx_lobby_client.h b/runtime/include/psx_lobby_client.h index b70521a3..60e62746 100644 --- a/runtime/include/psx_lobby_client.h +++ b/runtime/include/psx_lobby_client.h @@ -71,6 +71,11 @@ typedef struct PsxLobbyOnlinePlayer { /* The title that player is browsing for; rows of other titles are * dropped at parse time when this client has a game identity. */ char game_name[PSX_LOBBY_NAME_LEN]; + /* Opaque, stable id for the ACCOUNT behind this connection; "" for a + * guest. The server's own row key, published so a client-side block + * list can survive the other player reconnecting or renaming. A key, + * never a label: nothing renders it. */ + char account[PSX_LOBBY_ID_LEN]; } PsxLobbyOnlinePlayer; #define PSX_LOBBY_MAX_ONLINE 64 @@ -95,6 +100,9 @@ typedef struct PsxLobbyMember { int memcard_share; /* peer opted in to bring it to the match */ /* Country (alpha-2) from the server's GeoIP; "" unknown. */ char country[4]; + /* Account key behind this seat; "" for a guest. See + * PsxLobbyOnlinePlayer.account. */ + char account[PSX_LOBBY_ID_LEN]; } PsxLobbyMember; /* @@ -137,6 +145,9 @@ typedef struct PsxLobbyChatMsg { * relayed under this id. Sending the words would let a client fabricate a * message and have somebody sanctioned for it. */ char mid[40]; + /* Account key of the sender; "" for a guest or a system line. What an + * ignore/block keys on, so it outlives a rename. */ + char account[PSX_LOBBY_ID_LEN]; int is_local; int is_system; uint32_t seq; @@ -461,6 +472,130 @@ void psx_lobby_clear_launch_pending(void); * Launch suspends them so they cannot steal match ICE signaling. */ void psx_lobby_resume_waiting_room_rtt(void); +/* + * Tell the server which accounts this player has blocked. + * + * ';'-separated opaque account ids, replacing the whole set -- the client's + * own file (recomp-ui moderation.ini) is the authority, so an unblock needs + * no separate op. NULL/"" clears. + * + * Sent to the SERVER rather than applied only here, because two of the three + * things a block has to do cannot be done from this side: the matchmaker must + * not pair the two, and the blocked player must not see or be able to join + * the blocker's room. A client can hide what it was sent; it cannot know it + * was blocked by somebody else, and that is the direction that matters. + * + * The server holds the list for the life of a connection and never persists + * it; recomp-ui re-sends it on every reconnect. + */ +int psx_lobby_set_blocks(const char *accounts); + +/* -- Automatch -------------------------------------------------------------- + * + * Server-run pairing: the player asks for a match and the SERVER creates the + * room, is its host, and owns its match_caps from a named ruleset. Protocol: + * recomp-net-server docs/AUTOMATCH.md. Everything downstream of the pairing + * is the ordinary `joined` / `lobby_update` / `launch` path already in this + * file, so this adds a queue and an accept gate and nothing else. + * + * Requires a signed-in account: the accept gate charges a dodge cooldown, and + * a cost that reconnecting erases is not a cost. + * + * Same shape as the SNES client (snes_lobby_client.h), on purpose: the host + * adapter in main.cpp is then a one-line translation per entry point. + */ +#define PSX_LOBBY_MAX_RULESETS 8 +#define PSX_LOBBY_RULESET_ID_LEN 48 +#define PSX_LOBBY_RULESET_LABEL_LEN 64 +#define PSX_LOBBY_CAPS_SUMMARY_LEN 128 + +/* One queue type this server offers for this title. */ +typedef struct PsxLobbyRuleset { + char id[PSX_LOBBY_RULESET_ID_LEN]; + char label[PSX_LOBBY_RULESET_LABEL_LEN]; + /* Server-derived one-liner ("Delay 2 - Rollback on"). Derived from the + * caps rather than authored, so it cannot drift from what the match + * runs. The caps themselves arrive with `joined` like any host's. */ + char caps_summary[PSX_LOBBY_CAPS_SUMMARY_LEN]; + /* Empty = any release may queue (still only pooling with its own). */ + char game_version[PSX_LOBBY_VERSION_LEN]; + int max_slots; +} PsxLobbyRuleset; + +/* Matches RECOMP_LAUNCHER_AUTOMATCH_* so the host layer can hand these + * straight to the launcher without a translation table that could drift. */ +enum { + PSX_LOBBY_AUTOMATCH_IDLE = 0, + PSX_LOBBY_AUTOMATCH_QUEUED = 1, + PSX_LOBBY_AUTOMATCH_FOUND = 2, + PSX_LOBBY_AUTOMATCH_ACCEPTED = 3, + PSX_LOBBY_AUTOMATCH_FAILED = 4 +}; + +/* The offer on the table while state is FOUND. */ +typedef struct PsxLobbyAutomatchFound { + char opponent[PSX_LOBBY_NAME_LEN]; + /* Discord @username: display names are not unique, so this is the + * disambiguator. "" when the server did not say. */ + char opponent_username[PSX_LOBBY_NAME_LEN]; + char opponent_country[4]; + char ruleset_id[PSX_LOBBY_RULESET_ID_LEN]; + char ruleset_label[PSX_LOBBY_RULESET_LABEL_LEN]; + int est_rtt_ms; /* the pair's estimate: rtt_a + rtt_b; <0 unknown */ + int accept_secs; /* seconds left to answer, recomputed per read */ +} PsxLobbyAutomatchFound; + +/* Ask the server what queues it offers for this title. Answered into + * psx_lobby_automatch_ruleset_count/get; 0 of them means automatch is off + * here. Refuses to re-send while one is outstanding. */ +int psx_lobby_automatch_request_rulesets(void); +/* 1 once the server has answered with at least one ruleset. Not having + * asked yet is "no": the button must not be offered on an assumption. */ +int psx_lobby_automatch_available(void); +int psx_lobby_automatch_ruleset_count(void); +int psx_lobby_automatch_ruleset_get(int index, PsxLobbyRuleset *out); + +/* + * Join the queue. `ruleset_id` NULL/"" takes the first one. + * + * `mods_enabled` is the caller's assertion that a SIM-AFFECTING mod feature is + * on locally, beyond whatever the ruleset itself imposes. The server refuses a + * true with mods_not_pooled and cannot check it -- the assertion exists so a + * modified client makes a deliberate false statement rather than exploiting + * an omission (AUTOMATCH.md 5). The host layer decides it; this file only + * carries it. `mod_exempt` is ';'-separated evidence for cosmetic exemptions + * (NULL or "" = none), sent as a JSON array for the server to check. + * + * 0 = sent. <0 = refused locally (not connected, no fingerprint, already + * queued). A server refusal arrives asynchronously as state FAILED with a + * reason in psx_lobby_automatch_error(). + */ +int psx_lobby_automatch_queue(const char *ruleset_id, int mods_enabled, + const char *mod_exempt); +int psx_lobby_automatch_cancel(void); +int psx_lobby_automatch_state(void); +int psx_lobby_automatch_queued_secs(void); +int psx_lobby_automatch_pool(void); +int psx_lobby_automatch_found_get(PsxLobbyAutomatchFound *out); +/* accept != 0 accepts; 0 declines and takes the dodge strike. */ +int psx_lobby_automatch_accept(int accept); +/* + * Non-zero while the room this client is seated in was created by AUTOMATCH. + * + * A human-hosted room is somebody's room: it survives the match, and staying + * seated is how a rematch happens. An automatch room is the server's, it is + * created at both-accept and is not joinable by anyone, and there is no host + * to rematch with -- so staying in it leaves the player parked in a room that + * can never fill, and the server refuses their next ticket with + * `already_in_lobby`. Cleared by leaving, and by anything else that ends the + * seating. + */ +int psx_lobby_automatch_room(void); +/* Refuse a queue locally, with the reason the player sees. For when the HOST + * already knows the server would bounce the ticket. */ +void psx_lobby_automatch_refuse_local(const char *why); +const char *psx_lobby_automatch_error(void); + #ifdef __cplusplus } #endif diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index b62e2b70..4ed1904b 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -10037,6 +10037,139 @@ namespace { int ae_np_account_sign_out(void*) { return rnet_account_sign_out(); } int ae_np_account_set_handle(void*, const char* h) { return rnet_account_set_handle(h); } + /* Point the account client at the lobby host -- once per URL, not once + * per pump: the login worker thread reads the host while a sign-in is in + * flight, and re-initialising it 60 times a second under that read is a + * data race for no gain. The secret is anchored to the EXECUTABLE + * directory before the first init: its default is the bare relative name + * "netplay_secret", resolved against the working directory, so the same + * install signed itself out depending on where it was launched from. + * rnet_auth migrates an old CWD-relative file into this path on first + * load, so nobody is signed out by the move. Same shape as the SNES + * host (snes_host_lobby.c cb_pump). */ + void ae_np_account_sync(void) { + static std::string s_auth_url; + const std::string& url = g_lnch_lobby_url; + if (url.empty() || url == s_auth_url) return; + if (s_auth_url.empty()) { + const std::string secret = + (exe_dir_from_argv(g_lnch_argv0 ? g_lnch_argv0 : "") / + "netplay_secret").string(); + rnet_account_set_secret_path(secret.c_str()); + } + s_auth_url = url; + rnet_account_init(url.c_str()); + } + + /* ---- list scope -------------------------------------------------------- + * The launcher forks LAN / Direct IP from online before the browser, and + * only it knows which fork the player took. Without the scope a player + * who chose LAN was shown online rooms they had no connection for, and + * one who chose online was shown LAN rooms from their own machine. The + * values are RECOMP_LAUNCHER_LIST_SCOPE_*; 0 (any) is the historical + * merge, and what a recomp-ui without the callback leaves us in. */ + int g_lnch_list_scope = 0; + int ae_np_list_scope_set(void*, int scope) { + g_lnch_list_scope = scope; + return 0; + } + bool ae_np_list_want_online(void) { return g_lnch_list_scope != 1; } + bool ae_np_list_want_lan(void) { return g_lnch_list_scope != 2; } + + /* ---- moderation ------------------------------------------------------ + * Both go to the lobby server; a LAN room has none, and the launcher + * only offers them online. What a report contains is recomp-net's + * (chat_report.h); the block list is the launcher's own file, pushed + * here so the server can refuse to pair or seat the two together. */ + int ae_np_chat_report(void*, const char* const* mids, int mid_count, + const char* reason, const char* note) { + if (g_lnch_hosting_lan || g_lnch_joined_lan) return -1; + return psx_lobby_report_chat(mids, mid_count, reason, note); + } + int ae_np_set_blocks(void*, const char* accounts) { + if (g_lnch_hosting_lan || g_lnch_joined_lan) return -1; + return psx_lobby_set_blocks(accounts); + } + + /* ---- automatch ------------------------------------------------------- + * Thin: the lobby client owns the protocol and the state machine, and + * this only translates its vocabulary into the launcher's. The one piece + * of POLICY here is mods_enabled -- see ae_np_automatch_queue. */ + bool ae_np_online_mode(void) { + return !g_lnch_hosting_lan && !g_lnch_joined_lan && psx_lobby_connected(); + } + int ae_np_automatch_available(void*) { + if (!ae_np_online_mode()) return 0; + /* Ask once the answer could exist. The launcher polls this every + * frame while the netplay page is up, which is exactly when a reply + * is useful, and the client refuses to re-send while one is + * outstanding. */ + if (!psx_lobby_automatch_available()) + (void)psx_lobby_automatch_request_rulesets(); + return psx_lobby_automatch_available(); + } + int ae_np_automatch_ruleset_count(void*) { + return psx_lobby_automatch_ruleset_count(); + } +#if defined(RECOMP_LAUNCHER_HAS_AUTOMATCH) + int ae_np_automatch_ruleset_get(void*, int index, RecompLauncherCNetplayRuleset* out) { + PsxLobbyRuleset r{}; + if (!out || !psx_lobby_automatch_ruleset_get(index, &r)) return 0; + std::memset(out, 0, sizeof(*out)); + std::snprintf(out->id, sizeof(out->id), "%s", r.id); + std::snprintf(out->label, sizeof(out->label), "%s", r.label); + std::snprintf(out->caps_summary, sizeof(out->caps_summary), "%s", r.caps_summary); + std::snprintf(out->game_version, sizeof(out->game_version), "%s", r.game_version); + out->max_slots = r.max_slots > 0 ? r.max_slots : 2; + return 1; + } + int ae_np_automatch_found_get(void*, RecompLauncherCNetplayFound* out) { + PsxLobbyAutomatchFound f{}; + if (!out || !psx_lobby_automatch_found_get(&f)) return 0; + std::memset(out, 0, sizeof(*out)); + std::snprintf(out->handle, sizeof(out->handle), "%s", f.opponent); + std::snprintf(out->username, sizeof(out->username), "%s", f.opponent_username); + std::snprintf(out->country, sizeof(out->country), "%s", f.opponent_country); + std::snprintf(out->ruleset_label, sizeof(out->ruleset_label), "%s", f.ruleset_label); + out->est_rtt_ms = f.est_rtt_ms; + out->accept_secs_left = f.accept_secs; + return 1; + } +#endif + int ae_np_automatch_queue(void*, const char* ruleset_id) { + if (!ae_np_online_mode()) return -1; + /* mods_enabled asserts that a SIM-AFFECTING mod feature is on locally + * beyond what the ruleset imposes. On PSX that is never true for a + * netplay session: every netplay launch, rematch included, goes + * through mod_runtime_clear_for_netplay and refuses to start if the + * plan cannot be cleared, and the caps a match runs (aspect, turbo + * loads, BIOS, FMV skip) are the server's ruleset, settled through + * match_caps like any host's. There is no cosmetic-exemption + * mechanism on this runtime either, so the evidence list is empty. */ + return psx_lobby_automatch_queue(ruleset_id, 0, "") == 0 ? 0 : -1; + } + int ae_np_automatch_cancel(void*) { return psx_lobby_automatch_cancel(); } + int ae_np_automatch_state(void*) { return psx_lobby_automatch_state(); } + int ae_np_automatch_queued_secs(void*) { return psx_lobby_automatch_queued_secs(); } + int ae_np_automatch_pool(void*) { return psx_lobby_automatch_pool(); } + int ae_np_automatch_accept(void*, int accept) { return psx_lobby_automatch_accept(accept); } + const char* ae_np_automatch_error(void*) { return psx_lobby_automatch_error(); } + + /* After a match: an automatch room is the server's, not a host's. It is + * created at both-accept, nobody can join it, and there is no host to + * rematch with -- staying seated parks the player in a room that can + * never fill, and the server refuses their next ticket with + * already_in_lobby. So leave it, and tell the caller not to reopen the + * launcher on the room. 1 when a room was left. */ + int ae_np_leave_automatch_room_after_match(void) { + if (g_lnch_hosting_lan || g_lnch_joined_lan) return 0; + if (!psx_lobby_automatch_room()) return 0; + std::fprintf(stderr, + "psxrecomp: leaving the automatch room (no host to rematch with)\n"); + (void)psx_lobby_leave(); + return 1; + } + int ae_np_chat_count(void*) { ae_np_chat_track_room(); if (g_lnch_hosting_lan || g_lnch_joined_lan) return g_lnch_lan_chat_count; @@ -10060,6 +10193,14 @@ namespace { if (!psx_lobby_chat_get(index, &msg)) return 0; std::snprintf(out->from, sizeof(out->from), "%s", msg.from); std::snprintf(out->text, sizeof(out->text), "%s", msg.text); +#if defined(RECOMP_LAUNCHER_HAS_CHAT_REPORT) + /* The server's id for the line -- what a report names. Empty for a + * system line or an older server, and then it cannot be reported. */ + std::snprintf(out->mid, sizeof(out->mid), "%s", msg.mid); +#endif +#if defined(RECOMP_LAUNCHER_HAS_PLAYER_ACCOUNT) + std::snprintf(out->account, sizeof(out->account), "%s", msg.account); +#endif out->is_local = msg.is_local; out->is_system = msg.is_system; out->seq = msg.seq; @@ -10085,6 +10226,14 @@ namespace { if (!psx_lobby_server_chat_get(index, &msg)) return 0; std::snprintf(out->from, sizeof(out->from), "%s", msg.from); std::snprintf(out->text, sizeof(out->text), "%s", msg.text); +#if defined(RECOMP_LAUNCHER_HAS_CHAT_REPORT) + /* The server's id for the line -- what a report names. Empty for a + * system line or an older server, and then it cannot be reported. */ + std::snprintf(out->mid, sizeof(out->mid), "%s", msg.mid); +#endif +#if defined(RECOMP_LAUNCHER_HAS_PLAYER_ACCOUNT) + std::snprintf(out->account, sizeof(out->account), "%s", msg.account); +#endif out->is_local = msg.is_local; out->is_system = msg.is_system; out->seq = msg.seq; @@ -10244,7 +10393,7 @@ namespace { ae_np_save_identity(nullptr, g_lnch_lobby_url.c_str()); /* The auth endpoints live on the same host and port as the lobby * socket, so the sign-in follows whatever server the player points at. */ - rnet_account_init(g_lnch_lobby_url.c_str()); + ae_np_account_sync(); } int ae_np_connect(void*) { @@ -10822,11 +10971,27 @@ namespace { } void ae_np_pump(void*) { - psx_lobby_pump(); /* Redeems a stored device key on the first pump, so a machine that has * signed in once comes up signed in with no player action. */ - rnet_account_init(g_lnch_lobby_url.c_str()); + ae_np_account_sync(); rnet_account_pump(); + /* Publish the account name to the lobby. The lobby's display name is + * what seats and the players-online list show, and nothing else + * pushed the handle into it: signing in -- including the automatic + * sign-in from a stored secret -- only updated the ACCOUNT, so a + * signed-in player created a room and appeared under the LAN name. + * Done here rather than at a sign-in edge because there is no single + * such edge: interactive login, stored-secret redemption and a + * server-side handle change all land asynchronously in the pump. + * Comparing against the live name makes this idempotent -- + * set_display_name only re-sends hello when the value changed. */ + if (rnet_account_state() == RNET_ACCOUNT_SIGNED_IN) { + const char* handle = rnet_account_handle(); + const char* shown = psx_lobby_display_name(); + if (handle && handle[0] && (!shown || std::strcmp(shown, handle) != 0)) + psx_lobby_set_display_name(handle); + } + psx_lobby_pump(); ae_np_lan_browse_pump(); ae_np_lan_udp_pump(); /* Lobby UI has no Ready toggle; production WS still requires every @@ -10870,14 +11035,16 @@ namespace { } int ae_np_list_count(void*) { - return psx_lobby_list_count() + ae_np_lan_list_extra_count(); + return (ae_np_list_want_online() ? psx_lobby_list_count() : 0) + + (ae_np_list_want_lan() ? ae_np_lan_list_extra_count() : 0); } int ae_np_list_get(void*, int index, RecompLauncherCNetplayLobby* out) { if (!out) return 0; - const int remote_count = psx_lobby_list_count(); + const int remote_count = ae_np_list_want_online() ? psx_lobby_list_count() : 0; if (index >= remote_count) { const int lan_i = index - remote_count; + if (!ae_np_list_want_lan()) return 0; ae_np_lan_prune_discovered(); if (g_lnch_lan_discovered_n > 0) return ae_np_lan_fill_lobby_from_discovered(lan_i, out); @@ -10914,6 +11081,9 @@ namespace { std::memset(out, 0, sizeof(*out)); std::snprintf(out->display_name, sizeof(out->display_name), "%s", p.display_name); std::snprintf(out->country, sizeof(out->country), "%s", p.country); +#if defined(RECOMP_LAUNCHER_HAS_PLAYER_ACCOUNT) + std::snprintf(out->account, sizeof(out->account), "%s", p.account); +#endif std::snprintf(out->lobby_name, sizeof(out->lobby_name), "%s", p.lobby_name); out->in_lobby = p.lobby_id[0] != '\0'; out->hosting = p.hosting; @@ -11472,6 +11642,9 @@ namespace { out->memcard_has_card = mem.memcard_has_card; out->memcard_share = mem.memcard_share; std::snprintf(out->country, sizeof(out->country), "%s", mem.country); + #if defined(RECOMP_LAUNCHER_HAS_PLAYER_ACCOUNT) + std::snprintf(out->account, sizeof(out->account), "%s", mem.account); + #endif return 1; } if (ae_np_use_lan_members()) { @@ -11533,6 +11706,9 @@ namespace { out->memcard_has_card = mem.memcard_has_card; out->memcard_share = mem.memcard_share; std::snprintf(out->country, sizeof(out->country), "%s", mem.country); + #if defined(RECOMP_LAUNCHER_HAS_PLAYER_ACCOUNT) + std::snprintf(out->account, sizeof(out->account), "%s", mem.account); + #endif return 1; } @@ -12094,6 +12270,30 @@ namespace { g_lnch_netplay_callbacks.account_error = ae_np_account_error; g_lnch_netplay_callbacks.account_sign_out = ae_np_account_sign_out; g_lnch_netplay_callbacks.account_set_handle = ae_np_account_set_handle; +#endif +#if defined(RECOMP_LAUNCHER_HAS_LIST_SCOPE) + g_lnch_netplay_callbacks.list_scope_set = ae_np_list_scope_set; +#endif +#if defined(RECOMP_LAUNCHER_HAS_CHAT_REPORT) + g_lnch_netplay_callbacks.chat_report = ae_np_chat_report; +#endif +#if defined(RECOMP_LAUNCHER_HAS_SET_BLOCKS) + g_lnch_netplay_callbacks.set_blocks = ae_np_set_blocks; +#endif +#if defined(RECOMP_LAUNCHER_HAS_AUTOMATCH) + /* Server-run pairing. Same guard discipline as the account block: + * the runner and the UI land in either order. */ + g_lnch_netplay_callbacks.automatch_available = ae_np_automatch_available; + g_lnch_netplay_callbacks.automatch_ruleset_count = ae_np_automatch_ruleset_count; + g_lnch_netplay_callbacks.automatch_ruleset_get = ae_np_automatch_ruleset_get; + g_lnch_netplay_callbacks.automatch_queue = ae_np_automatch_queue; + g_lnch_netplay_callbacks.automatch_cancel = ae_np_automatch_cancel; + g_lnch_netplay_callbacks.automatch_state = ae_np_automatch_state; + g_lnch_netplay_callbacks.automatch_queued_secs = ae_np_automatch_queued_secs; + g_lnch_netplay_callbacks.automatch_pool = ae_np_automatch_pool; + g_lnch_netplay_callbacks.automatch_found_get = ae_np_automatch_found_get; + g_lnch_netplay_callbacks.automatch_accept = ae_np_automatch_accept; + g_lnch_netplay_callbacks.automatch_error = ae_np_automatch_error; #endif g_lnch_netplay_callbacks.seat_move_self = ae_np_seat_move_self; g_lnch_netplay_callbacks.seat_swap_request = ae_np_seat_swap_request; @@ -15645,6 +15845,11 @@ soft_return_lobby: std::string rui_title = (game_name.empty() ? std::string("PSX") : game_name) + " - Launcher"; std::string rui_initial_disc = disc_path_str; + /* A human-hosted room survives the match and is where a rematch + * happens; an automatch room is the server's and cannot. Leave it + * here, and reopen the launcher on the browser instead. */ + const int rui_resume_room = + ae_np_leave_automatch_room_after_match() ? 0 : 1; ae_rui_set_sidecar_paths(argv[0]); g_lnch_expected_serial = game_id; @@ -15822,7 +16027,7 @@ soft_return_lobby: ctrl_locked_mode[0], rui_lang_labels.empty() ? nullptr : rui_lang_labels.data(), (int)rui_lang_labels.size(), - /*resume_netplay_room=*/1); + /*resume_netplay_room=*/rui_resume_room); gi.discs = rui_discs.empty() ? nullptr : rui_discs.data(); gi.num_discs = (int)rui_discs.size(); #if defined(PSX_HAS_SETUP_WIZARD) && defined(PSX_HAS_CODEGEN_SETUP_HOST) diff --git a/runtime/src/psx_lobby_client.c b/runtime/src/psx_lobby_client.c index e0615747..4dc5d738 100644 --- a/runtime/src/psx_lobby_client.c +++ b/runtime/src/psx_lobby_client.c @@ -126,6 +126,22 @@ int psx_lobby_send_chat(const char *text) { (void)text; return -1; } int psx_lobby_report_chat(const char *const *m, int c, const char *r, const char *n) { (void)m; (void)c; (void)r; (void)n; return -1; } +int psx_lobby_set_blocks(const char *accounts) { (void)accounts; return -1; } +int psx_lobby_automatch_request_rulesets(void) { return -1; } +int psx_lobby_automatch_available(void) { return 0; } +int psx_lobby_automatch_ruleset_count(void) { return 0; } +int psx_lobby_automatch_ruleset_get(int i, PsxLobbyRuleset *o) { (void)i; (void)o; return 0; } +int psx_lobby_automatch_queue(const char *r, int m, const char *e) +{ (void)r; (void)m; (void)e; return -1; } +int psx_lobby_automatch_cancel(void) { return -1; } +int psx_lobby_automatch_state(void) { return PSX_LOBBY_AUTOMATCH_IDLE; } +int psx_lobby_automatch_queued_secs(void) { return 0; } +int psx_lobby_automatch_pool(void) { return 0; } +int psx_lobby_automatch_found_get(PsxLobbyAutomatchFound *o) { (void)o; return 0; } +int psx_lobby_automatch_accept(int a) { (void)a; return -1; } +int psx_lobby_automatch_room(void) { return 0; } +void psx_lobby_automatch_refuse_local(const char *w) { (void)w; } +const char *psx_lobby_automatch_error(void) { return ""; } int psx_lobby_seat_move_self(int to_slot) { (void)to_slot; return -1; } int psx_lobby_seat_swap_request(int target_slot) { (void)target_slot; return -1; } int psx_lobby_seat_swap_incoming(char *who, size_t who_cap, int *from_slot) @@ -1351,6 +1367,8 @@ static void lobby_list_parse_players(const char *json) sizeof(g_lc.online[n].lobby_name)); g_lc.online[n].hosting = json_get_bool(chunk, "hosting", 0); json_get_str(chunk, "tag", g_lc.online[n].tag, sizeof(g_lc.online[n].tag)); + json_get_str(chunk, "account", g_lc.online[n].account, + sizeof(g_lc.online[n].account)); json_get_str(chunk, "game_name", g_lc.online[n].game_name, sizeof(g_lc.online[n].game_name)); /* Players of another title are not "online" for this one. A row @@ -1695,6 +1713,8 @@ static int parse_seat_array(const char *json, const char *key, int is_spectator, } json_get_str(chunk, "country", g_lc.members[n].country, sizeof(g_lc.members[n].country)); + json_get_str(chunk, "account", g_lc.members[n].account, + sizeof(g_lc.members[n].account)); if (json_extract_object(chunk, "memcard_offer", offer, sizeof(offer))) { g_lc.members[n].memcard_offer_valid = 1; g_lc.members[n].memcard_has_card = @@ -1761,6 +1781,699 @@ static void ingest_host_player_id(const char *json) } } + +/* ── Block list ────────────────────────────────────────────────────────────── */ + +int psx_lobby_set_blocks(const char *accounts) +{ + /* Room for the server's cap (256 ids) at 40 characters each, plus the + * separators and the envelope. A list that would not fit is truncated at + * a separator rather than sent malformed -- a half-written id at the end + * would name nobody, and a malformed op would leave the server enforcing + * nothing at all. */ + static char msg[256 * 41 + 64]; + static char list[256 * 41]; + size_t n = 0; + int first = 1; + const char *p = accounts ? accounts : ""; + + if (!g_lc.connected) return -1; + list[0] = '\0'; + while (*p) { + const char *sep = strchr(p, ';'); + size_t len = sep ? (size_t)(sep - p) : strlen(p); + if (len && len < 40 && n + len + 8 < sizeof(list)) { + if (!first) { list[n++] = ','; } + list[n++] = '"'; + memcpy(list + n, p, len); + n += len; + list[n++] = '"'; + list[n] = '\0'; + first = 0; + } + if (!sep) break; + p = sep + 1; + } + snprintf(msg, sizeof(msg), "{\"op\":\"set_blocks\",\"accounts\":[%s]}", list); + queue_send(msg); + flush_pending(); + return 0; +} + +/* ── Automatch ───────────────────────────────────────────────────────────── + * + * State machine and wire for recomp-net-server docs/AUTOMATCH.md. The room a + * pairing produces arrives as an ordinary `joined` and is handled by the + * existing code; this owns the ticket, the accept gate, and the latency + * probe the server uses to pick an input delay. + */ +#define AUTOMATCH_PROBE_LEN 14 + +static int set_nonblock(int fd); /* defined with the socket code below */ + +typedef struct { + int have_rulesets; /* the server has answered once this connection */ + int rulesets_in_flight; + int ruleset_count; + PsxLobbyRuleset rulesets[PSX_LOBBY_MAX_RULESETS]; + + int state; /* PSX_LOBBY_AUTOMATCH_* */ + char ticket_id[PSX_LOBBY_ID_LEN]; + char match_id[PSX_LOBBY_ID_LEN]; + int queued_secs; + int pool; + char error[160]; + + PsxLobbyAutomatchFound found; + /* When the offer lapses, as a monotonic timestamp rather than the count + * the server sent. The server states the deadline once; a client that + * stored the number and showed it unchanged would display "15s" for the + * whole fifteen seconds, which reads as a frozen dialog. */ + uint64_t found_deadline_ms; + + /* Where to send the latency probe, published on rulesets_ok and again on + * automatch_queued. Kept from whichever arrived last. */ + char probe_host[128]; + int probe_port; + unsigned probe_magic; + int probe_type; + /* The measurement, and the nonce that identifies our outstanding probe. */ + int rtt_ms; /* <0 = not measured yet */ + uint32_t probe_nonce; + uint64_t probe_sent_ms; /* 0 = none outstanding */ + int probe_socket; /* -1 = not open */ + int rtt_reported; /* the server has our number */ + /* A queue op is out and its answer -- automatch_queued, or an error -- + * has not arrived. Without it a refusal of the FIRST queue attempt + * (need_account is the common one) would arrive while state is still + * IDLE and be filed as somebody else's error. */ + int queue_in_flight; + /* This seat came from a pairing, not from a room somebody hosts. */ + int in_automatch_room; +} LobbyAutomatch; + +static LobbyAutomatch g_am = { .rtt_ms = -1, .probe_socket = -1 }; + +static void automatch_reset_queue_state(void) +{ + g_am.state = PSX_LOBBY_AUTOMATCH_IDLE; + g_am.queue_in_flight = 0; + g_am.ticket_id[0] = '\0'; + g_am.match_id[0] = '\0'; + g_am.queued_secs = 0; + g_am.pool = 0; + g_am.rtt_reported = 0; + g_am.found_deadline_ms = 0; + memset(&g_am.found, 0, sizeof(g_am.found)); +} + +static void automatch_fail(const char *why) +{ + g_am.state = PSX_LOBBY_AUTOMATCH_FAILED; + g_am.queue_in_flight = 0; + snprintf(g_am.error, sizeof(g_am.error), "%s", why ? why : "automatch failed"); + fprintf(stderr, "psx_lobby: automatch failed: %s\n", g_am.error); +} + +static void automatch_probe_close(void) +{ + if (g_am.probe_socket >= 0) { + close(g_am.probe_socket); + g_am.probe_socket = -1; + } + g_am.probe_sent_ms = 0; +} + +/* A new connection has no ticket and has not been told what queues exist: + * the server holds both per socket. Called from `welcome` and on disconnect + * so the availability question is re-asked, not answered from a stale yes. */ +static void automatch_on_connection_reset(void) +{ + automatch_reset_queue_state(); + automatch_probe_close(); + g_am.have_rulesets = 0; + g_am.rulesets_in_flight = 0; + g_am.ruleset_count = 0; + g_am.rtt_ms = -1; + g_am.in_automatch_room = 0; +} + +/* + * The 14-byte probe: magic, type, then a nonce. + * + * LITTLE-ENDIAN, because that is what the relay's header is (`read_u32_le` / + * `read_u16_le` in input_relay.rs) -- a big-endian magic is simply not this + * protocol's magic, so the packet is dropped on the `magic` counter with no + * reply. The nonce sits at offset 6, where an ordinary packet carries its + * session id; the relay echoes those bytes untouched and only rewrites the + * type (200 -> 201), so it comes back as sent. + */ +static void automatch_probe_pack(unsigned char *out, uint32_t nonce) +{ + const unsigned magic = g_am.probe_magic; + const int type = g_am.probe_type ? g_am.probe_type : 200; + memset(out, 0, AUTOMATCH_PROBE_LEN); + out[0] = (unsigned char)(magic & 0xFF); + out[1] = (unsigned char)((magic >> 8) & 0xFF); + out[2] = (unsigned char)((magic >> 16) & 0xFF); + out[3] = (unsigned char)((magic >> 24) & 0xFF); + out[4] = (unsigned char)(type & 0xFF); + out[5] = (unsigned char)((type >> 8) & 0xFF); + out[6] = (unsigned char)(nonce & 0xFF); + out[7] = (unsigned char)((nonce >> 8) & 0xFF); + out[8] = (unsigned char)((nonce >> 16) & 0xFF); + out[9] = (unsigned char)((nonce >> 24) & 0xFF); +} + +static int automatch_probe_send(void) +{ + struct addrinfo hints, *res = NULL; + unsigned char pkt[AUTOMATCH_PROBE_LEN]; + char portstr[16]; + int fd; + + if (!g_am.probe_host[0] || g_am.probe_port <= 0) return -1; + if (g_am.probe_sent_ms) return 0; /* one outstanding at a time */ + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + snprintf(portstr, sizeof(portstr), "%d", g_am.probe_port); + if (getaddrinfo(g_am.probe_host, portstr, &hints, &res) != 0 || !res) + return -1; + + fd = (int)socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (fd < 0) { freeaddrinfo(res); return -1; } + set_nonblock(fd); + + /* A fresh nonce per attempt, so a late reply to a previous probe cannot + * be timed against this one's clock and report an absurdly low number. */ + g_am.probe_nonce = (uint32_t)(lobby_mono_ms() * 2654435761u) ^ 0x9E3779B9u; + automatch_probe_pack(pkt, g_am.probe_nonce); + + if (sendto(fd, (const char *)pkt, (int)sizeof(pkt), 0, + res->ai_addr, (int)res->ai_addrlen) < 0) { + close(fd); + freeaddrinfo(res); + return -1; + } + freeaddrinfo(res); + + automatch_probe_close(); + g_am.probe_socket = fd; + g_am.probe_sent_ms = lobby_mono_ms(); + return 0; +} + +static void automatch_send_rtt(void) +{ + char msg[128]; + if (g_am.rtt_ms < 0 || g_am.rtt_reported) return; + snprintf(msg, sizeof(msg), + "{\"op\":\"automatch_rtt\",\"rtt_ms\":%d}", g_am.rtt_ms); + queue_send(msg); + g_am.rtt_reported = 1; +} + +/* Polled every pump. Times the 201 reply, or gives up after 2 s. */ +static void automatch_probe_poll(void) +{ + unsigned char buf[64]; + uint64_t now; + + if (g_am.probe_socket < 0 || !g_am.probe_sent_ms) return; + now = lobby_mono_ms(); + + for (;;) { + int n = (int)recv(g_am.probe_socket, (char *)buf, (int)sizeof(buf), 0); + if (n < 0) break; + if (n < 10) continue; + /* Match the nonce: the socket is unconnected and anything can arrive + * on it, and an unrelated packet timed as our reply is a wrong number + * reported as fact. */ + if (((uint32_t)buf[6] | (uint32_t)buf[7] << 8 | + (uint32_t)buf[8] << 16 | (uint32_t)buf[9] << 24) != g_am.probe_nonce) + continue; + g_am.rtt_ms = (int)(now - g_am.probe_sent_ms); + if (g_am.rtt_ms < 0) g_am.rtt_ms = 0; + if (g_am.rtt_ms > 2000) g_am.rtt_ms = 2000; /* the server clamps here too */ + fprintf(stderr, "psx_lobby: automatch probe %s:%d rtt=%d ms\n", + g_am.probe_host, g_am.probe_port, g_am.rtt_ms); + automatch_probe_close(); + /* Queued already? Then the ticket was enqueued on an unknown latency + * and the server is holding it out of pairing for the probe grace -- + * tell it now rather than letting the grace lapse. */ + if (g_am.state == PSX_LOBBY_AUTOMATCH_QUEUED) automatch_send_rtt(); + return; + } + if (now - g_am.probe_sent_ms > 2000) { + fprintf(stderr, "psx_lobby: automatch probe timed out (%s:%d) -- " + "queueing without a latency estimate\n", + g_am.probe_host, g_am.probe_port); + automatch_probe_close(); + } +} + +/* Walk to the next {...} in an array, copying it out. Returns 0 at ']'. */ +static int automatch_next_object(const char **pp, char *out, size_t cap) +{ + const char *p = *pp; + const char *start; + int depth = 0; + size_t n; + + while (*p && *p != '{') { + if (*p == ']') { *pp = p; return 0; } + ++p; + } + if (*p != '{') { *pp = p; return 0; } + start = p; + do { + if (*p == '{') ++depth; + else if (*p == '}') --depth; + ++p; + } while (*p && depth > 0); + n = (size_t)(p - start); + if (n >= cap) n = cap - 1; + memcpy(out, start, n); + out[n] = '\0'; + *pp = p; + return 1; +} + +/* `probe: { endpoint, magic, type }` -- where to measure the path. Published + * on both rulesets_ok and queued, and taken from whichever arrived last. */ +static void automatch_ingest_probe(const char *obj) +{ + char endpoint[160]; + char *colon; + endpoint[0] = '\0'; + json_get_str(obj, "endpoint", endpoint, sizeof(endpoint)); + /* Rightmost colon: an IPv6 literal has several, and the port is last. */ + colon = strrchr(endpoint, ':'); + if (!colon || !colon[1]) return; + *colon = '\0'; + snprintf(g_am.probe_host, sizeof(g_am.probe_host), "%s", endpoint); + g_am.probe_port = atoi(colon + 1); + g_am.probe_magic = (unsigned)json_get_int(obj, "magic", 0); + g_am.probe_type = json_get_int(obj, "type", 200); +} + +/* `titles: [ { ..., pool: N } ]` -- this host queues one title, so the first + * row is the one the player is waiting in. */ +static int automatch_first_pool(const char *json) +{ + const char *p = strstr(json, "\"titles\""); + char obj[512]; + if (!p) return g_am.pool; + p = strchr(p, '['); + if (!p) return g_am.pool; + ++p; + if (!automatch_next_object(&p, obj, sizeof(obj))) return 0; + return json_get_int(obj, "pool", 0); +} + +int psx_lobby_automatch_request_rulesets(void) +{ + char msg[256]; + char gn_esc[JSON_ESC_CAP(PSX_LOBBY_NAME_LEN)]; + const char *gn = g_lc.filter_game_name; + if (!g_lc.connected) return -1; + if (!gn || !gn[0]) return -1; + if (g_am.rulesets_in_flight) return 0; + json_escape(gn, gn_esc, sizeof(gn_esc)); + snprintf(msg, sizeof(msg), + "{\"op\":\"automatch_rulesets\",\"game_name\":\"%s\"}", gn_esc); + queue_send(msg); + g_am.rulesets_in_flight = 1; + return 0; +} + +int psx_lobby_automatch_available(void) +{ + /* Zero rulesets is a real answer and means the same as "no": this + * deployment has none loaded for this title. Not having ASKED yet is also + * no -- the button must not be offered on an assumption. */ + return g_am.have_rulesets && g_am.ruleset_count > 0; +} + +int psx_lobby_automatch_ruleset_count(void) { return g_am.ruleset_count; } + +int psx_lobby_automatch_ruleset_get(int index, PsxLobbyRuleset *out) +{ + if (!out || index < 0 || index >= g_am.ruleset_count) return 0; + *out = g_am.rulesets[index]; + return 1; +} + +int psx_lobby_automatch_queue(const char *ruleset_id, int mods_enabled, + const char *mod_exempt) +{ + char msg[2048]; + char exempt_json[768]; + char gn_esc[JSON_ESC_CAP(PSX_LOBBY_NAME_LEN)]; + char gv_esc[JSON_ESC_CAP(PSX_LOBBY_VERSION_LEN)]; + char rid_esc[JSON_ESC_CAP(PSX_LOBBY_RULESET_ID_LEN)]; + const char *rid = (ruleset_id && ruleset_id[0]) ? ruleset_id + : (g_am.ruleset_count > 0 ? g_am.rulesets[0].id : ""); + const char *disc_fp = psx_lobby_disc_fp(); + char rtt[48]; + + if (!g_lc.connected) return -1; + if (!rid[0]) return -1; + if (g_am.state == PSX_LOBBY_AUTOMATCH_QUEUED || + g_am.state == PSX_LOBBY_AUTOMATCH_FOUND || + g_am.state == PSX_LOBBY_AUTOMATCH_ACCEPTED || + g_am.queue_in_flight) + return -1; + /* The queue key REQUIRES a fingerprint: in `join` an empty one means + * "legacy host, no check", and a wildcard in a queue silently pairs a + * different dump against this one. Refuse here rather than let the server + * answer need_disc_fp, so the reason is available before the round trip. */ + if (!disc_fp || strlen(disc_fp) != 64) { + automatch_fail("this build cannot fingerprint its disc, so it cannot queue"); + return -1; + } + + json_escape(g_lc.filter_game_name, gn_esc, sizeof(gn_esc)); + json_escape(psx_lobby_game_version(), gv_esc, sizeof(gv_esc)); + json_escape(rid, rid_esc, sizeof(rid_esc)); + + /* The exemption evidence, as a JSON ARRAY of strings -- never a + * ';'-joined string, which is valid JSON that every reader using + * as_array() sees as empty and so fails open (recomp-net's + * chat_report.h records the instance this repo already paid for). */ + { + size_t o = 0; + const char *p = mod_exempt; + int first = 1; + exempt_json[o++] = '['; + while (p && *p) { + const char *end = p; + char entry[160]; + char esc[JSON_ESC_CAP(sizeof(entry))]; + size_t len; + while (*end && *end != ';' && *end != '\n') ++end; + len = (size_t)(end - p); + if (len && len < sizeof(entry)) { + memcpy(entry, p, len); + entry[len] = '\0'; + json_escape(entry, esc, sizeof(esc)); + if (o + strlen(esc) + 4 < sizeof(exempt_json)) { + if (!first) exempt_json[o++] = ','; + exempt_json[o++] = '"'; + memcpy(exempt_json + o, esc, strlen(esc)); + o += strlen(esc); + exempt_json[o++] = '"'; + first = 0; + } else { + /* Refuse rather than send a SHORT list: a truncated list + * of exemptions is one the server approves in full while + * the client relies on more than it declared. */ + automatch_fail("too many mod exemptions to declare"); + return -1; + } + } + p = *end ? end + 1 : end; + } + exempt_json[o++] = ']'; + exempt_json[o] = '\0'; + } + + /* Measure before queueing when we can: a client that probes first never + * waits out the server's probe grace at all. */ + if (g_am.rtt_ms < 0) automatch_probe_send(); + rtt[0] = '\0'; + if (g_am.rtt_ms >= 0) + snprintf(rtt, sizeof(rtt), ",\"rtt_ms\":%d", g_am.rtt_ms); + + /* One title: this host runs one game. A multi-title launcher sends + * several here, in preference order; the wire has always allowed it. */ + snprintf(msg, sizeof(msg), + "{\"op\":\"automatch_queue\",\"titles\":[{" + "\"game_name\":\"%s\",\"game_version\":\"%s\"," + "\"disc_fp\":\"%s\",\"ruleset_id\":\"%s\",\"max_slots\":2}]," + "\"mods_enabled\":%s,\"mod_exempt\":%s%s}", + gn_esc, gv_esc, disc_fp, rid_esc, + mods_enabled ? "true" : "false", exempt_json, rtt); + queue_send(msg); + flush_pending(); + g_am.error[0] = '\0'; + g_am.queue_in_flight = 1; + g_am.rtt_reported = (g_am.rtt_ms >= 0); + return 0; +} + +int psx_lobby_automatch_cancel(void) +{ + if (!g_lc.connected) return -1; + queue_send("{\"op\":\"automatch_cancel\"}"); + flush_pending(); + return 0; +} + +int psx_lobby_automatch_state(void) { return g_am.state; } +int psx_lobby_automatch_queued_secs(void) { return g_am.queued_secs; } +int psx_lobby_automatch_pool(void) { return g_am.pool; } + +int psx_lobby_automatch_found_get(PsxLobbyAutomatchFound *out) +{ + if (!out || g_am.state != PSX_LOBBY_AUTOMATCH_FOUND) return 0; + *out = g_am.found; + /* Recomputed on every read, so the caller can poll it each frame and get + * a live count. Never below 0: the offer is about to lapse, and a + * negative would draw as one. */ + if (g_am.found_deadline_ms) { + uint64_t now = lobby_mono_ms(); + out->accept_secs = now >= g_am.found_deadline_ms + ? 0 + : (int)((g_am.found_deadline_ms - now + 999ull) / 1000ull); + } + return 1; +} + +int psx_lobby_automatch_accept(int accept) +{ + char msg[192]; + char mid_esc[JSON_ESC_CAP(PSX_LOBBY_ID_LEN)]; + if (!g_lc.connected) return -1; + if (g_am.state != PSX_LOBBY_AUTOMATCH_FOUND) return -1; + /* The match id is echoed so a late answer to a LAPSED offer is discarded + * rather than applied to whatever offer is current by then. */ + json_escape(g_am.match_id, mid_esc, sizeof(mid_esc)); + snprintf(msg, sizeof(msg), + "{\"op\":\"automatch_accept\",\"match_id\":\"%s\"," + "\"accept\":%s}", mid_esc, accept ? "true" : "false"); + queue_send(msg); + flush_pending(); + if (accept) { + g_am.state = PSX_LOBBY_AUTOMATCH_ACCEPTED; + } else { + /* Declining ends the ticket. The strike is the server's to record. */ + automatch_reset_queue_state(); + } + return 0; +} + +int psx_lobby_automatch_room(void) { return g_am.in_automatch_room; } + +void psx_lobby_automatch_refuse_local(const char *why) +{ + automatch_fail(why); +} + +const char *psx_lobby_automatch_error(void) { return g_am.error; } + +/* Inbound automatch ops. Returns 1 when `op` was one of ours. */ +static int automatch_handle_op(const char *op, const char *json) +{ + if (strcmp(op, "automatch_rulesets_ok") == 0) { + const char *p2 = strstr(json, "\"rulesets\""); + char probe[256]; + int n = 0; + g_am.have_rulesets = 1; + g_am.rulesets_in_flight = 0; + g_am.ruleset_count = 0; + if (json_extract_object(json, "probe", probe, sizeof(probe))) + automatch_ingest_probe(probe); + if (p2) { + p2 = strchr(p2, '['); + if (p2) { + ++p2; + while (*p2 && n < PSX_LOBBY_MAX_RULESETS) { + char obj[1024]; + PsxLobbyRuleset *r = &g_am.rulesets[n]; + if (!automatch_next_object(&p2, obj, sizeof(obj))) break; + memset(r, 0, sizeof(*r)); + json_get_str(obj, "id", r->id, sizeof(r->id)); + json_get_str(obj, "label", r->label, sizeof(r->label)); + json_get_str(obj, "caps_summary", r->caps_summary, + sizeof(r->caps_summary)); + json_get_str(obj, "game_version", r->game_version, + sizeof(r->game_version)); + r->max_slots = json_get_int(obj, "max_slots", 2); + if (r->id[0]) ++n; + } + } + } + g_am.ruleset_count = n; + fprintf(stderr, "psx_lobby: automatch %d ruleset(s) for \"%s\"\n", + n, g_lc.filter_game_name); + /* Measure now rather than at queue time: a client that has already + * probed never waits out the server's probe grace. */ + if (n > 0 && g_am.rtt_ms < 0) automatch_probe_send(); + return 1; + } + if (strcmp(op, "automatch_queued") == 0) { + char probe[256]; + g_am.state = PSX_LOBBY_AUTOMATCH_QUEUED; + g_am.error[0] = '\0'; + g_am.queue_in_flight = 0; + json_get_str(json, "ticket_id", g_am.ticket_id, sizeof(g_am.ticket_id)); + g_am.queued_secs = 0; + g_am.pool = automatch_first_pool(json); + if (json_extract_object(json, "probe", probe, sizeof(probe))) + automatch_ingest_probe(probe); + if (g_am.rtt_ms < 0) automatch_probe_send(); + else automatch_send_rtt(); + fprintf(stderr, "psx_lobby: automatch queued (pool=%d)\n", g_am.pool); + return 1; + } + if (strcmp(op, "automatch_status") == 0) { + /* Pushed at most 1 Hz while queued. Not a state change: a status for + * a ticket we already gave up on must not resurrect the queue. */ + if (g_am.state != PSX_LOBBY_AUTOMATCH_QUEUED) return 1; + g_am.queued_secs = json_get_int(json, "queued_secs", g_am.queued_secs); + g_am.pool = automatch_first_pool(json); + return 1; + } + if (strcmp(op, "automatch_found") == 0) { + char opp[512]; + memset(&g_am.found, 0, sizeof(g_am.found)); + json_get_str(json, "match_id", g_am.match_id, sizeof(g_am.match_id)); + /* The opponent is an OBJECT: { handle, discord_username, country }. + * Read from the extracted object, not the whole frame -- "country" + * at top level would be found inside it either way, but "handle" is + * theirs and nothing else's. */ + if (json_extract_object(json, "opponent", opp, sizeof(opp))) { + json_get_str(opp, "handle", g_am.found.opponent, + sizeof(g_am.found.opponent)); + json_get_str(opp, "discord_username", g_am.found.opponent_username, + sizeof(g_am.found.opponent_username)); + json_get_str(opp, "country", g_am.found.opponent_country, + sizeof(g_am.found.opponent_country)); + } + json_get_str(json, "ruleset_id", g_am.found.ruleset_id, + sizeof(g_am.found.ruleset_id)); + json_get_str(json, "ruleset_label", g_am.found.ruleset_label, + sizeof(g_am.found.ruleset_label)); + g_am.found.est_rtt_ms = json_get_int(json, "est_rtt_ms", -1); + g_am.found.accept_secs = json_get_int(json, "accept_secs", 15); + g_am.found_deadline_ms = + lobby_mono_ms() + (uint64_t)(g_am.found.accept_secs > 0 + ? g_am.found.accept_secs : 0) * 1000ull; + g_am.state = PSX_LOBBY_AUTOMATCH_FOUND; + fprintf(stderr, "psx_lobby: automatch found opponent=\"%s\" " + "est_rtt=%d ms, %d s to answer\n", + g_am.found.opponent, g_am.found.est_rtt_ms, + g_am.found.accept_secs); + return 1; + } + if (strcmp(op, "automatch_accept_ok") == 0) { + /* The echo carries WHICH answer was acknowledged, and honouring it is + * not optional: setting ACCEPTED unconditionally reopened the gate on + * a player who had just declined. A declined ack ends the ticket + * here; the server's automatch_cancelled follows and is idempotent + * with this. */ + if (json_get_bool(json, "accept", 1)) { + if (g_am.state == PSX_LOBBY_AUTOMATCH_FOUND || + g_am.state == PSX_LOBBY_AUTOMATCH_ACCEPTED) + g_am.state = PSX_LOBBY_AUTOMATCH_ACCEPTED; + } else { + automatch_reset_queue_state(); + } + return 1; + } + if (strcmp(op, "automatch_requeue") == 0) { + /* The other side declined or let it lapse. Back to waiting, with the + * ticket intact -- this is not a failure and must not read as one. + * `queued:false` (lobby_limit) is the one case the ticket is gone. */ + if (json_get_bool(json, "queued", 1)) { + g_am.state = PSX_LOBBY_AUTOMATCH_QUEUED; + memset(&g_am.found, 0, sizeof(g_am.found)); + g_am.found_deadline_ms = 0; + g_am.pool = automatch_first_pool(json); + fprintf(stderr, "psx_lobby: automatch re-queued (the offer lapsed)\n"); + } else { + automatch_fail("The server is out of rooms -- try again shortly"); + } + return 1; + } + if (strcmp(op, "automatch_cancelled") == 0) { + char reason[32]; + int cooldown; + reason[0] = '\0'; + json_get_str(json, "reason", reason, sizeof(reason)); + cooldown = json_get_int(json, "cooldown_secs", 0); + automatch_reset_queue_state(); + /* A lapsed offer the player never answered is worth a line: the gate + * simply vanishes otherwise, and the cooldown it cost is invisible. */ + if (strcmp(reason, "timeout") == 0 || strcmp(reason, "declined") == 0) { + if (cooldown > 0) { + char line[96]; + snprintf(line, sizeof(line), + "%d Second Cooldown For Declining", cooldown); + automatch_fail(line); + } else if (strcmp(reason, "timeout") == 0) { + automatch_fail("The offer lapsed before you answered"); + } + } + return 1; + } + if (strcmp(op, "automatch_rtt_ok") == 0) { + return 1; /* acknowledgement only */ + } + return 0; +} + +/* An automatch refusal arrives as a plain `error`, so it has to be claimed + * or it would be filed as a join failure and the queue would sit waiting for + * a pairing that was never going to come. Only while an attempt is actually + * in flight: these codes are automatch's, but `error` is everyone's. + * Returns 1 when claimed. */ +static int automatch_claim_error(const char *code, const char *json) +{ + const char *why = NULL; + char line[160]; + if (!(g_am.queue_in_flight || + g_am.state == PSX_LOBBY_AUTOMATCH_QUEUED || + g_am.state == PSX_LOBBY_AUTOMATCH_FOUND || + g_am.state == PSX_LOBBY_AUTOMATCH_ACCEPTED)) + return 0; + if (!strcmp(code, "need_account")) why = "Sign in to use automatch"; + else if (!strcmp(code, "automatch_off")) why = "This server has no automatch queues"; + else if (!strcmp(code, "already_queued")) why = "This account is already in a queue"; + else if (!strcmp(code, "already_in_lobby")) why = "Leave the room first"; + else if (!strcmp(code, "unknown_ruleset")) why = "That queue type is gone -- refresh"; + else if (!strcmp(code, "need_disc_fp")) why = "The server needs a disc fingerprint this build did not send"; + else if (!strcmp(code, "version_not_pooled")) why = "This release is not the one this queue pools"; + else if (!strcmp(code, "mods_not_pooled")) why = "Turn off sim-affecting mods to queue"; + else if (!strcmp(code, "mod_not_approved")) why = "A mod exemption is not on this queue's approved list"; + else if (!strcmp(code, "slots_not_pooled")) why = "Automatch is two-player only"; + else if (!strcmp(code, "queue_full")) why = "The queue is full -- try again shortly"; + else if (!strcmp(code, "cooldown")) { + int retry = json_get_int(json, "retry_secs", 0); + if (retry > 0) + snprintf(line, sizeof(line), "%d Second Cooldown For Declining", retry); + else + snprintf(line, sizeof(line), "Cooldown For Declining"); + why = line; + } + if (!why) return 0; + automatch_fail(why); + return 1; +} + static void handle_server_json(const char *json); /* Parse complete unmasked server text frames from ws_pending; leave remainder. */ @@ -1807,7 +2520,7 @@ static void drain_ws_pending(void) } static void chat_push(const char *player_id, const char *from, const char *text, - const char *mid, int is_system) + const char *mid, const char *account, int is_system) { PsxLobbyChatMsg *m; int idx; @@ -1826,6 +2539,8 @@ static void chat_push(const char *player_id, const char *from, const char *text, snprintf(m->text, sizeof(m->text), "%s", text); snprintf(m->mid, sizeof(m->mid), "%s", (mid && !is_system) ? mid : ""); + snprintf(m->account, sizeof(m->account), "%s", + (account && !is_system) ? account : ""); /* Masked on arrival, whatever relayed it: the server already did this, * an older server did not, and the rule is that nothing unmasked is * ever shown. */ @@ -1837,7 +2552,7 @@ static void chat_push(const char *player_id, const char *from, const char *text, } static void schat_push(const char *player_id, const char *from, const char *text, - const char *mid) + const char *mid, const char *account) { PsxLobbyChatMsg *m; int idx; @@ -1855,6 +2570,7 @@ static void schat_push(const char *player_id, const char *from, const char *text snprintf(m->from, sizeof(m->from), "%s", from ? from : ""); snprintf(m->text, sizeof(m->text), "%s", text); snprintf(m->mid, sizeof(m->mid), "%s", mid ? mid : ""); + snprintf(m->account, sizeof(m->account), "%s", account ? account : ""); (void)rnet_chat_filter_apply(m->text, sizeof(m->text)); m->is_local = (g_lc.player_id[0] && player_id && strcmp(player_id, g_lc.player_id) == 0) ? 1 : 0; @@ -1906,6 +2622,9 @@ static void handle_server_json(const char *json) json_get_str(json, "op", op, sizeof(op)); if (strcmp(op, "welcome") == 0) { json_get_str(json, "player_id", g_lc.player_id, sizeof(g_lc.player_id)); + /* A fresh socket: the server holds the ticket and the ruleset answer + * per connection, so neither survives from the last one. */ + automatch_on_connection_reset(); { /* Say who we are AND what we are playing in the first message: * the server scopes players-online and server chat by title, and @@ -2158,6 +2877,15 @@ static void handle_server_json(const char *json) } if (strcmp(op, "joined") == 0) { psx_lobby_chat_clear(); + /* A ticket that reached a room is spent: `joined` is that moment for + * automatch (AUTOMATCH.md 8). Recorded before the reset below, which + * erases the evidence -- ACCEPTED (or FOUND, if the pair resolved in + * the same breath) is the only signal that this seat came from a + * queue rather than from somebody's room. */ + g_am.in_automatch_room = + (g_am.state == PSX_LOBBY_AUTOMATCH_ACCEPTED || + g_am.state == PSX_LOBBY_AUTOMATCH_FOUND); + automatch_reset_queue_state(); g_lc.in_lobby = 1; g_lc.is_host = 0; g_lc.join.ok = 1; @@ -2308,15 +3036,18 @@ static void handle_server_json(const char *json) char from_id[PSX_LOBBY_ID_LEN]; char from[PSX_LOBBY_NAME_LEN]; char mid[40]; + char account[PSX_LOBBY_ID_LEN]; text[0] = '\0'; from_id[0] = '\0'; from[0] = '\0'; mid[0] = '\0'; + account[0] = '\0'; json_get_str(json, "text", text, sizeof(text)); json_get_str(json, "from_player_id", from_id, sizeof(from_id)); json_get_str(json, "from", from, sizeof(from)); json_get_str(json, "mid", mid, sizeof(mid)); - schat_push(from_id, from, text, mid); + json_get_str(json, "from_account", account, sizeof(account)); + schat_push(from_id, from, text, mid, account); return; } if (strcmp(op, "chat") == 0) { @@ -2324,15 +3055,19 @@ static void handle_server_json(const char *json) char from_id[PSX_LOBBY_ID_LEN]; char from[PSX_LOBBY_NAME_LEN]; char mid[40]; + char account[PSX_LOBBY_ID_LEN]; text[0] = '\0'; from_id[0] = '\0'; from[0] = '\0'; mid[0] = '\0'; + account[0] = '\0'; json_get_str(json, "text", text, sizeof(text)); json_get_str(json, "from_player_id", from_id, sizeof(from_id)); json_get_str(json, "from", from, sizeof(from)); json_get_str(json, "mid", mid, sizeof(mid)); - chat_push(from_id, from, text, mid, json_get_bool(json, "system", 0)); + json_get_str(json, "from_account", account, sizeof(account)); + chat_push(from_id, from, text, mid, account, + json_get_bool(json, "system", 0)); return; } if (strcmp(op, "signal") == 0) { @@ -2364,11 +3099,17 @@ static void handle_server_json(const char *json) (void)flag; return; } + if (automatch_handle_op(op, json)) { + return; + } if (strcmp(op, "error") == 0) { char code[64]; json_get_str(json, "code", code, sizeof(code)); strncpy(g_lc.join.last_error, code, sizeof(g_lc.join.last_error) - 1); g_lc.join.last_error[sizeof(g_lc.join.last_error) - 1] = '\0'; + if (automatch_claim_error(code, json)) { + return; + } /* Create/join failures are fatal to the seat. In-lobby ops (kick/move * on an older server, not_host, …) must not clear join.ok or the room * looks abandoned after a rejected host action. */ @@ -2388,6 +3129,10 @@ static void handle_server_json(const char *json) if (strcmp(op, "lobby_closed") == 0 || strcmp(op, "left") == 0 || strcmp(op, "kicked") == 0) { psx_lobby_chat_clear(); + /* Leaving a room must never leave a queue ticket looking live, and + * no seat means no automatch room either. */ + automatch_reset_queue_state(); + g_am.in_automatch_room = 0; g_lc.swap_in_valid = 0; g_lc.swap_out = 0; g_lc.ice_rtt_suspended = 0; @@ -2805,6 +3550,7 @@ void psx_lobby_disconnect(void) { /* Never block the UI on DNS/connect — cancel and let pump reap. */ lobby_cancel_connect_async(); + automatch_on_connection_reset(); g_lc.ice_rtt_suspended = 0; lobby_ice_rtt_close(); @@ -3153,6 +3899,7 @@ static void lobby_ice_rtt_tick(void) void psx_lobby_pump(void) { char buf[4096]; + automatch_probe_poll(); #if defined(_WIN32) int n; #else From 3a275d49397ba2e2f4cc4399d76ffd47465e8d44 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen <alex@technicallycomputers.ca> Date: Sat, 12 Sep 2026 20:54:12 -0400 Subject: [PATCH 11/56] launcher: Allow Spectators is a preference, not an in-room setting The toggle lives in the Create Lobby modal, before any room exists, and its value goes out with `create`. The setter refused unless the client was already seated in a server lobby, so the modal read the unchanged preference back every frame and the checkbox snapped off. Refuse only in a LAN / Direct-IP room, which has no gallery to allow -- the SNES host's rule. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FPXfGGhBBBxQPcEzaMsauF --- runtime/src/main.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 4ed1904b..6a2268ea 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -11590,7 +11590,14 @@ namespace { return psx_lobby_allow_spectators_pref(); } int ae_np_allow_spectators_set(void*, int allow) { - if (!ae_np_use_ws_members()) return -1; + /* A PREFERENCE, read back by the Create Lobby modal and sent with + * `create` -- so it has to be settable before any room exists. + * Gating this on being seated (as it was) refused the toggle in the + * one place it is offered, and the modal, reading the unchanged + * value back each frame, snapped the switch off again. Only a LAN / + * Direct-IP room refuses, because it has no gallery to allow: same + * rule as the SNES host (snes_host_lobby.c cb_allow_spectators_set). */ + if (g_lnch_hosting_lan || g_lnch_joined_lan) return -1; psx_lobby_set_allow_spectators(allow); return 0; } From 72b3ead89270ba2bf25c45427ba24b1fd57a089f Mon Sep 17 00:00:00 2001 From: Aliet Exposito Garcia <aliet.exposito@proton.me> Date: Sun, 13 Sep 2026 01:30:10 -0400 Subject: [PATCH 12/56] feat(OpenGL renderer): add post presentation gamma control Some titles and enhancement mods/plugins might require display level gamma correction to improve shadow detail and overall visibility without altering guest rendering state. This introduces a public gamma adjustment interface for the OpenGL renderer via `gl_renderer_set_post_gamma`, that enables external plugins to adjust the display curve cleanly at runtime. This adjustment applies exclusively to game content and temporal interpolation passes while leaving host interface elements, bezels, and previously composed frames untouched. Modifying the gamma setting immediately invalidates the presentation buffer so that changes become visible on screen without requiring new video memory writes from the guest system. --- runtime/include/gpu_gl_renderer.h | 9 +++++ runtime/src/gpu_gl_renderer.c | 67 +++++++++++++++++++++++++++---- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/runtime/include/gpu_gl_renderer.h b/runtime/include/gpu_gl_renderer.h index 26aa5417..a0007bd1 100644 --- a/runtime/include/gpu_gl_renderer.h +++ b/runtime/include/gpu_gl_renderer.h @@ -130,6 +130,15 @@ void gl_renderer_set_display_aspect(int num, int den); void gl_renderer_set_scanlines(int on, float strength); int gl_renderer_get_scanlines(float *strength); +/* Presentation-only gamma adjustment. gamma = 1.0 is the identity; values + * above 1.0 lift shadow detail and values below 1.0 darken it. The adjustment + * is applied once to game content in the final GL presentation pass, including + * temporal interpolation, but not to the bezel, host OSD, black margins, or an + * already-composed hold-last image. Non-finite and out-of-range values are + * clamped to a safe range. Safe to call before GL context creation. */ +void gl_renderer_set_post_gamma(float gamma); +float gl_renderer_get_post_gamma(void); + /* Select full native-wide mirror rendering instead of the centre-splice fast * path. Textured edge expansion needs the complete mirror surface. */ void gl_renderer_set_wide_fast(int on); diff --git a/runtime/src/gpu_gl_renderer.c b/runtime/src/gpu_gl_renderer.c index b85cd551..e2befd6f 100644 --- a/runtime/src/gpu_gl_renderer.c +++ b/runtime/src/gpu_gl_renderer.c @@ -320,6 +320,11 @@ static void gl_swap_with_osd(void); static GLint s_present_uTex = -1, s_present_uUvRect = -1; static GLint s_present_uTexSize = -1, s_present_uSharpScale = -1; static GLint s_present_uSharp = -1; +static GLint s_present_uGamma = -1; +/* Host-side presentation state. The shader applies this only to the source + * image being presented; overlays and already-composed hold-last images set + * the uniform back to the identity explicitly. */ +static float s_present_gamma = 1.0f; /* Scanline post-process state (host display setting; see gl_renderer_set_scanlines). * s_scanline_on/strength are pushed to whichever program draws game content; * OSD/bezel and the already-composed hold-last re-present force it off so the @@ -331,7 +336,7 @@ static GLint s_present_uScanLines = -1, s_present_uScanScale = -1; static GLuint s_interp_prog = 0, s_interp_tex[3]; static GLint s_interp_uPrev = -1, s_interp_uCurr = -1; static GLint s_interp_uAlpha = -1, s_interp_uUvRect = -1; -static GLint s_interp_uBlendMode = -1; +static GLint s_interp_uBlendMode = -1, s_interp_uGamma = -1; static GLint s_interp_uScanline = -1, s_interp_uScanStrength = -1; static GLint s_interp_uScanLines = -1, s_interp_uScanScale = -1; static int s_interp_enabled = 0, s_interp_valid = 0; @@ -776,7 +781,8 @@ static void letterbox_rect_aspect(int ww, int wh, int num, int den, static void letterbox_rect(int ww, int wh, int *x, int *y, int *w, int *h); static void present_target_quad(GLuint tex, int tex_w, int tex_h, int x, int y, int w, int h, int linear, - int lx, int ly, int lw, int lh, int v_flip); + int lx, int ly, int lw, int lh, int v_flip, + int apply_gamma); static void coh_record(int kind, int x0, int y0, int x1, int y1) { GlCohEvent *e = &s_coh_ring[s_coh_seq % GL_COH_RING_CAP]; @@ -917,6 +923,7 @@ static const char *PRESENT_FS = "uniform vec2 u_tex_size;\n" "uniform vec2 u_sharp_scale;\n" "uniform int u_sharp;\n" + "uniform float u_gamma;\n" PSX_SCANLINE_UNIFORMS /* Catmull-Rom bicubic via 9 bilinear taps. Sharper than plain bilinear at * the same smoothness, with mild overshoot that reads as edge definition. @@ -967,12 +974,13 @@ static const char *PRESENT_FS = " c = texture(u_tex, uv);\n" " }\n" " c.rgb = psx_scanline(c.rgb, v_uv.y);\n" + " if (u_gamma > 0.0 && u_gamma != 1.0) c.rgb = pow(max(c.rgb, vec3(0.0)), vec3(1.0 / u_gamma));\n" " frag = c;\n" "}\n"; static const char *INTERP_FS = "#version 330\n" "in vec2 v_uv; uniform sampler2D u_prev; uniform sampler2D u_curr;\n" - "uniform float u_alpha; uniform int u_blend_mode; out vec4 frag;\n" + "uniform float u_alpha; uniform int u_blend_mode; uniform float u_gamma; out vec4 frag;\n" PSX_SCANLINE_UNIFORMS PSX_SCANLINE_FUNC "void main(){\n" @@ -986,6 +994,7 @@ static const char *INTERP_FS = " }\n" " vec4 c=mix(prev,curr,alpha);\n" " c.rgb=psx_scanline(c.rgb, v_uv.y);\n" + " if (u_gamma > 0.0 && u_gamma != 1.0) c.rgb=pow(max(c.rgb, vec3(0.0)), vec3(1.0/u_gamma));\n" " frag=c;\n" "}\n"; @@ -2664,6 +2673,15 @@ static void present_set_sharp(int mode, int tex_w, int tex_h, (float)out_h / (float)tex_h); } +/* The presentation shader is shared by game content, bezel art, host OSD, and + * hold-last redraws. Uniform state persists across draws, so every draw must + * explicitly choose whether it is raw game content or an already-composed + * image. */ +static void present_set_gamma(GLint uniform, int apply) { + if (uniform >= 0) + p_glUniform1f(uniform, apply ? s_present_gamma : 1.0f); +} + /* Push scanline uniforms into the currently-bound present or interpolation * program. * pitch_lines = the height of the TEXTURE that v_uv is normalized against @@ -2724,6 +2742,29 @@ int gl_renderer_get_scanlines(float *strength) { return s_scanline_on; } +void gl_renderer_set_post_gamma(float gamma) { + if (!isfinite(gamma)) + gamma = 1.0f; + if (gamma < 0.5f) gamma = 0.5f; + if (gamma > 3.0f) gamma = 3.0f; + if (fabsf(gamma - 1.0f) < 0.005f) gamma = 1.0f; + if (gamma == s_present_gamma) + return; + s_present_gamma = gamma; + + /* Gamma is presentation state, not guest VRAM state. Invalidate the + * presentation latches so a hot change is visible on an unchanged frame. + * Force at least 2 frames so all backbuffers in the swapchain update. */ + for (int i = 0; i < PRES_ROWS; i++) s_present_dirty[i] = ~0ull; + s_last_present_path = -1; + s_force_present_remaining = 2; + hold_invalidate(); +} + +float gl_renderer_get_post_gamma(void) { + return s_present_gamma; +} + /* Letterbox: largest num:den rect centered in the drawable. */ static void letterbox_rect_aspect(int ww, int wh, int num, int den, int *x, int *y, int *w, int *h) { @@ -2985,6 +3026,8 @@ int gl_renderer_init_context(SDL_Window *win) { p_glGetUniformLocation(s_present_prog, "u_sharp_scale"); s_present_uSharp = p_glGetUniformLocation(s_present_prog, "u_sharp"); + s_present_uGamma = + p_glGetUniformLocation(s_present_prog, "u_gamma"); s_present_uScanline = p_glGetUniformLocation(s_present_prog, "u_scanline"); s_present_uScanStrength = @@ -3007,6 +3050,8 @@ int gl_renderer_init_context(SDL_Window *win) { s_interp_uUvRect = p_glGetUniformLocation(s_interp_prog, "u_uv_rect"); s_interp_uBlendMode = p_glGetUniformLocation(s_interp_prog, "u_blend_mode"); + s_interp_uGamma = + p_glGetUniformLocation(s_interp_prog, "u_gamma"); glGenTextures(3, s_interp_tex); for (int i = 0; i < 3; i++) { glBindTexture(GL_TEXTURE_2D, s_interp_tex[i]); @@ -3127,6 +3172,7 @@ void gl_renderer_present(const uint32_t *pixels, int src_w, int src_h, int linea p_glActiveTexture(PSXGL_TEXTURE0); upload_present_tex(pixels, src_w, src_h, filt_mode >= 0 ? 1 : 0); p_glUseProgram(s_present_prog); p_glUniform1i(s_present_uTex, 0); + present_set_gamma(s_present_uGamma, 1); present_set_sharp(filt_mode, src_w, src_h, lw, lh); /* CPU present texture holds exactly the display rect, so v_uv spans it and * pitch == display height == src_h. */ @@ -3961,6 +4007,7 @@ static void interp_draw_quad(float alpha, int lx, int ly, int lw, int lh) { p_glUniform1i(s_interp_uCurr, 1); p_glUniform1f(s_interp_uAlpha, alpha); p_glUniform1i(s_interp_uBlendMode, s_interp_blend_mode); + present_set_gamma(s_interp_uGamma, 1); p_glUniform4f(s_interp_uUvRect, 0.f, 0.f, 1.f, 1.f); /* Interp textures hold exactly the display rect (uv_rect is 0..1), so pitch * == display height == s_interp_h. */ @@ -4090,6 +4137,7 @@ static void gl_draw_osd_image(const uint32_t *px, int ow, int oh, glViewport(vx, wh - vy - dh, dw, dh); p_glUseProgram(s_present_prog); p_glUniform1i(s_present_uTex, 0); + present_set_gamma(s_present_uGamma, 0); present_set_sharp(0, 0, 0, 0, 0); /* OSD is authored at output res */ PRESENT_SCANLINE(0, 0, 0); /* never scanline the host OSD */ /* Host OSD bitmaps are top-down (row 0 = top), same as guest CPU @@ -4182,7 +4230,8 @@ static void gl_swap_with_osd(void) { static void present_target_quad(GLuint tex, int tex_w, int tex_h, int x, int y, int w, int h, int linear, - int lx, int ly, int lw, int lh, int v_flip) { + int lx, int ly, int lw, int lh, int v_flip, + int apply_gamma) { float u0, v0, u1, v1; p_glBindFramebuffer(PSXGL_FRAMEBUFFER, 0); glViewport(lx, ly, lw, lh); @@ -4192,6 +4241,7 @@ static void present_target_quad(GLuint tex, int tex_w, int tex_h, glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, linear ? GL_LINEAR : GL_NEAREST); p_glUseProgram(s_present_prog); p_glUniform1i(s_present_uTex, 0); + present_set_gamma(s_present_uGamma, apply_gamma); /* The rasterized path already renders at the internal scale, so it has no * low-res source to reconstruct — keep the plain sample. */ present_set_sharp(0, 0, 0, 0, 0); @@ -4264,6 +4314,7 @@ static void present_bezel(int ww, int wh, int lx, int ly, int lw, int lh) { glBindTexture(GL_TEXTURE_2D, s_bezel_tex); p_glUseProgram(s_present_prog); p_glUniform1i(s_present_uTex, 0); + present_set_gamma(s_present_uGamma, 0); p_glUniform4f(s_present_uUvRect, 0.0f, 0.0f, 1.0f, 1.0f); PRESENT_SCANLINE(0, 0, 0); /* bezel art is not scanlined */ p_glBindVertexArray(s_present_vao); @@ -4309,7 +4360,7 @@ int gl_renderer_present_hold_last(void) { } /* v_flip=0: drawable capture is already screen-oriented. */ present_target_quad(s_hold_tex, s_hold_tw, s_hold_th, - 0, 0, s_hold_tw, s_hold_th, 0, lx, ly, lw, lh, 0); + 0, 0, s_hold_tw, s_hold_th, 0, lx, ly, lw, lh, 0, 0); } else { if (s_hold_force_4_3) letterbox_rect_aspect(ww, wh, 4, 3, &lx, &ly, &lw, &lh); @@ -4317,7 +4368,7 @@ int gl_renderer_present_hold_last(void) { letterbox_rect(ww, wh, &lx, &ly, &lw, &lh); present_target_quad(s_hold_tex, s_hold_tw, s_hold_th, 0, 0, s_hold_tw, s_hold_th, s_hold_linear, - lx, ly, lw, lh, 1); + lx, ly, lw, lh, 1, 1); /* Upgrade to drawable after letterboxing once. Live+interp leaves * HOLD_NATIVE (no Swap on main); drawable path is the soak-proven * full-window image (GL without interp). */ @@ -4382,7 +4433,7 @@ void gl_renderer_present_vram(int disp_x, int disp_y, int w, int h, int linear, } present_bezel(ww, wh, lx, ly, lw, lh); present_target_quad(s_hr_tex, VRAM_W, VRAM_H, - disp_x, disp_y, w, h, linear, lx, ly, lw, lh, 1); + disp_x, disp_y, w, h, linear, lx, ly, lw, lh, 1, 1); pres_record(GL_PRES_VRAM, disp_x, disp_y, w, h, lx, ly, lw, lh); hold_capture_drawable(); latency_ring_mark(LAT_SWAP_BEGIN); @@ -4486,7 +4537,7 @@ int gl_renderer_present_wide_fbo(int disp_x, int disp_y, int disp_h, int linear) return 1; } present_target_quad(tex, g_wide_w, VRAM_H, - 0, disp_y, g_wide_w, disp_h, linear, lx, ly, lw, lh, 1); + 0, disp_y, g_wide_w, disp_h, linear, lx, ly, lw, lh, 1, 1); pres_record(GL_PRES_WIDE, disp_x, disp_y, g_wide_w, disp_h, lx, ly, lw, lh); hold_capture_drawable(); latency_ring_mark(LAT_SWAP_BEGIN); From fa655333fb57e9c0cfb297f91c12c186f7d825bc Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:06:05 -0700 Subject: [PATCH 13/56] fix(runtime): guard optional netplay authentication dependencies --- recompiler/CMakeLists.txt | 4 ++ .../tests/test_optional_netplay_auth.py | 56 +++++++++++++++++++ runtime/src/main.cpp | 7 ++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 recompiler/tests/test_optional_netplay_auth.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index dae9d0bf..e2fc759b 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -496,6 +496,10 @@ if(BUILD_TESTING) find_package(Python3 COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) + add_test(NAME optional_netplay_auth + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_optional_netplay_auth.py + --cxx ${CMAKE_CXX_COMPILER}) add_test( NAME cli_project_packaging COMMAND ${Python3_EXECUTABLE} diff --git a/recompiler/tests/test_optional_netplay_auth.py b/recompiler/tests/test_optional_netplay_auth.py new file mode 100644 index 00000000..1a3b7af7 --- /dev/null +++ b/recompiler/tests/test_optional_netplay_auth.py @@ -0,0 +1,56 @@ +"""Preprocess actual main.cpp guards without needing SDL/UI/net dependencies. + +Only unrelated include directives are removed; conditionals and production +account code stay intact. This is a feature-boundary test, not a C++ link test. +Real no-netplay title compile/link regression is additionally required. +""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cxx", required=True) + ap.add_argument("--source", type=Path, + default=Path(__file__).resolve().parents[2] / "runtime/src/main.cpp") + args = ap.parse_args() + source = args.source.read_text(encoding="utf-8") + lines = source.splitlines() + source = "\n".join(line for line in lines if not re.match(r"\s*#\s*include\b", line) + or "recomp_net/" in line) + with tempfile.TemporaryDirectory(prefix="psx-auth-guard-") as tmp: + root = Path(tmp) + unit = root / "main-guards.cpp" + unit.write_text(source, encoding="utf-8") + is_msvc = Path(args.cxx).stem.lower() in ("cl", "clang-cl") + for enabled in (False, True): + if enabled: + (root / "recomp_net").mkdir() + (root / "recomp_net/auth.h").write_text("RNET_AUTH_HEADER_INCLUDED\n") + (root / "recomp_net/chat_filter.h").write_text("RNET_CHAT_HEADER_INCLUDED\n") + flags = ["RECOMP_LAUNCHER=1", "RECOMP_LAUNCHER_HAS_ACCOUNT=1", + "SDL_VERSION_ATLEAST(x,y,z)=1", "DEFAULT_DEBUG_PORT=4370"] + if enabled: + flags += ["PSX_HAS_RECOMP_NET=1", "PSX_HAS_LOBBY_CLIENT=1"] + if is_msvc: + cmd = [args.cxx, "/nologo", "/EP", "/TP", "/I"+str(root), + *["/D"+f for f in flags], str(unit)] + else: + cmd = [args.cxx, "-E", "-P", "-x", "c++", "-I", str(root), + *["-D"+f for f in flags], str(unit)] + p = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=30) + assert p.returncode == 0, p.stderr + if enabled: + for token in ("RNET_AUTH_HEADER_INCLUDED", "rnet_account_pump()", + "ae_np_account_login_begin", "account_available = ae_np_account_available"): + assert token in p.stdout, f"enabled path lost {token}" + else: + assert "rnet_account_" not in p.stdout, "offline launcher retained account linkage" + assert "RNET_ACCOUNT_" not in p.stdout, "offline launcher retained account state dependency" + print("PASS: offline launcher needs no auth headers/symbols; enabled account path retained") + +if __name__ == "__main__": + main() diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 6a2268ea..7ce17301 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -57,8 +57,8 @@ extern "C" void psx_event_step_conservative_env_init(void); #include "psx_netplay_rb.h" #include "psx_selfcheck.h" #include "psx_lobby_client.h" -#include "recomp_net/auth.h" #if defined(PSX_HAS_RECOMP_NET) +#include "recomp_net/auth.h" #include "recomp_net/chat_filter.h" /* chat profanity mask, LAN rooms too */ #endif #include "spu.h" @@ -10024,6 +10024,7 @@ namespace { } return psx_lobby_send_chat(line); } +#if defined(PSX_HAS_RECOMP_NET) /* ---- optional Discord sign-in ------------------------------------- * Thin adapters over psx_netplay_auth, which owns the HTTP, the worker * thread and the device key. Nothing here blocks a frame except the @@ -10061,6 +10062,8 @@ namespace { rnet_account_init(url.c_str()); } +#endif /* PSX_HAS_RECOMP_NET: account client is not linked in offline builds */ + /* ---- list scope -------------------------------------------------------- * The launcher forks LAN / Direct IP from online before the browser, and * only it knows which fork the player took. Without the scope a player @@ -10971,6 +10974,7 @@ namespace { } void ae_np_pump(void*) { +#if defined(PSX_HAS_RECOMP_NET) /* Redeems a stored device key on the first pump, so a machine that has * signed in once comes up signed in with no player action. */ ae_np_account_sync(); @@ -10991,6 +10995,7 @@ namespace { if (handle && handle[0] && (!shown || std::strcmp(shown, handle) != 0)) psx_lobby_set_display_name(handle); } +#endif psx_lobby_pump(); ae_np_lan_browse_pump(); ae_np_lan_udp_pump(); From ce7f646f420de233b2af8c91d06366e3978ec26e Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:10:42 -0700 Subject: [PATCH 14/56] fix(runtime): keep chat-report dependency inside enabled lobby path --- recompiler/tests/test_optional_netplay_auth.py | 15 ++++++++++++++- runtime/src/psx_lobby_client.c | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/recompiler/tests/test_optional_netplay_auth.py b/recompiler/tests/test_optional_netplay_auth.py index 1a3b7af7..b1073c88 100644 --- a/recompiler/tests/test_optional_netplay_auth.py +++ b/recompiler/tests/test_optional_netplay_auth.py @@ -50,7 +50,20 @@ def main(): else: assert "rnet_account_" not in p.stdout, "offline launcher retained account linkage" assert "RNET_ACCOUNT_" not in p.stdout, "offline launcher retained account state dependency" - print("PASS: offline launcher needs no auth headers/symbols; enabled account path retained") + # Compile the REAL offline lobby TU too: its early chat-report include + # used to leak an independent optional dependency before the #else. + lobby = args.source.parent / "psx_lobby_client.c" + includes = args.source.parent.parent / "include" + if is_msvc: + cmd = [args.cxx, "/nologo", "/TC", "/c", "/I"+str(includes), + str(lobby), "/Fo"+str(root / "lobby.obj")] + else: + cmd = [args.cxx, "-x", "c", "-c", "-I", str(includes), str(lobby), + "-o", str(root / "lobby.obj")] + p = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=30) + assert p.returncode == 0, p.stderr + print("PASS: offline launcher/auth guards and real offline lobby compilation; enabled account path retained") if __name__ == "__main__": main() diff --git a/runtime/src/psx_lobby_client.c b/runtime/src/psx_lobby_client.c index 4dc5d738..4d656440 100644 --- a/runtime/src/psx_lobby_client.c +++ b/runtime/src/psx_lobby_client.c @@ -1,5 +1,4 @@ #include "psx_lobby_client.h" -#include "recomp_net/chat_report.h" #include <ctype.h> #include <errno.h> @@ -173,6 +172,7 @@ void psx_lobby_clear_launch_pending(void) {} #include "recomp_net/lan_beacon.h" #include "recomp_net/rtt_probe.h" #include "recomp_net/chat_filter.h" +#include "recomp_net/chat_report.h" #include "recomp_net/auth.h" /* optional Discord session for `hello` */ #include "host_time.h" From 6c55d698bb3776a75e83981e63fb708c613c4866 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:06:06 -0700 Subject: [PATCH 15/56] fix(recompiler): rebuild CFG metadata and prove natural back edges --- recompiler/CMakeLists.txt | 7 + recompiler/include/control_flow.h | 22 ++- recompiler/src/code_generator.cpp | 11 +- recompiler/src/control_flow.cpp | 153 ++++++++++++++---- .../tests/control_flow_metadata_test.cpp | 135 ++++++++++++++++ 5 files changed, 285 insertions(+), 43 deletions(-) create mode 100644 recompiler/tests/control_flow_metadata_test.cpp diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index e2fc759b..fa35779f 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -346,6 +346,13 @@ target_include_directories(emitter_directive_line_test PRIVATE target_link_libraries(emitter_directive_line_test PRIVATE fmt rabbitizer) if(BUILD_TESTING) + add_executable(control_flow_metadata_test + tests/control_flow_metadata_test.cpp + src/control_flow.cpp) + target_include_directories(control_flow_metadata_test PRIVATE include) + target_link_libraries(control_flow_metadata_test PRIVATE fmt) + add_test(NAME control_flow_metadata_test COMMAND control_flow_metadata_test) + add_test(NAME emitter_directive_line_test COMMAND emitter_directive_line_test) add_executable(debug_trace_ranges_test diff --git a/recompiler/include/control_flow.h b/recompiler/include/control_flow.h index b29d8845..4f91f6f2 100644 --- a/recompiler/include/control_flow.h +++ b/recompiler/include/control_flow.h @@ -41,12 +41,13 @@ struct BasicBlock { // Control flow information ControlFlowInstr exit_instr; // How this block exits (branch/jump/return) std::vector<uint32_t> successors; // Next basic blocks (targets + fall-through) - std::vector<uint32_t> predecessors; // Blocks that jump to this one + std::vector<uint32_t> predecessors; // Unique in-graph sources, including unreachable ones // Block properties bool is_entry; // Function entry point bool is_exit; // Contains return instruction - bool is_loop_header; // Target of back edge (loop entry) + bool is_loop_header; // Header dominates a reachable back-edge source + bool is_reachable = false; // Reachable through known edges from a declared entry }; // Control flow graph for a function @@ -58,11 +59,19 @@ struct ControlFlowGraph { std::map<uint32_t, BasicBlock> blocks; // Map: block start address -> block std::vector<uint32_t> block_order; // Blocks in address order - // Loop information - std::vector<std::pair<uint32_t, uint32_t>> loops; // (header, back_edge_source) - int loop_count; + // Natural back edges in the KNOWN static graph, not backward addresses. + // Missing indirect edges / runtime CPS entries mean this is NOT proof that + // a block is private or that IRQ, I-cache or device checks may be removed. + std::vector<std::pair<uint32_t, uint32_t>> loops; // (dominating header, source) + int loop_count = 0; // Number of back edges, not distinct/nested loop bodies }; +// Rebuild reverse edges, static reachability and dominance-proven back edges. +// function_start, is_entry blocks and extra_entries are independent roots. +// Does not change block ownership, instruction ranges or successor ordering. +void rebuild_control_flow_metadata( + ControlFlowGraph& cfg, const std::vector<uint32_t>& extra_entries = {}); + class ControlFlowAnalyzer { public: explicit ControlFlowAnalyzer(const PS1Executable& exe); @@ -108,9 +117,6 @@ private: // Link basic blocks (build CFG edges) void link_basic_blocks(std::map<uint32_t, BasicBlock>& blocks); - // Detect loops (back edges in CFG) - std::vector<std::pair<uint32_t, uint32_t>> detect_loops( - const std::map<uint32_t, BasicBlock>& blocks); }; } // namespace PSXRecomp diff --git a/recompiler/src/code_generator.cpp b/recompiler/src/code_generator.cpp index e2c0adae..ef96f8d8 100644 --- a/recompiler/src/code_generator.cpp +++ b/recompiler/src/code_generator.cpp @@ -2837,11 +2837,12 @@ GeneratedFunction CodeGenerator::generate_function( } if (needs_fallthrough) { - // Emit fallthrough if the last block is reachable (has predecessors - // or is the entry block). Dead code after a return (e.g., padding - // nops) has no predecessors and should NOT get a fallthrough call. + // Incoming edges alone do not prove reachability: unreachable + // padding/loops can have predecessors too. This final safety net + // uses declared-entry reachability; block-level CPS/indirect entry + // handling above remains independent of this static metadata. const BasicBlock& last = cfg.blocks.at(cfg.block_order.back()); - bool is_reachable = last.is_entry || !last.predecessors.empty(); + bool is_reachable = last.is_entry || last.is_reachable; if (is_reachable) { body_ss << emit_stale_static_guard_named(fallthrough_name, " "); body_ss << fmt::format(" {}(cpu); /* fallthrough to next function */\n", @@ -2860,7 +2861,7 @@ GeneratedFunction CodeGenerator::generate_function( ((last_block.exit_instr.type == ControlFlowType::Branch || last_block.exit_instr.type == ControlFlowType::Jump) && last_block.successors.empty()); - bool is_reachable = last_block.is_entry || !last_block.predecessors.empty(); + bool is_reachable = last_block.is_entry || last_block.is_reachable; if (runs_off_end && is_reachable) { body_ss << fmt::format( " cpu->pc = 0x{:08X}u; return; /* image-edge fallthrough: tail-transfer */\n", diff --git a/recompiler/src/control_flow.cpp b/recompiler/src/control_flow.cpp index 833db11e..664f334a 100644 --- a/recompiler/src/control_flow.cpp +++ b/recompiler/src/control_flow.cpp @@ -187,8 +187,8 @@ uint32_t ControlFlowAnalyzer::analysis_walk_hi(const Function& func) const { return func.end_addr; } -// NOTE: find_block_boundaries / build_basic_blocks / link_basic_blocks / -// detect_loops below are NOT on the live path — analyze_function does its own +// NOTE: find_block_boundaries / build_basic_blocks / link_basic_blocks +// below are NOT on the live path — analyze_function does its own // single-pass walk (see the comment there). They are kept in sync with the // analysis bound anyway so wiring them up can never reintroduce the // guard-word-as-block-leader defect. @@ -346,27 +346,129 @@ void ControlFlowAnalyzer::link_basic_blocks(std::map<uint32_t, BasicBlock>& bloc } } -std::vector<std::pair<uint32_t, uint32_t>> ControlFlowAnalyzer::detect_loops( - const std::map<uint32_t, BasicBlock>& blocks) { +void rebuild_control_flow_metadata( + ControlFlowGraph& cfg, const std::vector<uint32_t>& extra_entries) { + cfg.loops.clear(); + cfg.loop_count = 0; + std::map<uint32_t, size_t> index; + std::vector<uint32_t> addresses; + for (auto& [addr, block] : cfg.blocks) { + index.emplace(addr, addresses.size()); + addresses.push_back(addr); + block.predecessors.clear(); + block.is_reachable = block.is_loop_header = false; + } + // A synthetic root preserves dominance with multiple alias entries. + const size_t root = addresses.size(), none = root + 1; + std::vector<std::vector<size_t>> succ(root + 1), pred(root + 1); + for (const auto& [addr, block] : cfg.blocks) { + const size_t source = index.at(addr); + for (uint32_t target : block.successors) { + auto it = index.find(target); + if (it == index.end()) continue; // External edges do not create blocks. + succ[source].push_back(it->second); + pred[it->second].push_back(source); + } + } + for (size_t i = 0; i < root; ++i) { + auto& incoming = pred[i]; + std::sort(incoming.begin(), incoming.end()); + incoming.erase(std::unique(incoming.begin(), incoming.end()), incoming.end()); + for (size_t source : incoming) + cfg.blocks.at(addresses[i]).predecessors.push_back(addresses[source]); + } + std::set<size_t> entries; + auto add_entry = [&](uint32_t addr) { + auto it = index.find(addr); + if (it != index.end()) entries.insert(it->second); + }; + add_entry(cfg.function_start); + for (const auto& [addr, block] : cfg.blocks) if (block.is_entry) add_entry(addr); + for (uint32_t addr : extra_entries) add_entry(addr); + for (size_t entry : entries) { + succ[root].push_back(entry); + pred[entry].push_back(root); // Synthetic edges are NOT public predecessors. + } - std::vector<std::pair<uint32_t, uint32_t>> loops; + // Iterative DFS avoids host stack exhaustion on large capture CFGs. + std::vector<bool> seen(root + 1, false); + std::vector<size_t> postorder; + std::vector<std::pair<size_t, size_t>> stack{{root, 0}}; + seen[root] = true; + while (!stack.empty()) { + auto& [node, next] = stack.back(); + if (next < succ[node].size()) { + const size_t child = succ[node][next++]; + if (!seen[child]) { + seen[child] = true; + stack.emplace_back(child, 0); + } + } else { + postorder.push_back(node); + stack.pop_back(); + } + } + std::reverse(postorder.begin(), postorder.end()); + std::vector<size_t> rank(root + 1, none), idom(root + 1, none); + for (size_t i = 0; i < postorder.size(); ++i) rank[postorder[i]] = i; + idom[root] = root; + // Reverse-postorder immediate-dominator fixed point. Storage is O(V+E), + // unlike a set of all dominators for every block. Intersections walk up + // strictly decreasing RPO ranks, never address order. + auto intersect = [&](size_t a, size_t b) { + while (a != b) { + while (rank[a] > rank[b]) a = idom[a]; + while (rank[b] > rank[a]) b = idom[b]; + } + return a; + }; + bool changed; + do { + changed = false; + for (size_t i = 1; i < postorder.size(); ++i) { + const size_t node = postorder[i]; + size_t parent = none; + for (size_t p : pred[node]) { + if (idom[p] == none) continue; + parent = parent == none ? p : intersect(parent, p); + } + if (idom[node] != parent) { idom[node] = parent; changed = true; } + } + } while (changed); - for (const auto& [addr, block] : blocks) { - // Check each successor - for (uint32_t successor : block.successors) { - // Back edge: successor address < current block address - if (successor <= addr) { - loops.push_back({successor, addr}); // (header, back_edge_source) - - // Mark loop header - if (blocks.count(successor)) { - const_cast<BasicBlock&>(blocks.at(successor)).is_loop_header = true; - } + // Dominator-tree intervals make each back-edge dominance query O(1). + std::vector<std::vector<size_t>> children(root + 1); + for (size_t node = 0; node < root; ++node) { + cfg.blocks.at(addresses[node]).is_reachable = seen[node]; + if (seen[node]) children[idom[node]].push_back(node); + } + std::vector<size_t> enter(root + 1), leave(root + 1); + size_t clock = 0; + stack.emplace_back(root, 0); + enter[root] = clock++; + while (!stack.empty()) { + auto& [node, next] = stack.back(); + if (next < children[node].size()) { + const size_t child = children[node][next++]; + enter[child] = clock++; + stack.emplace_back(child, 0); + } else { + leave[node] = clock++; + stack.pop_back(); + } + } + for (size_t source = 0; source < root; ++source) { + if (!seen[source]) continue; + for (size_t target : succ[source]) { + if (enter[target] <= enter[source] && leave[source] <= leave[target]) { + cfg.loops.emplace_back(addresses[target], addresses[source]); + cfg.blocks.at(addresses[target]).is_loop_header = true; } } } - - return loops; + std::sort(cfg.loops.begin(), cfg.loops.end()); + cfg.loops.erase(std::unique(cfg.loops.begin(), cfg.loops.end()), cfg.loops.end()); + cfg.loop_count = static_cast<int>(cfg.loops.size()); } ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { @@ -493,18 +595,9 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { cfg.blocks[block.start_addr] = block; } - // Detect back-edges (loops) - for (const auto& [addr, block] : cfg.blocks) { - for (uint32_t succ : block.successors) { - if (succ <= addr) { - cfg.loops.push_back({succ, addr}); - if (cfg.blocks.count(succ)) { - cfg.blocks[succ].is_loop_header = true; - } - } - } - } - cfg.loop_count = static_cast<int>(cfg.loops.size()); + auto entries = func.alias_group_entries; + entries.push_back(func.start_addr); + rebuild_control_flow_metadata(cfg, entries); for (const auto& [addr, block] : cfg.blocks) { cfg.block_order.push_back(addr); diff --git a/recompiler/tests/control_flow_metadata_test.cpp b/recompiler/tests/control_flow_metadata_test.cpp new file mode 100644 index 00000000..13a00828 --- /dev/null +++ b/recompiler/tests/control_flow_metadata_test.cpp @@ -0,0 +1,135 @@ +#include "control_flow.h" +#include <algorithm> +#include <iostream> +#include <random> +#include <set> +#include <stdexcept> + +using namespace PSXRecomp; +using Edges = std::vector<std::pair<uint32_t, uint32_t>>; +static void check(bool ok, const char* why) { + if (!ok) throw std::runtime_error(why); +} +static ControlFlowGraph graph(uint32_t count, const Edges& edges) { + ControlFlowGraph cfg{}; + cfg.function_start = 0; + for (uint32_t i = 0; i < count; ++i) { + BasicBlock b{}; + b.start_addr = i; b.is_entry = i == 0; + cfg.blocks.emplace(i, b); + } + for (auto [a, b] : edges) cfg.blocks.at(a).successors.push_back(b); + return cfg; +} +static std::set<uint32_t> reach(const ControlFlowGraph& cfg, + std::vector<uint32_t> roots, uint32_t omit = UINT32_MAX) { + std::set<uint32_t> seen; + while (!roots.empty()) { + auto node = roots.back(); roots.pop_back(); + if (node == omit || !cfg.blocks.count(node) || !seen.insert(node).second) continue; + const auto& next = cfg.blocks.at(node).successors; + roots.insert(roots.end(), next.begin(), next.end()); + } + return seen; +} +// Independent oracle: deleting a dominator must disconnect the source from +// ALL entries. No RPO/immediate-dominator algorithm is shared with production. +static void verify(ControlFlowGraph cfg, std::vector<uint32_t> extra = {}) { + auto original = cfg.blocks; + auto roots = extra; + roots.push_back(cfg.function_start); + for (auto& [a, b] : cfg.blocks) if (b.is_entry) roots.push_back(a); + auto reachable = reach(cfg, roots); + std::set<std::pair<uint32_t, uint32_t>> expected; + for (auto& [target, block] : cfg.blocks) { + auto without = reach(cfg, roots, target); + for (auto& [source, b] : cfg.blocks) + if (reachable.count(source) && !without.count(source) && + std::find(b.successors.begin(), b.successors.end(), target) != b.successors.end()) + expected.emplace(target, source); + } + rebuild_control_flow_metadata(cfg, extra); + check(std::set(cfg.loops.begin(), cfg.loops.end()) == expected, "back edges disagree with vertex-removal oracle"); + check(cfg.loop_count == static_cast<int>(expected.size()), "duplicate loop edges"); + for (auto& [a, b] : cfg.blocks) { + std::set<uint32_t> incoming; + for (auto& [p, pb] : original) + if (std::find(pb.successors.begin(), pb.successors.end(), a) != pb.successors.end()) incoming.insert(p); + check(b.predecessors == std::vector(incoming.begin(), incoming.end()), "reverse edges incomplete/duplicated"); + check(b.successors == original.at(a).successors, "successor order was changed"); + check(b.is_reachable == bool(reachable.count(a)), "reachability incorrect"); + bool header = false; + for (auto edge : expected) header |= edge.first == a; + check(b.is_loop_header == header, "loop header flag incorrect"); + } + auto loops = cfg.loops; + rebuild_control_flow_metadata(cfg, extra); + check(loops == cfg.loops, "metadata rebuild not idempotent"); + // Change edges after the first pass: no predecessor/loop state may linger. + for (auto& [a, b] : cfg.blocks) b.successors.clear(); + rebuild_control_flow_metadata(cfg, extra); + check(cfg.loops.empty() && cfg.loop_count == 0, "stale loop metadata"); + for (auto& [a, b] : cfg.blocks) check(b.predecessors.empty(), "stale predecessor metadata"); +} +static void live_analyzer() { + constexpr uint32_t base = 0x80010000; + PS1Executable exe{}; + exe.header.load_address = exe.header.initial_pc = base; + exe.header.file_size = 0x28; exe.code_data.resize(0x28); + auto word = [&](uint32_t off, uint32_t value) { + for (unsigned i = 0; i < 4; ++i) exe.code_data[off+i] = value >> (i*8); + }; + auto jump = [&](uint32_t off) { return 0x08000000u | ((base+off)>>2 & 0x03ffffffu); }; + word(0x00, jump(0x20)); word(0x08, 0x03e00008); + word(0x10, 0x03e00008); word(0x18, 0x03e00008); word(0x20, jump(0x10)); + Function f{}; f.start_addr = base; f.end_addr = base+0x28; f.size = 0x28; + auto cfg = ControlFlowAnalyzer(exe).analyze_function(f); + check(cfg.blocks.at(base+0x20).predecessors == std::vector<uint32_t>{base}, "live path omitted predecessor"); + check(cfg.blocks.at(base+0x10).predecessors == std::vector<uint32_t>{base+0x20}, "live backward edge omitted"); + check(cfg.loops.empty(), "acyclic backward jump was called a loop"); + check(!cfg.blocks.at(base+0x18).is_reachable, "padding is not entry-reachable"); + word(0x10, jump(0x20)); + cfg = ControlFlowAnalyzer(exe).analyze_function(f); + check(cfg.loops == Edges{{base+0x20, base+0x10}}, "forward-address natural back edge missed"); + f.alias_walk_lo = base; f.start_addr = base+0x10; + f.alias_group_entries = {base+0x10, base+0x18}; + cfg = ControlFlowAnalyzer(exe).analyze_function(f); + check(cfg.loops.empty(), "independent alias entry was ignored by dominance"); + check(cfg.blocks.at(base+0x18).is_reachable, "alias root not reachable"); + word(0x00, 0x10000001); // beq zero,zero,+1: taken and fallthrough both +8 + f.alias_walk_lo = 0; f.start_addr = base; f.alias_group_entries.clear(); + cfg = ControlFlowAnalyzer(exe).analyze_function(f); + check(cfg.blocks.at(base+8).predecessors == std::vector<uint32_t>{base}, "duplicate branch edges duplicated predecessor"); +} +int main() { + try { + live_analyzer(); + verify(graph(0, {})); + verify(graph(1, {{0,0}, {0,0}})); + verify(graph(4, {{0,3}, {3,1}, {1,2}})); // backward, acyclic + verify(graph(5, {{0,1}, {1,2}, {2,3}, {3,2}, {2,4}, {4,1}})); // nested + verify(graph(4, {{0,1}, {1,2}, {1,3}, {2,1}, {3,1}})); // two latches + verify(graph(4, {{0,1}, {0,2}, {1,2}, {2,1}, {2,3}})); // irreducible + verify(graph(5, {{0,1}, {2,3}, {3,2}, {3,4}})); // unreachable cycle/tail + verify(graph(4, {{0,1}, {1,2}, {2,1}, {2,3}}), {2,2,999}); // aliases + verify(graph(3, {{0,1}, {0,1}, {1,999}, {1,2}})); // duplicate/external + std::mt19937 rng(0x434647u); + for (unsigned i = 0; i < 1000; ++i) { + unsigned n = 2 + rng()%11; + Edges edges; + for (unsigned a = 0; a < n; ++a) + for (unsigned b = 0; b < n; ++b) if (rng()%9 == 0) edges.emplace_back(a,b); + verify(graph(n, edges), i%2 ? std::vector<uint32_t>{unsigned(rng()%n)} : std::vector<uint32_t>{}); + } + Edges chain; + for (unsigned i = 1; i < 12000; ++i) chain.emplace_back(i-1, i); + auto deep = graph(12000, chain); + rebuild_control_flow_metadata(deep); + check(deep.blocks.at(11999).is_reachable && deep.loops.empty(), "deep CFG traversal failed"); + std::cout << "PASS: live CFG, aliases, loops, 1000 oracle graphs, 12000-block chain\n"; + return 0; + } catch (const std::exception& e) { + std::cout << "FAIL: " << e.what() << '\n'; + return 1; + } +} From 90c62097e039a332ebecade9a08a52f9f95969ef Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:08:51 -0700 Subject: [PATCH 16/56] test(recompiler): cover CFG reachability in fallthrough emission --- recompiler/tests/recompiler_patch_test.cpp | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/recompiler/tests/recompiler_patch_test.cpp b/recompiler/tests/recompiler_patch_test.cpp index bb458265..f88df7df 100644 --- a/recompiler/tests/recompiler_patch_test.cpp +++ b/recompiler/tests/recompiler_patch_test.cpp @@ -1125,6 +1125,40 @@ void cfg_codegen_load_delay_test() { "CFG codegen preserves MIPS-I dependent load-delay value semantics"); } +void cfg_fallthrough_reachability_test() { + constexpr uint32_t base = 0x80010000u; + PSXRecomp::PS1Executable exe{}; + exe.header.load_address = exe.header.initial_pc = base; + exe.header.file_size = 24; + append_word(exe.code_data, 0x03e00008u); // entry returns + append_word(exe.code_data, 0); + append_word(exe.code_data, 0x08004004u); // unreachable jump to final block + append_word(exe.code_data, 0); + append_word(exe.code_data, 0x24020001u); + append_word(exe.code_data, 0); + PSXRecomp::Function f{}; + f.start_addr = base; f.end_addr = base + 24; f.size = 24; + auto check_tail = [&](bool reachable, const char* name) { + const auto cfg = PSXRecomp::ControlFlowAnalyzer(exe).analyze_function(f); + check(!cfg.blocks.at(base + 16).predecessors.empty(), + std::string(name) + " retains the real incoming edge"); + PSXRecomp::CodeGenerator generator(exe); + const auto named = generator.generate_function(f, cfg, "func_80010018").full_code; + const auto edge = generator.generate_function(f, cfg).full_code; + check((named.find("/* fallthrough to next function */") != std::string::npos) == reachable, + std::string(name) + " named final safety net uses entry reachability"); + check((edge.find("/* image-edge fallthrough: tail-transfer */") != std::string::npos) == reachable, + std::string(name) + " image-edge final safety net uses entry reachability"); + }; + check_tail(false, "unreachable predecessor"); + write_word(exe, base, 0x08004002u); // entry now reaches +8 -> +16 + check_tail(true, "reachable predecessor"); + write_word(exe, base, 0x03e00008u); + f.alias_walk_lo = base; f.start_addr = base + 16; + f.alias_group_entries = {base + 16}; + check_tail(true, "independent alias entry"); +} + } // namespace int main() { @@ -1140,6 +1174,7 @@ int main() { gte_codegen_classification_tests(); jump_table_producer_codegen_test(); cfg_codegen_load_delay_test(); + cfg_fallthrough_reachability_test(); } catch (const std::exception& e) { fmt::print(stderr, "FAIL unexpected exception: {}\n", e.what()); ++failures; From c5b3b7a214563a2ee9f653495356c23f3650b7e7 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:28:23 -0700 Subject: [PATCH 17/56] docs: record CFG repair title regression and merge boundaries --- docs/internal/CFG_METADATA_VALIDATION.md | 88 ++++++++++++++++++++++++ docs/internal/FAITHFUL_TIMING_PLAN.md | 15 ++++ 2 files changed, 103 insertions(+) create mode 100644 docs/internal/CFG_METADATA_VALIDATION.md diff --git a/docs/internal/CFG_METADATA_VALIDATION.md b/docs/internal/CFG_METADATA_VALIDATION.md new file mode 100644 index 00000000..447c27e9 --- /dev/null +++ b/docs/internal/CFG_METADATA_VALIDATION.md @@ -0,0 +1,88 @@ +# CFG metadata repair: validation and scope + +2026-09-12; PR [#354](https://github.com/RetroPortingToolKit/psxrecomp/pull/354). +Tracking: `beads-eio.3.148`. Original baseline: `3a275d49397ba2e2f4cc4399d76ffd47465e8d44`. + +## Confirmed defects and repair + +The live analyzer omitted reverse edges and called any backward-address edge a +loop. The real MIPS fixture `80010000 -> 80010020 -> 80010010 -> return` reproduces +both errors without a game asset. The separate BasicBlockAnalyzer was not the +affected analyzer. No retail-game failure is claimed to have been caused by this. + +Rebuild unique in-graph reverse edges, declared-entry reachability and natural +back edges proven by dominance. Include independent alias entries through a +synthetic root. Preserve successor ordering, instruction ranges and ownership. +An iterative traversal avoids recursive stack growth; storage is O(V+E). +Final codegen fallthrough guards now consume reachability, since unreachable +blocks may also have predecessors. Block-level CPS/indirect handoffs are unchanged. + +This graph is incomplete for runtime indirect/CPS entry. Neither one predecessor +nor a natural-loop label authorizes eliminating cache, slice, IRQ or device checks. +See [LLVM's loop definition](https://llvm.org/docs/LoopTerminology.html#loop-definition). + +## Automated checks + +- All 67 enabled recompiler CTests pass on the combined source tree, repeated + after rebasing onto the offline build fix. Three pre-existing disabled tests + (`interpreter_perf_guards`, `runtime_perf_diag_guards`, + `overlay_pair_dedup_runtime`) are not counted as passes. +- Real MIPS fixtures, alias roots, duplicate/external edges, true forward-address + back edges, nested/multiple-latch loops, irreducible/unreachable cycles, + idempotence and a 12,000-block chain pass. +- 1000 deterministic random graphs agree with an independent vertex-removal + dominance oracle, not a second copy of the production algorithm. +- Generated-tail cases distinguish unreachable incoming edges from reachable + entry/alias paths. Existing emitter, overlay ownership and BIOS tests pass. + +Configure the recompiler with `BUILD_TESTING=ON`; build, then run +`ctest --test-dir <build> --output-on-failure`. Stage the normally generated +runtime codegen-hash header before tests which compile a runtime overlay fixture. + +## Fresh title builds and bounded live regression + +Clang 22.1.8, Windows x64, RelWithDebInfo, OpenGL/headless, debug tools ON, +netplay/launcher UI OFF, no game mods, normal guest timing defaults. Fresh game +and SCPH-1001 BIOS generation; retail LLE BIOS boot (not BIOS HLE). The existing +deterministic HLE thread scheduler remains unchanged; this is not a claim of +an entirely HLE-free runtime. Each variant has isolated saves/build/cache data. +The warm run reuses that variant's cold-run overlay cache. + +Master could not compile this offline configuration because it unconditionally +required netplay headers and account symbols. PR [#355](https://github.com/RetroPortingToolKit/psxrecomp/pull/355) +was applied identically to baseline and fixed runtime sources. Thus the A/B +comparison isolates CFG changes, not different build prerequisites. + +| Title | Dispatch entries (both) | Native builds | Baseline cold/warm frames | Fixed cold/warm frames | +| --- | ---: | --- | --- | --- | +| Tomba! USA SCUS-94236 | 2989 | both pass | 11003 / 11003 | 11000 / 11004 | +| Mega Man X6 USA v1.1 SLUS-01395 | 2494 | both pass | 11015 / 11000 | 11026 / 11016 | +| Ape Escape USA SCUS-94423 | 3094 | both pass | 11005 / 11005 | 11004 / 11011 | + +All twelve runs reached the 11,000-frame threshold, exited 0, reported native +overlay dispatch, nonzero guest SPU samples and zero kernel-bless mismatches. +Inspected screenshots show Tomba FMV/title and MMX6/Ape gameplay attract demos. +Polling captures nearby frames, not identical deterministic snapshots; no +frame-perfect or guest-state equivalence claim follows from those pictures. +Headless host audio output is inactive: guest SPU activity is not a listening test. + +Dispatch tables, declarations and guest code-range manifests are byte-identical +between variants for each title. Codegen identity changes from `30317be3` to +`4fe894d2`, so their generated overlay caches are separated. Added final safety-net +code changes generated tails; one Tomba unknown-function stub moves between +adjacent shards because of the changed shard line budget, without losing its entry. + +Evidence remains local under `_validation-fmv-prs/{baseline,fixed}/{tomba,mmx6,ape}`: +isolated `game.toml`, generated output, native builds and `results/retail-{cold,warm}` +JSON reports/screenshots. No copyrighted asset, capture, profile or save is committed. + +## Merge boundary + +This is a class-level metadata/codegen correctness repair, not an FMV speedup. +Local suite coverage supports integration but does not prove absence of regressions +in unvisited levels, other platforms, online play or full playthroughs. No GitHub +checks were reported. Tomba2 and all game-repository pins are deliberately untouched. + +Keep [#356](https://github.com/RetroPortingToolKit/psxrecomp/pull/356) (opt-in IPO/PGO validation) +and [#357](https://github.com/RetroPortingToolKit/psxrecomp/pull/357) (IRQ batching RFC and cache +counterexample) in draft. No proposed IRQ/slice/I-cache suppression is bundled here. diff --git a/docs/internal/FAITHFUL_TIMING_PLAN.md b/docs/internal/FAITHFUL_TIMING_PLAN.md index 6a29d014..4597f872 100644 --- a/docs/internal/FAITHFUL_TIMING_PLAN.md +++ b/docs/internal/FAITHFUL_TIMING_PLAN.md @@ -213,6 +213,21 @@ on a fixed region -> next. ## 5. Status / Log (update every session) +- **2026-09-12 (FMV brief follow-ups, correctness separated from experiments):** + `fix/cfg-metadata-integrity` repairs missing live reverse edges and replaces + address-order loop guesses with multi-entry reachability/dominance metadata. + Final fallthrough safety nets consume reachability, not predecessor presence; + no IRQ, slice, I-cache, guest cycle or device check is suppressed. A separate + offline netplay-header build regression is fixed in PR #355. All 67 enabled + recompiler tests pass, including 1000 independent-oracle random CFGs. Fresh + Tomba/MMX6/Ape builds preserve dispatch tables and code ranges; twelve cold/ + warm 11,000-frame SCPH-1001 LLE runs exit 0 with inspected screenshots and + native overlay coverage. No performance or full-playthrough claim. Tomba2 + and game repository pins are untouched. See [validation](CFG_METADATA_VALIDATION.md) + and PR #354. IPO/PGO experiment #356 and IRQ-batching RFC/counterexample #357 + remain drafts pending evidence, not shipping timing optimizations. + Tracking: `beads-eio.3.148` through `beads-eio.3.151`. + - **2026-09-12 (WO-3 observed overlay interior recovery):** Branch `fix/observed-overlay-interiors` preserves validated, executed dispatch demands even when shared CFG ownership rejects their hostless interior seeds. From 2677fccce377b9f3728ac7e98cde8783a868fa52 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:29:52 -0700 Subject: [PATCH 18/56] docs: include bounded input smoke results and limits --- docs/internal/CFG_METADATA_VALIDATION.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/internal/CFG_METADATA_VALIDATION.md b/docs/internal/CFG_METADATA_VALIDATION.md index 447c27e9..8d1dd469 100644 --- a/docs/internal/CFG_METADATA_VALIDATION.md +++ b/docs/internal/CFG_METADATA_VALIDATION.md @@ -66,6 +66,13 @@ Polling captures nearby frames, not identical deterministic snapshots; no frame-perfect or guest-state equivalence claim follows from those pictures. Headless host audio output is inactive: guest SPU activity is not a listening test. +Four additional input smoke runs (`results/retail-input`) reach 14,000 frames +and exit 0: Tomba baseline/fixed 14004/14011; MMX6 14005/14020. Tomba enters +its initial scene/dialogue and responds to pause; MMX6 enters the new-game story +sequence. These are not complete gameplay routes. Inputs are wall-clock polled, +not lockstep: actual injection frames and dialogue boundaries differ, so this +does not establish deterministic input/state equivalence. + Dispatch tables, declarations and guest code-range manifests are byte-identical between variants for each title. Codegen identity changes from `30317be3` to `4fe894d2`, so their generated overlay caches are separated. Added final safety-net From 847c2baf379bdbd5353a6d941643284e5a4b6dd9 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:16:40 -0700 Subject: [PATCH 19/56] docs: stage FMV IRQ batching RFC and cache-state counterexample --- docs/internal/FMV_IRQ_BATCHING_RFC.md | 97 +++++++++++++++++++ runtime/CMakeLists.txt | 7 ++ runtime/tests/test_psx_icache_elision_guard.c | 40 ++++++++ 3 files changed, 144 insertions(+) create mode 100644 docs/internal/FMV_IRQ_BATCHING_RFC.md create mode 100644 runtime/tests/test_psx_icache_elision_guard.c diff --git a/docs/internal/FMV_IRQ_BATCHING_RFC.md b/docs/internal/FMV_IRQ_BATCHING_RFC.md new file mode 100644 index 00000000..72df9781 --- /dev/null +++ b/docs/internal/FMV_IRQ_BATCHING_RFC.md @@ -0,0 +1,97 @@ +# Draft RFC: FMV callback overhead without guest-state changes + +Status: design review only. No IRQ batching implementation or runtime flag is +added. Tracking: `beads-eio.3.150`. The contributor's actual prototype patch, +instrumentation patch, PE capture bytes and raw measurements are not available. + +## What is established + +WO-10 correctly identifies repeated generated block callbacks. With precise +slicing disabled, the runtime slice body returns early, but the DLL still calls +through its callback shim. That observation does not quantify the host cost. +WO-11's own trials did not demonstrate a repeatable batching speedup. + +The reported missing live CFG predecessors were independently reproduced. +The live address-order loop heuristic also labels acyclic backward jumps as +loops. These are separately addressed in [PR #354](https://github.com/RetroPortingToolKit/psxrecomp/pull/354). +Even corrected static metadata does not enumerate every indirect, alias or CPS +resume edge; it cannot prove that a block is runtime-private. + +## A concrete merge blocker + +The supplied design suppresses `psx_icache_fetch` along with IRQ checks. +That is not simply reducing host calls. The added native test executes the real +cache model with a header, a cold body on another line, and the header again: + +| State | Preserve body fetch | Omit body fetch | +| --- | ---: | ---: | +| Charged fetch cycles | 14 | 7 | +| Pending selected load give-back | 0 | 99 | +| Body cache tag | `0x80010020` | invalid (`1`) | + +The header executes in both cases. The effect matches the in-tree Beetle +`PS_CPU::ReadInstruction` miss charging and `ReadAbsorb` clearing. This is a +counterexample to the described operation, not an execution of the missing +prototype and not certification of every detail of the current cache model. + +## Required invariants for any future patch + +1. Every executed instruction retains its guest cycle charge, including delay + slots and cache/memory/multiply/GTE effects. Cache tags and load give-back + evolve identically; removing a host callback must not remove its semantics. +2. Devices and interrupts remain visible at required guest boundaries. One loop + iteration or four blocks is not a cycle/deadline bound. Account for pending + IRQs, COP0, nested/irreducible loops, memory effects and long blocks. +3. Every supported CPS/alias/indirect entry remains valid, including entries + not represented by static predecessor edges. A single predecessor is not a + privacy proof and static unreachability is not permission to delete code. +4. Unsupported analysis fails closed to the existing faithful path. No per-game + exclusions, generated-C edits or default-on speculative behavior. +5. Behavior-changing flags participate in emitted-artifact/cache identity. + Prove cold/warm and off/on/off transitions cannot reuse the wrong DLL. + +Natural-loop classification requires graph structure, not address order; see +[LLVM's loop definition](https://llvm.org/docs/LoopTerminology.html#loop-definition). + +## Supplemental evidence requested + +- Exact base revision, complete prototype patch, diagnostic schema patch and + the exact enabled flags/build commands. Do not reconstruct the patch from + prose and call it the contributor's tested implementation. +- For PE-specific attribution: capture JSON with bytes, `game.toml`, seeds, + matching DLL/manifest identities and bounded raw timing records. Those assets + are unavailable locally and must not be invented. +- Inclusive versus exclusive callback timing; treatment of host preemption, + reentrancy and CPS continuations. Unserialized RDTSC/global counters need + explicit validation before reporting cycles per guest iteration. +- Repeatable content-aligned trials with every target invocation represented, + not only frames where it was the maximum dispatch. Include per-frame tail + latency, audio underruns and guest-state/event-order comparison. + +## Acceptance before implementation is considered mergeable + +- Synthetic cold/conflicting/uncached fetch, load give-back, pending IRQ, + deadline crossing, COP0, delay-slot, alias/CPS, nested-loop and irreducible-CFG + tests; explicit first-divergence comparison where state/timing can change. +- Unmodified default-path output comparison and cache-key transition tests. +- Regenerated Tomba/MMX6/Ape LLE boot, FMV, menus and gameplay regression with + screenshots and progress/state evidence; BIOS/device timing calibrated against + the independent Beetle oracle. A visual smoke run alone is insufficient. +- A demonstrated benefit, measured separately from PGO/LTO/debug configuration. + +## Other unresolved brief items + +WO-8's PE decoder is a profiling lead, not proof that the guest algorithm alone +causes the stalls. Any predecode cache or worker-thread proposal first needs +verified input identity, memory ownership, dependencies and ordering. + +WO-9's mismatched DLL/manifest rejection is correct. If a longer rejection +history is wanted, propose a bounded ring exposed over TCP, preserving pair +validation and atomic publication. Do not add runtime stderr/file logging. + +Current `pgo-train` already finishes the successful profile-use/debug-OFF build. +Its failure report and exact source revision are needed to investigate the +claimed early stop. Runtime PGO/LTO do not automatically cover independently +compiled overlay DLLs. Those build experiments are separate from this RFC. + +Nothing in this draft should be merged as an IRQ performance fix. diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 9b74e42c..cead44ad 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -99,6 +99,13 @@ if(BUILD_TESTING) PSX_OVERLAY_DLL_BUILD=1) add_test(NAME psx_icache_fastpath_test COMMAND psx_icache_fastpath_test) + add_executable(psx_icache_elision_guard_test + tests/test_psx_icache_elision_guard.c src/psx_icache.c) + target_include_directories(psx_icache_elision_guard_test PRIVATE include) + target_compile_definitions(psx_icache_elision_guard_test PRIVATE + PSX_ENABLE_BLOCK_CYCLES=1 PSX_OVERLAY_DLL_BUILD=1) + add_test(NAME psx_icache_elision_guard_test COMMAND psx_icache_elision_guard_test) + foreach(_ram_geometry IN ITEMS retail expanded) add_executable(psx_memory_geometry_${_ram_geometry}_test tests/test_psx_memory_geometry.c) diff --git a/runtime/tests/test_psx_icache_elision_guard.c b/runtime/tests/test_psx_icache_elision_guard.c new file mode 100644 index 00000000..7c1de1d1 --- /dev/null +++ b/runtime/tests/test_psx_icache_elision_guard.c @@ -0,0 +1,40 @@ +/* Counterexample to dropping interior fetches while retaining header fetches. + * This tests the existing cache model, NOT an implementation of IRQ batching. + */ +#include "cpu_state.h" +#include "psx_icache.h" +#include <stdio.h> +#include <string.h> + +int g_ls_replay_active = 0; +static uint64_t cycles; +void psx_advance_cycles(uint32_t count) { cycles += count; } + +static uint64_t run(int omit_body, uint32_t *giveback, uint32_t *tag) { + CPUState cpu; + memset(&cpu, 0, sizeof(cpu)); + psx_icache_reset(); + g_psx_icache_active = 1; + cycles = 0; + psx_icache_fetch(&cpu, 0x80010000u); /* header */ + cpu.read_absorb_which = 1; + cpu.read_absorb[1] = 99; + if (!omit_body) psx_icache_fetch(&cpu, 0x80010020u); /* cold body line */ + psx_icache_fetch(&cpu, 0x80010000u); /* header again */ + *giveback = cpu.read_absorb[1]; + *tag = g_psx_icache_tv[8]; + return cycles; +} + +int main(void) { + uint32_t full_giveback, omitted_giveback, full_tag, omitted_tag; + uint64_t full = run(0, &full_giveback, &full_tag); + uint64_t omitted = run(1, &omitted_giveback, &omitted_tag); + if (full != 14 || omitted != 7 || full_giveback != 0 || + omitted_giveback != 99 || full_tag != 0x80010020u || omitted_tag != 1) { + puts("FAIL: expected cold-body refill/give-back counterexample changed"); + return 1; + } + puts("PASS: header-only fetch loses 7 guest cycles and changes cache/load state"); + return 0; +} From 20ec948ab1143677f90947644591c5d5f033a034 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 12 Sep 2026 23:14:09 -0700 Subject: [PATCH 20/56] build: add opt-in runtime IPO experiment with validation gates --- cmake/psx_runtime_ipo.cmake | 29 ++++++++ docs/internal/RUNTIME_IPO_EXPERIMENT.md | 73 ++++++++++++++++++++ recompiler/CMakeLists.txt | 10 +++ recompiler/tests/test_pgo_rebuild_flow.py | 62 +++++++++++++++++ recompiler/tests/test_runtime_ipo.py | 83 +++++++++++++++++++++++ runtime/runtime.cmake | 2 + 6 files changed, 259 insertions(+) create mode 100644 cmake/psx_runtime_ipo.cmake create mode 100644 docs/internal/RUNTIME_IPO_EXPERIMENT.md create mode 100644 recompiler/tests/test_pgo_rebuild_flow.py create mode 100644 recompiler/tests/test_runtime_ipo.py diff --git a/cmake/psx_runtime_ipo.cmake b/cmake/psx_runtime_ipo.cmake new file mode 100644 index 00000000..4ab86eca --- /dev/null +++ b/cmake/psx_runtime_ipo.cmake @@ -0,0 +1,29 @@ +include_guard(GLOBAL) + +option(PSX_RUNTIME_IPO + "Experimental IPO/LTO for optimized runtime targets (requires supported C/C++ toolchain)" + OFF) + +function(psxrecomp_apply_runtime_ipo target) + # OFF must not alter a parent project's explicit IPO policy or add flags. + if(NOT PSX_RUNTIME_IPO) + return() + endif() + include(CheckIPOSupported) + check_ipo_supported(RESULT _psx_ipo_ok OUTPUT _psx_ipo_error LANGUAGES C CXX) + if(NOT _psx_ipo_ok) + message(FATAL_ERROR + "PSX_RUNTIME_IPO=ON was requested, but C/C++ IPO is unsupported.\n" + "Use a compatible compiler/linker or configure PSX_RUNTIME_IPO=OFF.\n" + "${_psx_ipo_error}") + endif() + # Target/configuration-scoped: do not silently enable IPO in third-party + # libraries, build helpers, Debug, or custom configurations. + set_property(TARGET ${target} PROPERTY INTERPROCEDURAL_OPTIMIZATION FALSE) + set_property(TARGET ${target} PROPERTY INTERPROCEDURAL_OPTIMIZATION_DEBUG FALSE) + foreach(_psx_ipo_config RELEASE RELWITHDEBINFO MINSIZEREL) + set_property(TARGET ${target} + PROPERTY INTERPROCEDURAL_OPTIMIZATION_${_psx_ipo_config} TRUE) + endforeach() + message(STATUS "psxrecomp: experimental runtime IPO enabled for ${target} (optimized configurations)") +endfunction() diff --git a/docs/internal/RUNTIME_IPO_EXPERIMENT.md b/docs/internal/RUNTIME_IPO_EXPERIMENT.md new file mode 100644 index 00000000..5a4061ac --- /dev/null +++ b/docs/internal/RUNTIME_IPO_EXPERIMENT.md @@ -0,0 +1,73 @@ +# Draft: opt-in runtime IPO/LTO validation + +Status: experimental, OFF by default. Tracking: `beads-eio.3.149`. +The PE report is supplemental evidence, not a locally reproduced benchmark. +No IRQ, device deadline, I-cache, guest codegen or overlay ABI behavior changes. + +## Build contract + +`-DPSX_RUNTIME_IPO=ON` checks C and C++ IPO support and enables it only on runtime +targets, for Release, RelWithDebInfo and MinSizeRel. Debug and custom configurations +are not enabled by this option. Unsupported toolchains fail with an explicit +diagnostic rather than silently ignoring a requested experiment. + +The default OFF path changes no target property, including an explicit parent +project IPO policy. This is not a project-wide +`CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON`: third-party libraries, emitters and other +targets are not opted in. This narrower scope is deliberate for validation and +may not reproduce the contributor's project-wide result. + +CMake provides the target/configuration properties and support probe used here: +[IPO target property](https://cmake.org/cmake/help/latest/prop_tgt/INTERPROCEDURAL_OPTIMIZATION.html), +[CheckIPOSupported](https://cmake.org/cmake/help/latest/module/CheckIPOSupported.html). + +## PGO is a separate variable + +The existing `pgo-train` command configures generate/debug-ON, builds, trains, +then configures use/debug-OFF and builds again. It reports failures. The supplied +early-stop report does not match that successful control path; do not change +the pipeline without the failing revision, command and exit/error evidence. + +Use the existing `--cmake-extra` interface to pass the IPO option to both PGO +build phases if testing the combination. Inspect actual compile/link commands; +an ON cache entry alone does not prove every target was built with IPO. + +Runtime CMake PGO and this IPO option affect linked runtime/generated C/C++. +`compile_overlays.py` launches a separate `-shared -O2` compiler command with +neither profile flags nor IPO. They do not directly profile/optimize PE's dynamic +decoder DLL. Gains may come from linked code or callbacks, which must be measured. + +Do not replace a user's working profiles or cache when running experiments. +Use isolated project/build/save directories and record profile identities. + +## Automated checks + +- Default OFF leaves target properties unset; parent IPO choice preserved. +- Injected unsupported toolchain fails only when IPO is requested. +- Real supported-toolchain two-file C/C++ IPO build/link/execution. +- Debug compilation emits no IPO flags; unrelated target remains unchanged. +- Mocked PGO success/failure orchestration and direct overlay compiler command. + These mocks do not validate real profile collection or performance. + +The real IPO smoke check explicitly reports a skip when the tested compiler +cannot support IPO; that does not count as a positive link/run validation. + +## Evidence needed before promotion + +1. Repeated content-aligned LTO off/on trials with the same PGO profile, compiler, + debug flags, assets, input and cache state. LTO-only/full-factorial trials can + answer broader attribution questions, but are not required merely to measure + LTO's incremental effect with one fixed profile. +2. Per-frame tails/maxima and audio underruns, not only periodic average FPS. + The reported small incremental gain overlaps the earlier run-to-run spread. +3. Regenerated title regression: Tomba, MMX6 and Ape, including LLE boot, FMV, + menus and gameplay. Do not infer unchanged behavior from frame counts alone. +4. Shipping compiler/linker/platform and launcher-enabled coverage, with profile + mismatch/missing-function warnings visible during profile validation. Record + binary size, build/link time and peak memory as well as host performance. +5. Exact PE configuration, profile/capture identities and raw measurements from + the contributor. Their project-wide IPO result is not a guarantee for this + narrower runtime-only option or other titles. + +Leave this PR in draft pending that feedback. Do not claim a performance win or +enable IPO by default merely because the isolated compiler test passes. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index fa35779f..d44933fe 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -502,11 +502,21 @@ if(BUILD_TESTING) add_test(NAME reachable_discovery_test COMMAND reachable_discovery_test) find_package(Python3 COMPONENTS Interpreter) + if(Python3_Interpreter_FOUND AND CMAKE_GENERATOR STREQUAL "Ninja") + add_test(NAME runtime_ipo_configuration + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_runtime_ipo.py + --cmake ${CMAKE_COMMAND} --cc ${CMAKE_C_COMPILER} + --cxx ${CMAKE_CXX_COMPILER} --ninja ${CMAKE_MAKE_PROGRAM}) + endif() if(Python3_Interpreter_FOUND) add_test(NAME optional_netplay_auth COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_optional_netplay_auth.py --cxx ${CMAKE_CXX_COMPILER}) + add_test(NAME pgo_rebuild_flow + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_pgo_rebuild_flow.py) add_test( NAME cli_project_packaging COMMAND ${Python3_EXECUTABLE} diff --git a/recompiler/tests/test_pgo_rebuild_flow.py b/recompiler/tests/test_pgo_rebuild_flow.py new file mode 100644 index 00000000..0edfdaf0 --- /dev/null +++ b/recompiler/tests/test_pgo_rebuild_flow.py @@ -0,0 +1,62 @@ +"""No real build, game launch, cache cleanup, or profile mutation.""" +import argparse +import contextlib +import io +from pathlib import Path +import sys +from unittest import mock + +root = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(root), str(root / "tools")] +import psxrecomp_cli as cli +import compile_overlays as overlays + +# Existing source paths satisfy the file guards; ALL consumers of their +# contents and all mutating/build/process operations are mocked below. +def check_flow(fail_training=False): + args = argparse.Namespace(config=str(root / "CLAUDE.md"), project_root=str(root), + build_dir="review-not-created", target="psx-runtime", exe_basename="psx-runtime", + no_pgo=True, force_pgo=False, disc=str(root / "CLAUDE.md"), cmake_extra=[], + train_secs=100, train_runs=1) + progress = mock.Mock() + events = [] + def configure(*a, **kw): + events.append(("configure", kw["pgo"], kw["extra"][-1])) + def train(*a, **kw): + events.append(("train", kw["train_secs"], kw["train_runs"])) + if fail_training: + raise RuntimeError("synthetic training failure") + with contextlib.ExitStack() as stack: + replacements = dict(load_sections=lambda _: {}, + activate_embedded_toolchain=lambda *a: True, + clamp_future_mtimes=lambda *a, **kw: 0, + _cmake_configure=configure, + _cmake_build=lambda *a: events.append(("build",)), + run_pgo_train=train, + _resolve_runtime_exe=lambda *a: (root / "fake-runtime.exe", None)) + for name, replacement in replacements.items(): + stack.enter_context(mock.patch.object(cli, name, replacement)) + result = cli.cmd_pgo_train(args, progress) + expected = [("configure", "generate", "-DPSX_DEBUG_TOOLS=ON"), ("build",), ("train", 100, 1)] + if not fail_training: + expected += [("configure", "use", "-DPSX_DEBUG_TOOLS=OFF"), ("build",)] + assert result == cli.EXIT_OK + assert progress.result.call_args.kwargs["ok"] is True + else: + assert result == cli.EXIT_ERROR + progress.result.assert_not_called() + progress.error.assert_called_once() + assert events == expected, events + print("PGO flow", "failure" if fail_training else "success", events) + +check_flow() +check_flow(True) +with mock.patch.object(overlays, "_toolchain_env", return_value=({}, "fake-toolchain")), \ + mock.patch.object(overlays.subprocess, "run", return_value=mock.Mock(returncode=0)) as run, \ + contextlib.redirect_stdout(io.StringIO()): + assert overlays._compile_dll_direct("fake.c", "fake.dll", [], gcc="fake-gcc") +command = run.call_args.args[0] +assert "-O2" in command and "-shared" in command +assert not any("profile" in arg.lower() for arg in command) +print("Overlay DLL command (mocked subprocess):", command) +print("PASS: successful PGO finishes use/debug-OFF; failures report failure; overlay DLL command has no PGO flags") diff --git a/recompiler/tests/test_runtime_ipo.py b/recompiler/tests/test_runtime_ipo.py new file mode 100644 index 00000000..e8726fc7 --- /dev/null +++ b/recompiler/tests/test_runtime_ipo.py @@ -0,0 +1,83 @@ +"""Isolated configure/build checks: no BIOS, game or network required.""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + +def run(command, success=True): + p = subprocess.run(list(map(str, command)), capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=180) + if success is not None and (p.returncode == 0) != success: + raise AssertionError(f"unexpected exit {p.returncode}: {command}\n{p.stdout}\n{p.stderr}") + return p + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cmake", required=True) + ap.add_argument("--cc", required=True) + ap.add_argument("--cxx", required=True) + ap.add_argument("--ninja", required=True) + args = ap.parse_args() + with tempfile.TemporaryDirectory(prefix="psx-ipo-") as temp: + root = Path(temp) + (root / "first.c").write_text("int other(void); int main(void) { return other() != 42; }\n") + (root / "second.cpp").write_text('extern "C" int other(void) { return 42; }\n') + (root / "fake_modules").mkdir() + (root / "fake_modules/CheckIPOSupported.cmake").write_text(''' +function(check_ipo_supported) + set(_psx_ipo_ok FALSE PARENT_SCOPE) + set(_psx_ipo_error "synthetic unsupported toolchain" PARENT_SCOPE) +endfunction() +''') + (root / "CMakeLists.txt").write_text(''' +cmake_minimum_required(VERSION 3.20) +project(IPOSmoke C CXX) +if(SIMULATE_UNSUPPORTED) + list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/fake_modules") +endif() +include("''' + (ROOT / "cmake/psx_runtime_ipo.cmake").as_posix() + '''") +add_executable(subject first.c second.cpp) +add_library(unrelated OBJECT first.c) +if(PARENT_IPO) + set_property(TARGET subject PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +endif() +psxrecomp_apply_runtime_ipo(subject) +get_target_property(base subject INTERPROCEDURAL_OPTIMIZATION) +get_target_property(release subject INTERPROCEDURAL_OPTIMIZATION_RELEASE) +get_target_property(debug subject INTERPROCEDURAL_OPTIMIZATION_DEBUG) +get_target_property(others unrelated INTERPROCEDURAL_OPTIMIZATION_RELEASE) +file(WRITE "${CMAKE_BINARY_DIR}/properties.txt" "${base}|${release}|${debug}|${others}") +''', encoding="utf-8") + def configure(name, *options, ok=True): + dest = root / name + result = run([args.cmake, "-S", root, "-B", dest, "-G", "Ninja", + f"-DCMAKE_MAKE_PROGRAM={args.ninja}", f"-DCMAKE_C_COMPILER={args.cc}", + f"-DCMAKE_CXX_COMPILER={args.cxx}", "-DCMAKE_BUILD_TYPE=Release", *options], ok) + return dest, result + off, _ = configure("off", "-DSIMULATE_UNSUPPORTED=ON") + assert (off / "properties.txt").read_text() == "base-NOTFOUND|release-NOTFOUND|debug-NOTFOUND|others-NOTFOUND" + parent, _ = configure("parent", "-DPARENT_IPO=ON", "-DSIMULATE_UNSUPPORTED=ON") + assert (parent / "properties.txt").read_text().startswith("TRUE|release-NOTFOUND|") + _, failure = configure("unsupported", "-DPSX_RUNTIME_IPO=ON", "-DSIMULATE_UNSUPPORTED=ON", ok=False) + assert "PSX_RUNTIME_IPO=ON was requested" in failure.stderr + on, result = configure("on", "-DPSX_RUNTIME_IPO=ON", ok=None) + if result.returncode != 0: + assert "PSX_RUNTIME_IPO=ON was requested" in result.stderr, result.stderr + print("PASS: default/parent/unsupported guards; SKIP actual IPO link: toolchain reports unsupported") + return + assert (on / "properties.txt").read_text() == "FALSE|TRUE|FALSE|others-NOTFOUND" + build = run([args.cmake, "--build", on, "--verbose"]) + # MSVC and GCC/Clang spell IPO flags differently; this test's explicit + # Ninja/compiler arguments cover either native toolchain. + assert any(flag in build.stdout for flag in ("-flto", "/GL", "-GL")), build.stdout + run([on / ("subject.exe" if os.name == "nt" else "subject")]) + debug, _ = configure("debug", "-DPSX_RUNTIME_IPO=ON", "-DCMAKE_BUILD_TYPE=Debug") + dbg_build = run([args.cmake, "--build", debug, "--verbose"]) + assert not any(flag in dbg_build.stdout for flag in ("-flto", "/GL", "-GL")) + print("PASS: default OFF, parent policy, unsupported failure, scoped ON, actual IPO link/run, Debug OFF") + +if __name__ == "__main__": + main() diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 25e34500..db6d8c48 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -14,6 +14,7 @@ endif() # costly PGO rebuild/train cycle a silent no-op. set(PSX_PGO "" CACHE STRING "PGO mode: empty, generate, or use") set_property(CACHE PSX_PGO PROPERTY STRINGS "" generate use) +include("${PSXRECOMP_ROOT}/cmake/psx_runtime_ipo.cmake") include("${PSXRECOMP_ROOT}/cmake/psx_dependency_archive.cmake") include("${PSXRECOMP_ROOT}/runtime/chd_dependency.cmake") @@ -1389,6 +1390,7 @@ function(psxrecomp_add_runtime_target target) # CMAKE_C_STANDARD setting. cxx_std_17 likewise — game CMakeLists may omit # CMAKE_CXX_STANDARD; mod_packages.cpp must not compile as a pre-17 dialect. target_compile_features(${target} PRIVATE c_std_11 cxx_std_17) + psxrecomp_apply_runtime_ipo(${target}) if(NOT PSX_PGO STREQUAL "") if(NOT PSX_PGO STREQUAL "generate" AND NOT PSX_PGO STREQUAL "use") From 718f2b227e4591a2118b271ff2012f24df4e3e99 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 00:21:45 -0700 Subject: [PATCH 21/56] docs: record IPO-on title validation --- docs/internal/RUNTIME_IPO_EXPERIMENT.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/internal/RUNTIME_IPO_EXPERIMENT.md b/docs/internal/RUNTIME_IPO_EXPERIMENT.md index 5a4061ac..bf5722a5 100644 --- a/docs/internal/RUNTIME_IPO_EXPERIMENT.md +++ b/docs/internal/RUNTIME_IPO_EXPERIMENT.md @@ -52,7 +52,7 @@ Use isolated project/build/save directories and record profile identities. The real IPO smoke check explicitly reports a skip when the tested compiler cannot support IPO; that does not count as a positive link/run validation. -## Evidence needed before promotion +## Evidence needed before a default-on or performance promotion 1. Repeated content-aligned LTO off/on trials with the same PGO profile, compiler, debug flags, assets, input and cache state. LTO-only/full-factorial trials can @@ -69,5 +69,16 @@ cannot support IPO; that does not count as a positive link/run validation. the contributor. Their project-wide IPO result is not a guarantee for this narrower runtime-only option or other titles. -Leave this PR in draft pending that feedback. Do not claim a performance win or -enable IPO by default merely because the isolated compiler test passes. +## Independent merge review + +The opt-in path was rebuilt with Clang 22.1.8 and ThinLTO, then exercised for +more than 11,000 frames each in Tomba, Mega Man X6 and Ape Escape. All three +runs completed normally with LLE boot, native overlay dispatch, nonzero SPU +output, zero audio underruns and no kernel-state mismatch; representative +screenshots also showed normal output. The automated IPO and mocked PGO checks +above passed in the same review. + +This establishes a reasonable correctness baseline for merging the experimental +option while it remains OFF by default. It does not establish a performance win, +validate real PGO profile collection, or justify enabling IPO by default. The +remaining evidence above is still required for either claim. From 543896dbe79829ea850e0b934606034659a851e6 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 00:27:46 -0700 Subject: [PATCH 22/56] docs: promote IPO experiment from draft --- docs/internal/RUNTIME_IPO_EXPERIMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internal/RUNTIME_IPO_EXPERIMENT.md b/docs/internal/RUNTIME_IPO_EXPERIMENT.md index bf5722a5..bc0df120 100644 --- a/docs/internal/RUNTIME_IPO_EXPERIMENT.md +++ b/docs/internal/RUNTIME_IPO_EXPERIMENT.md @@ -1,4 +1,4 @@ -# Draft: opt-in runtime IPO/LTO validation +# Experimental opt-in runtime IPO/LTO validation Status: experimental, OFF by default. Tracking: `beads-eio.3.149`. The PE report is supplemental evidence, not a locally reproduced benchmark. From 75012474c728c56a866c77410eed1b5dd90b22f3 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 00:29:05 -0700 Subject: [PATCH 23/56] docs: promote FMV batching RFC from draft --- docs/internal/FMV_IRQ_BATCHING_RFC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internal/FMV_IRQ_BATCHING_RFC.md b/docs/internal/FMV_IRQ_BATCHING_RFC.md index 72df9781..b70f3ed1 100644 --- a/docs/internal/FMV_IRQ_BATCHING_RFC.md +++ b/docs/internal/FMV_IRQ_BATCHING_RFC.md @@ -1,4 +1,4 @@ -# Draft RFC: FMV callback overhead without guest-state changes +# RFC: FMV callback overhead without guest-state changes Status: design review only. No IRQ batching implementation or runtime flag is added. Tracking: `beads-eio.3.150`. The contributor's actual prototype patch, From ab2aaf95be7fc2ef2f9c3ffe6bf7de9aa4b92acd Mon Sep 17 00:00:00 2001 From: Aliet Exposito Garcia <aliet.exposito@proton.me> Date: Sun, 13 Sep 2026 13:33:16 -0400 Subject: [PATCH 24/56] fix(runtime): fix compilation failure when netplay is disabled When `PSX_NETPLAY` is turned off, the build fails because `ae_np_account_sync` is referenced without being declared in that configuration. This issue caused automated continuous integration pipelines and local builds to fail unless netplay was explicitly forced on. Wrapping this function call ensures that the runtime compiles cleanly for projects that do not require netplay support, restoring the ability to build with netplay disabled. --- runtime/src/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 7ce17301..4f9b2eed 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -10396,7 +10396,9 @@ namespace { ae_np_save_identity(nullptr, g_lnch_lobby_url.c_str()); /* The auth endpoints live on the same host and port as the lobby * socket, so the sign-in follows whatever server the player points at. */ - ae_np_account_sync(); + #if defined(PSX_HAS_RECOMP_NET) + ae_np_account_sync(); + #endif } int ae_np_connect(void*) { From 7025bb5f88fcdbded7efa51b14383f99665ade94 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 14:16:20 -0700 Subject: [PATCH 25/56] Fix rewind keyboard load conflicting with cancel aliases --- runtime/src/main.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 4f9b2eed..e19f8d7d 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -6285,11 +6285,11 @@ static void rewind_poll_nav(uint32_t now_ms) { const Uint8 *keys = SDL_GetKeyboardState(NULL); int left = keys[SDL_SCANCODE_LEFT] ? 1 : 0; int right = keys[SDL_SCANCODE_RIGHT] ? 1 : 0; - /* Overlay: A/Cross load, B/Circle close. Also Enter/Space/Esc. */ - int acc = (keys[SDL_SCANCODE_RETURN] || keys[SDL_SCANCODE_SPACE] || - keys[SDL_SCANCODE_Z] || keys[SDL_SCANCODE_A]) ? 1 : 0; - int can = (keys[SDL_SCANCODE_ESCAPE] || keys[SDL_SCANCODE_BACKSPACE] || - keys[SDL_SCANCODE_X] || keys[SDL_SCANCODE_B]) ? 1 : 0; + /* Direct menu keys supplement the configured pad bindings below. Letter + * aliases conflict with remaps: X is Cross by default, so treating X as + * Cancel sets both edges and silently cancels every keyboard load. */ + int acc = (keys[SDL_SCANCODE_RETURN] || keys[SDL_SCANCODE_SPACE]) ? 1 : 0; + int can = (keys[SDL_SCANCODE_ESCAPE] || keys[SDL_SCANCODE_BACKSPACE]) ? 1 : 0; /* Honor remapped Cross/Circle (and Select/R3) via the same pad path as * gameplay — GameController A/B alone miss keyboard-as-pad and remaps. */ uint16_t btn = pad_buttons_for(g_players[0], 1, true); From d4c45adf62e2562d7e53a4b1b42f2a2ed0f3446a Mon Sep 17 00:00:00 2001 From: Matt Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 16:33:46 -0700 Subject: [PATCH 26/56] fix(sio): remove title-specific card IRQ repair and opt-in (#362) --- docs/internal/FAITHFUL_TIMING_PLAN.md | 20 ++ docs/internal/SIO_CARD_NO_HACKS_AUDIT.md | 187 +++++++++++++ runtime/CMakeLists.txt | 9 + runtime/include/sio.h | 36 +-- runtime/src/debug_server.c | 9 +- runtime/src/gpu.c | 6 - runtime/src/interrupts.c | 6 - runtime/src/memory.c | 15 +- runtime/src/sio.c | 258 ++---------------- .../test_gpu_textured_dot_nw_shift_exec.c | 1 - runtime/tests/test_sio_card_irq_integrity.c | 94 +++++++ runtime/tests/test_sio_card_protocol.c | 4 +- runtime/tests/test_sio_spu_istat_ownership.c | 4 +- 13 files changed, 340 insertions(+), 309 deletions(-) create mode 100644 docs/internal/SIO_CARD_NO_HACKS_AUDIT.md create mode 100644 runtime/tests/test_sio_card_irq_integrity.c diff --git a/docs/internal/FAITHFUL_TIMING_PLAN.md b/docs/internal/FAITHFUL_TIMING_PLAN.md index 4597f872..3a036f59 100644 --- a/docs/internal/FAITHFUL_TIMING_PLAN.md +++ b/docs/internal/FAITHFUL_TIMING_PLAN.md @@ -213,6 +213,26 @@ on a fixed region -> next. ## 5. Status / Log (update every session) +- **2026-09-13 (SIO card hack removal — branch-only review checkpoint):** + Reproduced fixed-Ape-RAM IRQ7/mask injection after an absent-card probe, + plus SELECT-time ACK fabrication and INTC-pending ACK requeueing with the + flag disabled. Candidate removes the opt-in mechanism and its GPU/IRQ/mask + hooks, uses device-only handoff diagnostics, cancels deselected ACKs and + consumes elapsed shift/ACK deadlines. Also deletes the unused ChangeThread + deferral helper/state. Owner confirmed Ape's actual Load Game screen and a + captured screenshot shows populated save slots; baseline ON also qualified + and its ring proves the old repair was active. Final cleanup builds all three + title executables and 57 runtime test executables; 83 executed CTests pass, + one skips and two remain disabled. Cold 7000-frame and final warm 11000-frame + Tomba/MMX6 boot/FMVs/title/attract smokes pass with inspected screenshots, + zero kernel mismatches and zero dirty aborts. This is not a save/write/reload, + full gameplay, netplay or live Beetle claim. User-tested Ape predates only + the final dead-code cleanup; see the audit for binary identity and limits. + Original cards unchanged. Owner authorized integration after the human + checkpoint; refresh onto upstream 7025bb5f (rewind-key aliases only) and + recheck before merging. Game pins and other PRs remain untouched. + See `SIO_CARD_NO_HACKS_AUDIT.md`; central issue beads-eio.3.154. + - **2026-09-12 (FMV brief follow-ups, correctness separated from experiments):** `fix/cfg-metadata-integrity` repairs missing live reverse edges and replaces address-order loop guesses with multi-entry reachability/dominance metadata. diff --git a/docs/internal/SIO_CARD_NO_HACKS_AUDIT.md b/docs/internal/SIO_CARD_NO_HACKS_AUDIT.md new file mode 100644 index 00000000..a7380b21 --- /dev/null +++ b/docs/internal/SIO_CARD_NO_HACKS_AUDIT.md @@ -0,0 +1,187 @@ +# SIO card workaround audit — 2026-09-13 + +Status: full removal implemented; owner-confirmed Ape Load Game and targeted +regression checks pass. Owner authorized integration after the review checkpoint +on 2026-09-13. This document records the pre-merge validation evidence; +the associated PR and Beads issue record the landed commit. +Beads: beads-eio.3.154. + +Tracker notes are saved locally. Dolt push still fails because the central +remote's referenced data ref is missing; this task did not repair the tracker. + +## Scope and acceptance + +Owner requires no per-game correctness workaround, including an opt-in. Ape +Escape must work on the common implementation, and other titles must not be +broken to achieve that. Integration authorization is limited to this no-hack +method. Other PRs are review-only and game repository pins remain untouched. + +Base: upstream master 85cd26f05c44999731f6b3320fb8a871fba68e9b. +Branch: fix/sio-card-no-title-hacks-20260913. + +Integration refresh found one additional upstream commit, 7025bb5f, changing +rewind keyboard aliases in main.cpp. It does not overlap the card changes; +the final integration is refreshed onto that base and rechecked before merging. + +## Findings + +- 251456ee introduced the Ape repair; 1bbda282 ungated it on August 9. +- The default-enabled repair reads fixed Ape RAM addresses, injects IRQ7 and + overrides guest I_MASK writes. Synthetic unrelated RAM can trigger this + after an absent-card probe. +- Turning PSX_APE_CARD_UNSTICK off does not remove SELECT-time card ACK + fabrication, INTC-pending ACK requeueing, or one-transition clock walking. +- The original SPU I_STAT clearing was already removed in 3592a19e on August + 17. It is not a current defect; the candidate preserves that correction. +- The cited ApeEscapeRecomp/tools/ape_memcard_loadtest.py and + docs/APE_MEMCARD_LOAD.md are absent in the current game checkout and were + not found in its tracked path history. Boot is not a load-path acceptance test. + +Independent mechanism references inspected: local Beetle frontio.cpp +DoDSRIRQ/Write/Update and irq.cpp IRQ_Assert/IRQ_Write. SELECT deassertion +cancels pending DSR pulses; SIO does not write the INTC mask or wait for an +INTC acknowledgement before setting its own IRQ latch. A live Beetle retail +comparison has NOT yet been performed. + +## Candidate changes (uncommitted) + +- Delete the fixed-Ape-address repair, environment switch and I_MASK override. +- Remove its interrupt-check and GPU VBlank pump callers. +- Replace the handoff ring's fixed game RAM probes with SIO hardware state. +- Cancel pending card ACK on SELECT deassertion instead of flushing/requeueing it. +- Do not queue ACK events based on I_STAT.7. +- Consume elapsed shift/ACK deadlines without a per-call one-transition cap. +- Add production-SIO synthetic regression coverage and CTest registration. +- Delete the unused ChangeThread-deferral helper, its timer state and arming + calls. Repository search confirmed that the helper had no callers. Correct + the two old test-clock comments that incorrectly described it as load-bearing. + +There is no runtime environment getter or opt-in path left. The old variable +name remains only in CTest setup to prove setting it to 1 cannot reactivate +the repair. The fixed addresses remain only in the reproducer and two comments +about a separate, retained exception-register-restoration fix in interrupts.c. +Device-only diagnostic rings are retained; they cannot repair guest state. + +This is not a claim of complete SIO hardware accuracy. Other timing/device +semantics remain subject to the joint validation gate. + +## Evidence and limits + +Local evidence root: F:/Projects/psxrecomp/_validation-sio-card-20260913. + +Production SIO, Clang 22.1.8 -O2, synthetic clock/card and unrelated RAM: + +| Scenario | Master | Candidate | +| --- | --- | --- | +| Absent-card reset, repair enabled | 2 failures / 3 checks | 3 / 3 pass | +| Absent-card reset, repair disabled | 3 / 3 pass | switch deleted | +| SELECT before pending ACK, repair disabled | 2 failures / 4 checks | 4 / 4 pass | +| Existing INTC IRQ7, repair disabled | 2 failures / 3 checks | 3 / 3 pass | + +Candidate results above were executed with the old environment variable set to +1, demonstrating that it cannot reactivate the removed repair. The coarse-clock +scenario also passes: a single advance consumes both shift and ACK deadlines. + +The final cleaned source builds all 57 registered runtime test executables. +Full runtime CTest: **83 executed passes, zero failures, one skipped test and +two disabled tests** (CTest reports 84 including the skip). This includes all +four new scenarios, the existing card protocol test and SPU interrupt-ownership +test. Skipped: exit_critical_section_test. Disabled: sio_dualshock_rumble_test +and spu_fidelity_test. Netplay and the optional hardware GL readback test were +not registered in this dependency configuration and are not counted as passes. + +Fresh recompiler, retail SCPH1001 BIOS and Ape game generation/build completed. +The candidate Ape executable also builds. A missing retired pump caller in +gpu.c was caught at link time and removed, followed by a successful link. +The initial direct-emitter BIOS build lacked its fingerprint stamp. Running +the canonical tools/regen_bios.sh subsequently produced the genuine stamp; +the missing-stamp warning is resolved, not suppressed. SCPH1001 generation +emitted 1309 functions, with one pre-existing interpreted exclusion and one +skipped unsupported-COP1 function; this is not a zero-fallback coverage claim. + +Master-with-flag-off was explored on two bounded live runs using copied cards, +screenshots and TCP rings. Those runs are INCONCLUSIVE for Load Game: no +confirmed complete card-load route was captured. Attract/title/scene images +and boot card reads must not be represented as successful save loading. +The headless input/card route still needs qualification; an initial isolated +config omitted the game's locked analog controller table and was corrected. + +### Ape Load Game: qualified with the owner + +The owner supplied the route: title screen, Up, Load Game, X. On the preserved +master baseline with the repair ON, route-baseline-up.png shows the populated +save selector. Its handoff ring contains actual unstick/nest-pulse/mask-hold +events, confirming that the baseline exercised the repair. This is not merely +an enabled-but-idle control. + +On the no-hack candidate, the owner accidentally selected New Game first. +After relaunching the same binary (PID 51960, port 4591), the owner reported +"it looks good!". Captured and visually inspected +route-candidate-user-confirmed.png shows Wlau1 100%, Wlau2 and File3 83.3%, +and File4 No Data. This directly qualifies completion of the reported Checking +screen, not just boot or a transaction counter. + +Associated captures: route-candidate-user-confirmed-txns.json (721 closed +transactions), -handoff.json (hardware-state-only entries), and -freeze.json +(172 complete card reads, zero dirty-RAM aborts, SIO IRQ not pending at capture). +The single BASCUS-94423SYS card file contains gameplay save slots, as the +screenshots demonstrate; its filename does not imply settings-only content. + +The user-tested binary has SHA256 +A23179DF711CEB82AA15801CC1E0DF01658E6F34DB8FA3E04DB3092102E984AD. +After that confirmation, only the unused deferral helper/state and obsolete +comments were removed. The final build is psx-card-validation-final.exe, +SHA256 2958F8AE9B038234AA3FD4B000E78C51EB98315B6D97820D950DAA487031610D. +It was built under a different name to preserve the user's open test window. +The final-cleanup executable has not itself received another manual Load Game +test; the confirmed executable already lacked the opt-in and all active repair +paths. + +### Cross-title regression + +Fresh Tomba and MMX6 codegen/builds succeeded against the same candidate. +Initial cold runs: Tomba 7014 frames and MMX6 7013 frames, both exit 0, zero +kernel-blessing mismatches and zero dirty-RAM aborts. Inspected screenshots +show Tomba's intro FMV and MMX6's title/logo. These are bounded boot/FMV/title +smokes, not proof of gameplay or game-level save loading. Reports live under +<title>/results/sio-removal-cold/report.json in the evidence root. + +Final-cleanup builds and longer warm-cache smoke reruns also pass: + +| Title | Final frame | Exit | Inspected milestones | +| --- | --- | --- | --- | +| Tomba | 11002 | 0 | Intro FMV, title, attract FMV replay | +| MMX6 | 11001 | 0 | Title/loading transition, in-level attract demo | + +Both final reports have zero kernel-blessing mismatches and zero dirty-RAM +aborts. Reports and screenshots: <title>/results/sio-removal-final-warm. +No input was injected by these smoke runners; the MMX6 level is an attract +demo, not a player-controlled gameplay test. + +All title validation uses the retail SCPH1001 LLE BIOS with the existing shared +runtime scheduler, copied cards and isolated generated output/cache/configs. +UI, netplay, rewind, PGXP and Vulkan are disabled in these validation builds. + +Original game cards and dirty source checkouts are outside the mutation scope. +Only copied cards under the evidence root were used. Original Ape card hashes +were rechecked after the owner's test and are unchanged: + +- card1.mcd: 8D116A5E2D7F29F35EF9D3D8724B8CFFE85ED6C5870C198515AB59FC67A06262 +- card2.mcd: 7706C7D43EDAF8CB7618E574F03457105153E3BDC196DB803A600AD96A8F58E8 + +Baseline processes exited. The owner's candidate window was left untouched; +it had also exited by the final check. Automated smoke runners closed only +their own child processes; no listeners remain on ports 4591-4593. + +## Remaining limits / integration gate + +- No full title playthrough, actual save/write/reload round trip, Tomba/MMX6 + in-game card-menu acceptance, live Beetle comparison or netplay validation + has been performed. Do not convert smoke coverage into those claims. +- Tomba2 and game repository pins are untouched. PR361's other runtime changes + are not included in this isolated candidate. +- Final smoke reports and screenshots have been inspected, and the owner + authorized merging after this post-build checkpoint. No opt-in workaround and no + merging a known cross-title regression. The reported Ape Load Game blocker + is now owner-confirmed clear; + that does not assert complete SIO hardware accuracy or zero regression risk. diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index cead44ad..c7a30f97 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -23,6 +23,15 @@ psxrecomp_add_runtime_target(psx-runtime # (docs/ENHANCEMENTS.md G1.10). Titles get their <exe>_pgxp the same way. if(BUILD_TESTING) + add_executable(sio_card_irq_integrity_test tests/test_sio_card_irq_integrity.c) + target_include_directories(sio_card_irq_integrity_test PRIVATE include) + foreach(scenario absent deselect pending coarse) + add_test(NAME sio_card_irq_integrity_${scenario} + COMMAND sio_card_irq_integrity_test ${scenario}) + # The retired environment variable must not reactivate the workaround. + set_tests_properties(sio_card_irq_integrity_${scenario} PROPERTIES + ENVIRONMENT "PSX_APE_CARD_UNSTICK=1") + endforeach() # Optional real-hardware regression. Configure its dependencies explicitly; # absence must remain visible, never counted as a passing GPU test. set(PSX_GL_READBACK_SDL_ROOT "" CACHE PATH "SDL3 static build root for GL readback test") diff --git a/runtime/include/sio.h b/runtime/include/sio.h index b943aab6..ab17fd6b 100644 --- a/runtime/include/sio.h +++ b/runtime/include/sio.h @@ -237,48 +237,22 @@ uint32_t sio_get_seq(void); * transaction by issuing 0x01 on the SIO bus mid-read. */ int sio_card_protocol_active(void); -/* 1 while a card txn/protocol is live or a card ACK just fired. interrupts.c - * uses this to defer in-exception ChangeThread so the game's SIO IntRP can - * finish its A6C10/B4E38 handshake (Ape Escape LOAD). */ -int sio_should_defer_thread_switch(void); - -/* ------------------------------------------------------------------------- - * Ape Escape LOAD GAME — IMPORTANT memcard path - * - * LibCardIntRP needs one distinct IRQ7 edge per nest pop. SIO pacing (MT=1), - * ACK defer, I_MASK.7 hold, and nest IRQ pulses below exist so presence-probe - * → directory → file-list can complete. - * EXPERIMENT (TwistedMetal4Recomp): formerly netplay-inert / MotK-uncapped; - * ungated for local netplay desync testing. PSX_APE_CARD_UNSTICK=0 disables - * nest repair + bit7 hold. See ApeEscapeRecomp/docs/APE_MEMCARD_LOAD.md. - * ------------------------------------------------------------------------- */ - -/* Post-probe handoff ring: bit7 enable → TX 0x57 (debug_server card_handoff). */ +/* Post-probe handoff ring: diagnostic device state only, no guest-RAM probes. */ typedef struct { - uint8_t kind; /* 1=probe_abort 2=b7_set 3=b7_clear 4=tx 5=card_ack - * 6=unstick 7=select_flush_ack 8=ack_deferred_istat7 - * 9=nest_irq_pulse 10=b7_hold */ + uint8_t kind; /* 1=probe_abort 2=b7_set 3=b7_clear 4=tx 5=card_ack */ uint8_t byte; uint16_t imask; uint32_t pc; uint32_t func; - uint32_t a6c10; - uint32_t b4e30; - uint32_t b4e38; + uint32_t ctrl; + uint32_t stat; + uint32_t card_state; uint64_t cyc; } SioCardHandoffEntry; const SioCardHandoffEntry *sio_get_card_handoff(int *idx_out, int *count_out); int sio_card_handoff_cap(void); int sio_card_handoff_armed(void); void sio_card_handoff_on_imask(uint32_t old_mask, uint32_t new_mask); -/* Ape Escape: 1 while libcard nest is stuck post-probe — keep I_MASK.7 so - * IntRP can finish (BIOS otherwise clears bit7 ~tens of cycles after SELECT - * abort, before the pending SIO edge is taken). */ -int sio_card_should_hold_imask_bit7(void); -/* Ape Escape: nest repair pump when the guest waiter polls RAM only - * (no SIO MMIO → sio_tick never runs). IRQ re-edge only — never poke A6C10 / - * invent B4E38. */ -void sio_ape_card_unstick_pump(void); /* Netplay deferred-present coexistence: 1 while native memcard SIO is * busy and still making progress. Callers keep s_present_pending and diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index cb9dda37..c5692bf6 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -7010,7 +7010,7 @@ static void handle_imask_trace(int id, const char *json) send_fmt("]}\n"); } -/* Post-probe bit7 → TX 0x57 handoff (Ape Escape LOAD). */ +/* Post-probe SIO/INTC handoff; records hardware state, not game RAM. */ static void handle_card_handoff(int id, const char *json) { int count = json_get_int(json, "count", 64); @@ -7023,8 +7023,7 @@ static void handle_card_handoff(int id, const char *json) int start = count ? (idx - count + cap) % cap : 0; static const char *kinds[] = { - "?", "probe_abort", "b7_set", "b7_clear", "tx", "card_ack", "unstick", - "select_flush_ack", "ack_deferred_istat7", "nest_irq_pulse", "b7_hold" + "?", "probe_abort", "b7_set", "b7_clear", "tx", "card_ack" }; send_fmt("{\"id\":%d,\"ok\":true,\"armed\":%d,\"total\":%d,\"count\":%d,\"entries\":[", id, sio_card_handoff_armed(), total, count); @@ -7035,11 +7034,11 @@ static void handle_card_handoff(int id, const char *json) if (i) send_fmt(","); send_fmt("{\"kind\":\"%s\",\"byte\":\"0x%02X\",\"imask\":\"0x%03X\"," "\"pc\":\"0x%08X\",\"func\":\"0x%08X\"," - "\"a6c10\":\"0x%08X\",\"b4e30\":\"0x%08X\",\"b4e38\":\"0x%08X\"," + "\"ctrl\":\"0x%04X\",\"stat\":\"0x%04X\",\"card_state\":%u," "\"cyc\":%llu}", k, e->byte, e->imask, (unsigned)e->pc, (unsigned)e->func, - (unsigned)e->a6c10, (unsigned)e->b4e30, (unsigned)e->b4e38, + (unsigned)e->ctrl, (unsigned)e->stat, (unsigned)e->card_state, (unsigned long long)e->cyc); } send_fmt("]}\n"); diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index ea3c79d6..5d880cdc 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -3140,12 +3140,6 @@ void gpu_vblank_tick(void) { /* Trusted package-selected plugins run on guest VBlank, independent of * host presentation, pacing, turbo, or skipped frames. */ mod_runtime_on_vblank(); - /* Ape LOAD: RAM-only libcard waiter + idle-skip can starve sio_tick / - * interrupt-check pumps; VBlank always runs. */ - { - extern void sio_ape_card_unstick_pump(void); - sio_ape_card_unstick_pump(); - } psx_irq_raise(0, 0); /* IRQ_VBLANK (gpu_vblank_tick) */ if (!vblank_callback) return; diff --git a/runtime/src/interrupts.c b/runtime/src/interrupts.c index c8338220..e3e6c143 100644 --- a/runtime/src/interrupts.c +++ b/runtime/src/interrupts.c @@ -1294,12 +1294,6 @@ void psx_check_interrupts(CPUState* cpu) { * waiting for it to re-appear. If we tick here, the IRQ fires * during the delay loop BEFORE the clear, and the BIOS never * sees it. */ - /* Ape LOAD: libcard may poll nest/busy in RAM with no SIO MMIO, so - * sio_tick never runs. Throttled nest-repair pump only. */ - if ((total_checks & 0xFFu) == 0) { - extern void sio_ape_card_unstick_pump(void); - sio_ape_card_unstick_pump(); - } interrupts_service_scheduled_events(); diff --git a/runtime/src/memory.c b/runtime/src/memory.c index f4b3cb78..f59ba1d5 100644 --- a/runtime/src/memory.c +++ b/runtime/src/memory.c @@ -970,20 +970,7 @@ static void interrupt_write_stat_masked(uint32_t val, uint32_t mask) { static void interrupt_write_mask_masked(uint32_t val, uint32_t mask, uint8_t width) { uint32_t old = i_mask; uint32_t next = ((i_mask & ~mask) | (val & mask)) & 0x7FFu; - /* IMPORTANT (Ape Escape LOAD): BIOS clears I_MASK.7 immediately after - * the probe SELECT abort while A6C10 is still nested. That drops the - * nest_irq_pulse before LibCardIntRP can pop to idle / set B4E38. - * Hold bit7 until the nest unwinds (sio_card_should_hold_imask_bit7). - * EXPERIMENT: helper used to no-op under netplay; ungated for TM4 test. - * See ApeEscapeRecomp/docs/APE_MEMCARD_LOAD.md. */ - if ((old & 0x80u) && !(next & 0x80u)) { - extern int sio_card_should_hold_imask_bit7(void); - if (sio_card_should_hold_imask_bit7()) { - next |= 0x80u; - if (!(i_stat & 0x80u)) - i_stat |= 0x80u; - } - } + /* INTC mask writes are owned by the guest, never by a device repair. */ i_mask = next; imask_trace_record(old, i_mask, width); { diff --git a/runtime/src/sio.c b/runtime/src/sio.c index 08dcadd0..b1fb5eaf 100644 --- a/runtime/src/sio.c +++ b/runtime/src/sio.c @@ -413,37 +413,12 @@ int sio_card_protocol_active(void) { return 0; } -/* Hold ChangeThread-defer across the card ACK → guest IntRP epilogue window. - * SELECT deassert clears mc_state before DeliverEvent / nested pops finish, so - * protocol_active alone is too narrow. ~2 VBlank periods of tail cover the - * libcard A6C10/B4E38 handshake without pinning defer forever on a wedge. */ -static uint64_t s_card_ct_defer_until_cyc = 0; -/* Declared early: sio_should_defer_thread_switch() needs it before the txn ring. */ -static int sio_txn_open = 0; - -static void sio_arm_card_ct_defer_guard(void) { - extern uint64_t psx_get_cycle_count(void); - uint64_t now = psx_get_cycle_count(); - uint64_t until = now + 564480ull * 2ull; - if (until > s_card_ct_defer_until_cyc) - s_card_ct_defer_until_cyc = until; -} - #if SIO_MODEL_CYCLE_PACED static void sio_fire_ack_irq(void); static void sio_handle_shift_complete(void); #endif -int sio_should_defer_thread_switch(void) { - /* Diagnostic helper for card_handoff / future gates. Not wired into - * can_defer — forcing defer while A6C10 is nested prevented recovery. */ - if (sio_txn_open) return 1; - if (sio_card_protocol_active()) return 1; - extern uint64_t psx_get_cycle_count(void); - return psx_get_cycle_count() < s_card_ct_defer_until_cyc; -} - -/* ---- bit7 → TX 0x57 handoff ring (Ape LOAD diagnosis) ---- */ +/* ---- Post-probe SIO / interrupt-mask handoff ring ---- */ #define CARD_HANDOFF_CAP 256 static SioCardHandoffEntry s_card_handoff[CARD_HANDOFF_CAP]; static int s_card_handoff_idx = 0; @@ -455,137 +430,27 @@ static void card_handoff_push(uint8_t kind, uint8_t byte) { extern uint32_t g_debug_last_store_pc; extern uint32_t i_mask; extern uint64_t psx_get_cycle_count(void); - extern uint32_t psx_read_word(uint32_t addr); SioCardHandoffEntry *e = &s_card_handoff[s_card_handoff_idx]; e->kind = kind; e->byte = byte; e->imask = (uint16_t)(i_mask & 0x7FFu); e->pc = g_debug_last_store_pc; e->func = g_debug_current_func_addr; - e->a6c10 = psx_read_word(0x800A6C10u); - e->b4e30 = psx_read_word(0x800B4E30u); - e->b4e38 = psx_read_word(0x800B4E38u); + e->ctrl = sio_ctrl; + e->stat = sio_stat; + e->card_state = (uint32_t)mc_state; e->cyc = psx_get_cycle_count(); s_card_handoff_idx = (s_card_handoff_idx + 1) % CARD_HANDOFF_CAP; s_card_handoff_count++; } -/* ========================================================================= - * Ape Escape LOAD GAME — IMPORTANT memcard nest repair - * - * Without this block, LOAD wedges on the empty Checking starfield after - * 81 52 00 (A6C10 stuck nested, B4E38 never latches). Do not remove, - * collapse A6C10 to 0, or host-synth B4E20/B4E30/B4E38 without re-running - * ApeEscapeRecomp/tools/ape_memcard_loadtest.py. - * Doc: ApeEscapeRecomp/docs/APE_MEMCARD_LOAD.md - * - * EXPERIMENT (TwistedMetal4Recomp): formerly netplay-gated OFF so MotK - * leftover-time SIO walks stayed uncapped. Ungated here for local netplay - * desync testing — PSX_APE_CARD_UNSTICK=0 still disables. - * ========================================================================= - * - * LibCardIntRP (0x800226C8) only sets B4E38 when A6C10 becomes idle - * (bit31 / 0xFFFFFFFF) after pop. Merged IRQ7 edges leave depth≥1 so one - * pop lands on 0 and skips B4E38 — guest never re-arms bit7 / directory. - * Repair = re-edge IRQ7 only (no host B4E* / A6C10 stores). - * Default ON; PSX_APE_CARD_UNSTICK=0 disables. */ -static int s_ape_unstick_env = -1; -static int s_ape_unstick_pending = 0; -static uint64_t s_ape_unstick_cool_cyc = 0; -static int s_ape_torn_pulses = 0; - -static int ape_unstick_enabled(void) { - if (s_ape_unstick_env < 0) { - const char *e = getenv("PSX_APE_CARD_UNSTICK"); - if (e && e[0] == '0') - s_ape_unstick_env = 0; - else - s_ape_unstick_env = 1; - } - return s_ape_unstick_env; -} - -/* Defined later; sio_tick calls the pump. */ -void sio_ape_card_unstick_pump(void); - -/* Nest repair for Ape Escape LOAD (LibCardIntRP). - * - * Idle test is A6C10 bit31; publish runs only when a successful pop leaves - * bit31 set. Tip post-probe hang without collapse: a6=1, busy=2, B4E38=0 - * (one IntRP short). Re-edge IRQ7 until idle/publish — never poke A6C10 - * or invent B4E20/B4E30/B4E38. */ -static void ape_card_unstick_maybe(int allow_b4e38_synth) { - if (!ape_unstick_enabled()) return; - if (!s_ape_unstick_pending) return; - if (sio_txn_open) return; - extern uint64_t psx_get_cycle_count(void); - extern uint32_t psx_read_word(uint32_t addr); - extern void psx_write_word(uint32_t addr, uint32_t val); - uint64_t now = psx_get_cycle_count(); - if (now < s_ape_unstick_cool_cyc) return; - uint32_t a6 = psx_read_word(0x800A6C10u); - uint32_t b30 = psx_read_word(0x800B4E30u); - uint32_t b38 = psx_read_word(0x800B4E38u); - (void)allow_b4e38_synth; - if (b38 != 0u || (a6 & 0x80000000u) != 0u) { - s_ape_unstick_pending = 0; - s_ape_torn_pulses = 0; - return; - } - if (b30 == 0u) { - s_ape_unstick_pending = 0; - return; - } - - /* Nested post-probe / mid-scan: keep re-edging IRQ7 so IntRP can pop. - * Depth 1 needs two successful pops (1→0→FFFFFFFF) before B4E38 latches; - * a single pulse after SELECT is often eaten. No A6C10/B4E* host stores. */ - extern uint32_t i_mask; - if (!(i_mask & 0x80u)) - i_mask |= 0x80u; - i_stat &= ~0x80u; - psx_irq_raise(IRQ_SIO0, 0); - /* ~2ms between pulses — faster than one VB/8 so two pops can land - * before BIOS clears I_MASK.7 for good. */ - s_ape_unstick_cool_cyc = now + 564480ull / 32ull; - if (s_card_handoff_armed) - card_handoff_push(6, (uint8_t)(a6 & 0xffu)); - if (++s_ape_torn_pulses >= 128) - s_ape_unstick_pending = 0; -} - +/* Diagnostic only: guest writes own the interrupt mask. */ void sio_card_handoff_on_imask(uint32_t old_mask, uint32_t new_mask) { if (!s_card_handoff_armed) return; if (!(old_mask & 0x80u) && (new_mask & 0x80u)) card_handoff_push(2, 0); - else if ((old_mask & 0x80u) && !(new_mask & 0x80u)) { + else if ((old_mask & 0x80u) && !(new_mask & 0x80u)) card_handoff_push(3, 0); - ape_card_unstick_maybe(1); - } -} - -int sio_card_should_hold_imask_bit7(void) { - static int s_hold_left = -1; /* -1 = idle; 0 = exhausted; >0 remaining */ - if (!ape_unstick_enabled()) { s_hold_left = -1; return 0; } - if (!s_card_handoff_armed && !s_ape_unstick_pending) { - s_hold_left = -1; - return 0; - } - extern uint32_t psx_read_word(uint32_t addr); - uint32_t a6 = psx_read_word(0x800A6C10u); - uint32_t b38 = psx_read_word(0x800B4E38u); - if ((a6 & 0x80000000u) != 0u || b38 != 0u) { - s_hold_left = -1; - return 0; - } - if (s_hold_left < 0) - s_hold_left = 32; /* fresh nest episode — need room for 1→0→idle */ - if (s_hold_left == 0) - return 0; /* exhausted this episode */ - s_hold_left--; - if (s_card_handoff_armed) - card_handoff_push(10, (uint8_t)(a6 & 0xffu)); /* b7_hold */ - return 1; } const SioCardHandoffEntry *sio_get_card_handoff(int *idx_out, int *count_out) { @@ -691,7 +556,7 @@ static void sr_record(uint8_t kind, uint8_t tx, uint8_t rx) { static PSX_BSS SioTxnEntry sio_txn_buf[SIO_TXN_CAP]; static int sio_txn_idx = 0; /* next-write slot */ static uint32_t sio_txn_seq = 0; /* monotonic id of next-to-close */ -/* sio_txn_open declared above (defer guard). */ +static int sio_txn_open = 0; static SioTxnEntry sio_txn_cur; /* in-progress txn, flushed on close */ uint32_t sio_get_card_txns(const SioTxnEntry **buf_out, int *write_idx_out, int *open_out) { @@ -740,42 +605,18 @@ static void txn_close(uint8_t end_reason, uint8_t terminal_state, uint32_t func) sio_txn_cur.end_reason = end_reason; sio_txn_cur.terminal_state = terminal_state; sio_txn_cur.end_func = func; - /* Ape Escape LOAD presence probe: 81 52 00 + SELECT abort. Arm handoff - * watch for the follow-up bit7 → directory / file-list path. */ + /* Short read-presence probe: 81 52 00 + SELECT abort. Record the + * following interrupt-mask and card traffic without altering it. */ if (end_reason == SIO_TXN_END_ABORT_OTHER && sio_txn_cur.byte_count == 3 && sio_txn_cur.tx[0] == 0x81 && sio_txn_cur.tx[1] == 0x52) { s_card_handoff_armed = 1; - s_ape_torn_pulses = 0; card_handoff_push(1, 0x52); - sio_arm_card_ct_defer_guard(); - /* IMPORTANT (Ape Escape): tip often ends this probe one IntRP short - * (A6C10=1, Ready=0). Do NOT poke A6C10 — writing 0 skips a nest - * level and leaves torn idle (a6=0, no B4E38). With MT=1 + ACK - * defer + I_MASK.7 hold, re-edge IRQ7 and let LibCardIntRP pop - * naturally (1→0→FFFFFFFF + publish). */ - if (ape_unstick_enabled()) { - extern uint32_t psx_read_word(uint32_t addr); - extern uint32_t i_mask; - uint32_t a6 = psx_read_word(0x800A6C10u); - uint32_t b38 = psx_read_word(0x800B4E38u); - if ((a6 & 0x80000000u) == 0u && b38 == 0u) { - if (!(i_mask & 0x80u)) - i_mask |= 0x80u; - i_stat &= ~0x80u; - card_handoff_push(9, (uint8_t)(a6 & 0xffu)); /* nest_irq_pulse */ - psx_irq_raise(IRQ_SIO0, 0); - } - } } - /* Any finished card txn can leave A6C10 nested; arm unstick check. */ - if (ape_unstick_enabled()) - s_ape_unstick_pending = 1; sio_txn_buf[sio_txn_idx] = sio_txn_cur; sio_txn_idx = (sio_txn_idx + 1) % SIO_TXN_CAP; sio_txn_seq++; sio_txn_open = 0; - ape_card_unstick_maybe(0); } /* ---- SIO IRQ #7 delivery ring ---- */ @@ -2396,30 +2237,8 @@ void sio_write(uint32_t addr, uint32_t value) { * buffer; harmless to retain since the next read * overwrites it). */ if ((old_ctrl & SIO_CTRL_SELECT) && !(value & SIO_CTRL_SELECT)) { -#if SIO_MODEL_CYCLE_PACED - /* Finish a card byte that already shifted before killing the bus. - * Cancelling a pending ACK here drops an IntRP pop and leaves - * libcard A6C10 nested so B4E38 never arms the next transfer. */ - int preserve_card_ack = 0; - int preserve_ack_rem = 0; - if (active_device == DEV_MEMCARD) { - if (sio_shift_active && sio_shift_remaining <= 0) - sio_handle_shift_complete(); - if (sio_pending_ack) { - sio_pending_ack = 0; - sio_ack_remaining = 0; - if (s_card_handoff_armed) - card_handoff_push(7, 0); /* select_flush_ack */ - sio_fire_ack_irq(); - /* fire may re-queue while I_STAT.7 is still set — keep it - * across the protocol reset below. */ - if (sio_pending_ack) { - preserve_card_ack = 1; - preserve_ack_rem = sio_ack_remaining; - } - } - } -#endif + /* SELECT deassertion cancels the device's pending DSR pulse. + * It cannot create an ACK or preserve one for later delivery. */ if (active_device == DEV_MEMCARD && mc_state != MC_IDLE && sio_txn_open) { extern uint32_t g_debug_current_func_addr; txn_close(SIO_TXN_END_ABORT_OTHER, mc_state, g_debug_current_func_addr); @@ -2458,13 +2277,6 @@ void sio_write(uint32_t addr, uint32_t value) { sio_pending_ack_irq_en = 0; sio_bus_owner = SIO_OWNER_NONE; sio_bus_byte_index = 0; g_sio_timing_active = 0; - if (preserve_card_ack) { - sio_pending_ack = 1; - sio_ack_remaining = preserve_ack_rem > 0 ? preserve_ack_rem : 16; - sio_pending_ack_irq_en = 1; - sio_irq_pending_source = SIO_IRQ_SRC_CARD_ACK; - g_sio_timing_active = 1; - } /* Killing an in-flight shifter mid-cycle leaves sio_stat * with TX_RDY/TX_EMPTY clear (they were masked by * SHIFT_START and only sio_handle_shift_complete restores @@ -2472,8 +2284,6 @@ void sio_write(uint32_t addr, uint32_t value) { * SIO instead of busy-waiting forever on TX_RDY=0. */ sio_stat |= SIO_STAT_TX_RDY | SIO_STAT_TX_EMPTY; #endif - /* Protocol is idle now — finish libcard B4E38 if still nested. */ - ape_card_unstick_maybe(1); sr_record(SR_EVT_SELECT_DEASS, 0, 0); } break; @@ -2645,20 +2455,8 @@ static void sio_fire_ack_irq(void) { return; } - /* IMPORTANT (Ape Escape): serialize card ACK IRQs. If I_STAT.7 is still - * set, raising again merges into the same bit — LibCardIntRP pops once - * → A6C10 stays nested → B4E38 never set → LOAD wedges after the - * presence probe. Re-queue until the guest clears bit7. - * EXPERIMENT: was offline-only; ungated for TM4 netplay test. */ - if (card_ack && (i_stat & 0x80u)) { - sio_pending_ack = 1; - sio_ack_remaining = 16; - sio_pending_ack_irq_en = 1; - g_sio_timing_active = 1; - if (s_card_handoff_armed) - card_handoff_push(8, 0); /* ack_deferred_istat7 */ - return; - } + /* The device IRQ latch is independent of INTC acknowledgement. Do not + * queue an ACK for replay when the guest later clears I_STAT.7. */ sio_pending_ack_irq_en = 0; sio_stat |= SIO_STAT_IRQ; @@ -2666,7 +2464,6 @@ static void sio_fire_ack_irq(void) { psx_irq_raise(IRQ_SIO0, 0); /* SIO ACK IRQ */ event_ring_record_aux(EV_DEQ, (uint8_t)SRC_SIO, 0u); /* SIO ACK IRQ fired */ if (card_ack) { - sio_arm_card_ct_defer_guard(); if (s_card_handoff_armed) card_handoff_push(5, 0); } @@ -2753,23 +2550,13 @@ static uint64_t s_sio_advance_with_work = 0; uint64_t sio_get_advance_called(void) { return s_sio_advance_called; } uint64_t sio_get_advance_with_work(void) { return s_sio_advance_with_work; } -/* Walk shift/ack deadlines for `cycles`. - * - * IMPORTANT (Ape Escape LOAD): MAX_TRANSITIONS=1 so each walk delivers at - * most one shift/ack edge. Uncapped walks can fire shift-complete + ACK - * pairs back-to-back under the card ISR; the next ACK merges into the same - * I_STAT.7 edge and LibCardIntRP pops once for two bytes → nest stuck after - * 81 52 00. Do not fake-collapse A6C10 to compensate. - * EXPERIMENT: was uncapped under netplay/selfcheck (MotK leftover-time); - * always MT=1 here for TM4 netplay desync testing. */ +/* Consume every elapsed device deadline. Device time cannot stop after one + * transition merely to space interrupts for a particular guest handler. */ static void sio_pace_walk(int cycles) { int remaining = cycles; - int transitions = 0; - const int MAX_TRANSITIONS = 1; - while (transitions < MAX_TRANSITIONS && - (remaining > 0 || - (sio_shift_active && sio_shift_remaining <= 0) || - (sio_pending_ack && sio_ack_remaining <= 0))) { + while (remaining > 0 || + (sio_shift_active && sio_shift_remaining <= 0) || + (sio_pending_ack && sio_ack_remaining <= 0)) { int dt = remaining; int next_event = -1; if (sio_shift_active && sio_shift_remaining > 0 @@ -2793,11 +2580,9 @@ static void sio_pace_walk(int cycles) { remaining -= dt; if (next_event == 0) { sio_handle_shift_complete(); - transitions++; } else if (next_event == 1) { sio_pending_ack = 0; sio_fire_ack_irq(); - transitions++; } else { break; } @@ -2872,13 +2657,6 @@ void sio_tick(int cycles) { } } - /* Nested A6C10 after card traffic: also pumped from interrupt checks. */ - sio_ape_card_unstick_pump(); -} - -void sio_ape_card_unstick_pump(void) { - if (s_ape_unstick_pending) - ape_card_unstick_maybe(1); } void sio_tick_quantum(void) { diff --git a/runtime/tests/test_gpu_textured_dot_nw_shift_exec.c b/runtime/tests/test_gpu_textured_dot_nw_shift_exec.c index d1707e6c..4986df9f 100644 --- a/runtime/tests/test_gpu_textured_dot_nw_shift_exec.c +++ b/runtime/tests/test_gpu_textured_dot_nw_shift_exec.c @@ -208,7 +208,6 @@ uint32_t psx_compiled_irq_resume_pc(void) { return 0; } uint32_t psx_last_irq_check_pc(void) { return 0; } uint32_t psx_netplay_rb_sticky_bb_pc(void) { return 0; } void mod_runtime_on_vblank(void) {} -void sio_ape_card_unstick_pump(void) {} bool screen_kind_from_name(const char *name, ScreenKind *out) { (void)name; if (out) *out = SCREEN_RAW; diff --git a/runtime/tests/test_sio_card_irq_integrity.c b/runtime/tests/test_sio_card_irq_integrity.c new file mode 100644 index 00000000..0a4f3e83 --- /dev/null +++ b/runtime/tests/test_sio_card_irq_integrity.c @@ -0,0 +1,94 @@ +/* Device-level regression: guest RAM is not SIO hardware state. + * Uses production SIO with a synthetic clock/card, no BIOS or retail data. + * Independent references: Beetle frontio.cpp DoDSRIRQ/Write and irq.cpp + * IRQ_Assert/IRQ_Write: SIO cannot modify the INTC mask, and deasserting + * SELECT cancels the device's pending DSR pulse. + */ +#include "../src/sio.c" +#include <stdio.h> + +uint32_t i_stat, i_mask, g_debug_current_func_addr, g_debug_last_store_pc; +static uint64_t clock_now; +static int card_present; +static int checks, failures; + +uint64_t psx_get_cycle_count(void) { return clock_now; } +int psx_get_in_exception(void) { return 0; } +uint8_t psx_read_byte(uint32_t a) { (void)a; return 0; } +uint32_t psx_read_word(uint32_t a) { + return (a == 0x800A6C10u || a == 0x800B4E30u) ? 1u : 0u; +} +uint32_t memory_get_sr(void) { return 0; } +void psx_irq_raise(uint32_t bit, uint32_t detail) { (void)detail; i_stat |= 1u << bit; } +void debug_server_poll(void) {} +void debug_server_log_sio_write(uint32_t a, uint32_t v, uint8_t w) { (void)a; (void)v; (void)w; } +void event_ring_record_aux(uint16_t a, uint8_t b, uint32_t c) { (void)a; (void)b; (void)c; } +void starvation_ring_record(uint8_t a, uint8_t b, uint8_t c, uint16_t d, uint16_t e, + int f, int g, int h, int i, int j, uint8_t k, uint32_t l, + uint8_t m, uint8_t n, uint8_t o, uint8_t p, int q) { + (void)a; (void)b; (void)c; (void)d; (void)e; (void)f; (void)g; + (void)h; (void)i; (void)j; (void)k; (void)l; + (void)m; (void)n; (void)o; (void)p; (void)q; +} +void card_read_summary_record(uint8_t a, uint8_t b, uint16_t c, uint8_t d, + uint8_t e, const uint8_t *f) { + (void)a; (void)b; (void)c; (void)d; (void)e; (void)f; +} +void card_data_writes_arm(uint8_t a, uint16_t b, uint8_t c, uint8_t d) { + (void)a; (void)b; (void)c; (void)d; +} +int memcard_is_present(int slot) { return card_present && slot == 0; } +int memcard_read_sector(int slot, int sector, uint8_t *buf) { + (void)slot; (void)sector; memset(buf, 0, 128); return 0; +} +int memcard_write_sector(int slot, int sector, const uint8_t *buf) { + (void)slot; (void)sector; (void)buf; return 0; +} +void memcard_flush(int slot) { (void)slot; } + +static void check(const char *name, int ok) { + checks++; if (!ok) failures++; + printf("%s %s (I_MASK=%03X I_STAT=%03X)\n", ok ? "PASS" : "FAIL", name, i_mask, i_stat); +} +static void advance(unsigned cycles) { clock_now += cycles; sio_advance((int)cycles); } +int main(int argc, char **argv) { + const char *scenario = argc > 1 ? argv[1] : "absent"; + sio_init(); clock_now = 0; i_mask = 0x0d; i_stat = 0x201; + card_present = strcmp(scenario, "absent") != 0; + sio_write(0x1F801048, 0x0d); + sio_write(0x1F80104e, 0x88); + sio_write(0x1F80104a, card_present ? 0x1003 : 3); + sio_write(0x1F801040, 0x81); + advance(1088); + if (!card_present) { + sio_write(0x1F80104a, 0x40); + check("absent-card reset cannot enable interrupts", i_mask == 0x0d); + check("absent-card reset cannot inject IRQ7", i_stat == 0x201); + } else if (!strcmp(scenario, "deselect")) { + check("ACK has not arrived before SELECT drop", !(i_stat & 0x80)); + sio_write(0x1F80104a, 0x1001); + check("SELECT drop cancels pending ACK", !(i_stat & 0x80)); + advance(1000); + check("cancelled ACK cannot fire later", !(i_stat & 0x80)); + } else if (!strcmp(scenario, "pending")) { + i_stat |= 0x80; + advance(170); + check("ACK cannot queue on INTC pending bit", !sio_pending_ack); + check("ACK sets device IRQ latch even with INTC pending", (sio_peek_stat() & 0x200) != 0); + } else if (!strcmp(scenario, "coarse")) { + /* Restart with a single step spanning shift completion and ACK. */ + sio_write(0x1F80104a, 0x40); + sio_write(0x1F801048, 0x0d); + sio_write(0x1F80104e, 0x88); + sio_write(0x1F80104a, 0x1003); + sio_write(0x1F801040, 0x81); + advance(1088 + 170); + check("coarse advance consumes shift and ACK deadlines", (i_stat & 0x80) != 0); + check("coarse advance leaves no overdue ACK", !sio_pending_ack); + } else { + return 2; + } + check("SIO preserves unrelated SPU pending interrupt", (i_stat & 0x200) != 0); + printf("%d/%d checks passed\n", checks - failures, checks); + return failures ? 1 : 0; +} diff --git a/runtime/tests/test_sio_card_protocol.c b/runtime/tests/test_sio_card_protocol.c index f4566224..35de303f 100644 --- a/runtime/tests/test_sio_card_protocol.c +++ b/runtime/tests/test_sio_card_protocol.c @@ -44,9 +44,7 @@ uint32_t g_debug_last_store_pc = 0; int psx_get_in_exception(void) { return 0; } uint8_t psx_read_byte(uint32_t addr) { (void)addr; return 0; } uint32_t psx_read_word(uint32_t addr) { (void)addr; return 0; } -/* sio.c gates card-transfer deferral on `psx_get_cycle_count() < deadline` - * (s_card_ct_defer_until_cyc). A constant clock would defer forever and stall - * the state machine, so the stub advances monotonically. */ +/* Advance the synthetic guest clock so SIO shift and ACK deadlines elapse. */ uint64_t psx_get_cycle_count(void) { static uint64_t t; return t += 64; diff --git a/runtime/tests/test_sio_spu_istat_ownership.c b/runtime/tests/test_sio_spu_istat_ownership.c index ad69eb02..b64f249f 100644 --- a/runtime/tests/test_sio_spu_istat_ownership.c +++ b/runtime/tests/test_sio_spu_istat_ownership.c @@ -26,9 +26,7 @@ uint32_t g_debug_last_store_pc = 0; int psx_get_in_exception(void) { return 0; } uint8_t psx_read_byte(uint32_t addr) { (void)addr; return 0; } uint32_t psx_read_word(uint32_t addr) { (void)addr; return 0; } -/* sio.c gates card-transfer deferral on `psx_get_cycle_count() < deadline` - * (s_card_ct_defer_until_cyc). A constant clock would defer forever and stall - * the state machine, so the stub advances monotonically. */ +/* Advance the synthetic guest clock so SIO shift and ACK deadlines elapse. */ uint64_t psx_get_cycle_count(void) { static uint64_t t; return t += 64; From 9037e7bd239ad7fb6bed2c756297dbd7c4fafba2 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 16:50:26 -0700 Subject: [PATCH 27/56] Add opt-in retained texture banks and enhancement memory snapshots --- docs/RETAINED-TERRAIN-EXPERIMENT.md | 57 ++++++++++++++ runtime/CMakeLists.txt | 10 +++ runtime/include/boot_state.h | 7 +- runtime/include/gpu_gl_renderer.h | 4 + runtime/include/mod_memory.h | 17 ++-- runtime/include/mod_plugins.h | 24 ++++++ runtime/include/mod_texture_banks.h | 11 +++ runtime/runtime.cmake | 1 + runtime/src/boot_state.c | 18 ++++- runtime/src/debug_server.c | 11 +++ runtime/src/gpu.c | 24 ++++++ runtime/src/gpu_gl_renderer.c | 58 +++++++++++++- runtime/src/main.cpp | 16 ++++ runtime/src/memory.c | 40 ++++++++++ runtime/src/mod_runtime.cpp | 37 +++++++++ runtime/src/mod_texture_banks.c | 94 +++++++++++++++++++++++ runtime/tests/run_gl_readback_region.py | 8 +- runtime/tests/test_gl_readback_region.c | 21 +++++ runtime/tests/test_mod_gpu_dma_aperture.c | 2 +- runtime/tests/test_mod_memory_snapshot.c | 29 +++++++ runtime/tests/test_mod_runtime.cpp | 39 ++++++++++ runtime/tests/test_mod_texture_banks.c | 53 +++++++++++++ 22 files changed, 563 insertions(+), 18 deletions(-) create mode 100644 docs/RETAINED-TERRAIN-EXPERIMENT.md create mode 100644 runtime/include/mod_texture_banks.h create mode 100644 runtime/src/mod_texture_banks.c create mode 100644 runtime/tests/test_mod_memory_snapshot.c create mode 100644 runtime/tests/test_mod_texture_banks.c diff --git a/docs/RETAINED-TERRAIN-EXPERIMENT.md b/docs/RETAINED-TERRAIN-EXPERIMENT.md new file mode 100644 index 00000000..02df57f3 --- /dev/null +++ b/docs/RETAINED-TERRAIN-EXPERIMENT.md @@ -0,0 +1,57 @@ +# Retained-terrain services (experimental, opt-in) + +Issue: `beads-eio.3.156`. These services do not replace the faithful renderer. +They support a trusted, statically linked game plugin retaining original assets +outside guest RAM/VRAM. The first consumer is Crash's N. Sanity Beach experiment. + +- `psx_mod_read_disc_file`: emulation-thread-only, reads the original or derived + mounted ISO/CUE/BIN/CHD through an independent reader. Query with NULL/0, then + supply capacity for the whole file (64 MiB ceiling). Applies active raw/user + sector patches; never seeks the emulated CD controller. Current ISO path + lookup supports root files and one subdirectory. +- Immutable texture banks: stable nonzero 16-bit IDs, owned copies of PS1 + texels/indices and palettes, lazy OpenGL upload. ID 0 means ordinary VRAM. + 256 MiB aggregate host budget, maximum 1024x512 words per bank. A resolver can + rebuild a missing bank after a cold save-state restore. IDs must not be reused + for different assets during a process lifetime. Software/Vulkan and the GL + CPU-authoritative dual path report unsupported. +- Trusted GT3 packet arenas: bank ID in unused C1/C2 colour high bytes; optional + fractional XY/reciprocal-depth suffix. Only explicitly allocated packet + arenas interpret this extension; stock packets retain their meaning. See + `runtime/include/mod_plugins.h` for the wire layout. CPU-side immutable assets + are intentionally not serialized. No host/GL pointers are written to guest + memory or save states. +- GPU-DMA enhancement aperture: allocated-only physical 0x00800000..0x00FFFFFF + (8 MiB), outside all four PS1 RAM mirrors. Unallocated DMA addresses retain + retail 2 MiB folding. BIOS/expansion/MMIO aliases cannot map to this aperture. +- Save states with allocations use v8 and a required enhancement-memory section. + Both CPU and DMA allocations are captured; a layout cookie rejects older or + differently sized layouts before applying RAM. Vanilla saves still write v7. + Allocate during activation, not while rewind is recording. A matching layout + is necessary, not a substitute for the game's asset/version compatibility. + +The texture bank store lives for the process lifetime. This prototype is not a +general eviction/reload API, and there is no global terrain depth buffer or bulk +native mesh submission API yet. The Crash prototype's textured whole-level path +is substantially slower than its stock path; do not treat it as production-ready. + +## Validation + +Source-owned CTests: `mod_texture_banks_test`, `mod_memory_snapshot_test`, +`mod_gpu_dma_aperture_test`, `mod_runtime_test`, `gl_readback_runner_test` pass. +The real-GL readback fixture passes 82 checks at both 1x and 4x on NVIDIA, +including retained 4-bit CLUT/16-bit textures, VRAM bank transitions and painter +order. `run_gl_readback_region.py --sdl-library` accepts an explicit system SDL3 +import library in addition to its default static dependency. + +Crash's expanded-memory save/load and cold asset reconstruction were exercised +live. Baseline rewind was previously validated; this new enhanced path still +needs a dedicated live rewind acceptance run and performance profiling. + +## TCP-only view diagnostics + +`display_aspect num=21 den=9` changes the rendered view without moving, resizing +or focusing the window. `display_aspect num=3 den=1 adaptive=1` resumes adaptive +view, capped at 3:1. Neither writes settings. Unlike `ws_aspect` (GTE-only), this +updates the presented view and native-wide surface. Use `screenshot` for the +rendered native-wide buffer or `present_shot` for the composed window surface. diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index cead44ad..0c41349c 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -23,6 +23,16 @@ psxrecomp_add_runtime_target(psx-runtime # (docs/ENHANCEMENTS.md G1.10). Titles get their <exe>_pgxp the same way. if(BUILD_TESTING) + add_executable(mod_texture_banks_test tests/test_mod_texture_banks.c src/mod_texture_banks.c) + target_include_directories(mod_texture_banks_test PRIVATE include) + add_test(NAME mod_texture_banks_test COMMAND mod_texture_banks_test) + if(NOT MSVC) + add_executable(mod_memory_snapshot_test tests/test_mod_memory_snapshot.c) + target_include_directories(mod_memory_snapshot_test PRIVATE include src) + target_compile_options(mod_memory_snapshot_test PRIVATE -O2 -flto -ffunction-sections -fdata-sections) + target_link_options(mod_memory_snapshot_test PRIVATE -flto -Wl,--gc-sections) + add_test(NAME mod_memory_snapshot_test COMMAND mod_memory_snapshot_test) + endif() # Optional real-hardware regression. Configure its dependencies explicitly; # absence must remain visible, never counted as a passing GPU test. set(PSX_GL_READBACK_SDL_ROOT "" CACHE PATH "SDL3 static build root for GL readback test") diff --git a/runtime/include/boot_state.h b/runtime/include/boot_state.h index 5ecf29b9..3ee203e1 100644 --- a/runtime/include/boot_state.h +++ b/runtime/include/boot_state.h @@ -42,7 +42,9 @@ extern "C" { * v5 = v4 + CD-ROM Sub-Q replacement state; * v6 = v5 + per-word GPU DMA2 linked-list progress; * v7 = v6 + pending XA DATA_END IRQ state. */ -#define BOOT_STATE_VERSION 7u +#define BOOT_STATE_VERSION 8u +/* v8 adds allocated enhancement memory. Vanilla writers remain v7; older + * runtimes must not accept enhanced snapshots while ignoring their arenas. */ /* v7 intentionally breaks older savestates after the CD-ROM wire grew. Reject * them at the header before any section changes the live machine. */ #define BOOT_STATE_VERSION_MIN_READ 7u @@ -65,7 +67,7 @@ typedef struct { uint32_t codegen_ver; /* PSX_OVERLAY_CODEGEN_VER */ /* ---- layout ---- */ uint32_t section_count; /* number of sections that follow */ - uint32_t reserved; /* 0 */ + uint32_t reserved; /* v8: enhancement-memory layout cookie; vanilla 0 */ } BootStateHeader; #define BOOT_STATE_HEADER_WIRE_BYTES 36u @@ -82,6 +84,7 @@ typedef struct { * never allowed) -> normal boot + recapture. */ enum { + BS_SEC_MODMEM = 0x11, /* allocated opt-in CPU/GPU enhancement arenas */ BS_SEC_CPU = 0x01, /* CPUState: gpr/pc/hi/lo/cop0/gte_data/gte_ctrl */ BS_SEC_RAM = 0x02, /* 2 MB main RAM */ BS_SEC_SPAD = 0x03, /* 1 KB scratchpad */ diff --git a/runtime/include/gpu_gl_renderer.h b/runtime/include/gpu_gl_renderer.h index a0007bd1..e9ad1263 100644 --- a/runtime/include/gpu_gl_renderer.h +++ b/runtime/include/gpu_gl_renderer.h @@ -18,6 +18,10 @@ extern "C" { /* Create the GL context on a window made with SDL_WINDOW_OPENGL. * Returns 1 on success, 0 to fall back to the SDL_Renderer present path. */ int gl_renderer_init_context(struct SDL_Window *win); +/* Select a retained immutable bank for the next textured submission; zero + * selects live VRAM. Emulation/GL owning thread only. Returns 0 if unavailable. */ +int gl_renderer_select_texture_bank(uint16_t id); +int gl_renderer_texture_banks_supported(void); /* Set the GL swap interval / vsync mode (1=vsync, 0=immediate, -1=adaptive). * Safe before or after context creation; applies live when a context exists. */ diff --git a/runtime/include/mod_memory.h b/runtime/include/mod_memory.h index 5f3e0fd0..3eb54b66 100644 --- a/runtime/include/mod_memory.h +++ b/runtime/include/mod_memory.h @@ -9,18 +9,18 @@ extern "C" { /* * GPU linked-list tags carry only 24 address bits. Trusted enhancements that - * need a larger primitive arena use this otherwise-unmapped 4 MiB aperture so + * need a larger primitive arena use this otherwise-unmapped 8 MiB aperture so * their CPU pointers survive the tag truncation. Nothing is mapped until an * enhancement explicitly allocates from it. */ -#define PSX_MOD_GPU_DMA_APERTURE_BASE 0x00C00000u -#define PSX_MOD_GPU_DMA_APERTURE_SIZE (4u * 1024u * 1024u) -#define PSX_MOD_GPU_DMA_GUEST_BASE 0x80C00000u +#define PSX_MOD_GPU_DMA_APERTURE_BASE 0x00800000u +#define PSX_MOD_GPU_DMA_APERTURE_SIZE (8u * 1024u * 1024u) +#define PSX_MOD_GPU_DMA_GUEST_BASE 0x80800000u static inline int psx_mod_gpu_dma_aperture_offset_for( uint32_t address, uint32_t width, uint32_t used, uint32_t *offset) { /* CPU addresses retain all physical bits after KSEG normalization. The - * BIOS at 0x1FC00000 must never alias the aperture at 0x00C00000. + * BIOS at 0x1FC00000 must never alias the enhancement aperture. * DMA callers pass an already-truncated 24-bit tag separately below. */ uint32_t canonical = address & 0x1FFFFFFFu; uint32_t off; @@ -48,6 +48,13 @@ static inline uint32_t psx_mod_gpu_dma_resolve_address_for( uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t size, uint32_t alignment); uint32_t psx_mod_gpu_dma_resolve_address(uint32_t address); +uint32_t psx_mod_memory_alloc(uint32_t size, uint32_t alignment); +/* Allocations are fixed during activation. Snapshot allocated bytes, never + * host pointers; immutable host assets are reconstructed separately. */ +uint32_t psx_mod_memory_snapshot_bytes(void); +void psx_mod_memory_snapshot_write(uint8_t* out); +int psx_mod_memory_snapshot_read(const uint8_t* data, uint32_t size); +uint32_t psx_mod_memory_layout_cookie(void); #ifdef __cplusplus } diff --git a/runtime/include/mod_plugins.h b/runtime/include/mod_plugins.h index a1e4645e..bacae0c2 100644 --- a/runtime/include/mod_plugins.h +++ b/runtime/include/mod_plugins.h @@ -28,6 +28,30 @@ void psx_mod_function_entry(struct CPUState* cpu, uint32_t address); /* Narrow guest services available to trusted plugin callbacks. */ int psx_mod_game_started(void); +/* Read an original mounted-disc file without changing guest CD state/timing. + * Emulation-thread callbacks only. NULL buffer + zero capacity queries size; + * otherwise capacity must hold the entire file. Active sector mods apply. */ +int psx_mod_read_disc_file(const char* path, void* buffer, uint32_t capacity, + uint32_t* size); +/* Experimental retained-texture service (currently OpenGL only). IDs are + * nonzero, stable game-owned identities, NOT GL names. Banks are immutable + * 16-bit PS1 texels/indices with a caller-selected row pitch (width). + * A missing bank may be reconstructed from original assets by the resolver, + * including when a restored DMA queue refers to a previously unseen level. */ +typedef int (*PSXModTextureBankResolver)(uint16_t id); +int psx_mod_texture_banks_supported(void); +int psx_mod_define_texture_bank(uint16_t id, uint32_t width, uint32_t height, + const uint16_t* pixels); +void psx_mod_set_texture_bank_resolver(PSXModTextureBankResolver resolver); +/* A dedicated GPU-DMA packet arena. Only GT3 commands sourced from this + * allocation interpret C1/C2's otherwise-unused high bytes as a bank ID: + * id = (C1 >> 24) | ((C2 >> 24) << 8). ID zero uses ordinary VRAM. The rest + * of the packet is standard GP0, retaining OT order, palettes and STP blend. + * Optional 40-byte suffix after the 40-byte tagged GT3: u32 magic 0x48545031, + * three IEEE float 1/z weights, six IEEE float x/y coordinates. This enables + * precise perspective rendering without transient host-pointer side tables. + * Allocate during activation; do not mix stock game packets into this arena. */ +uint32_t psx_mod_alloc_texture_packet_memory(uint32_t size, uint32_t alignment); uint8_t psx_mod_read_byte(uint32_t address); void psx_mod_write_byte(uint32_t address, uint8_t value); uint16_t psx_mod_read_half(uint32_t address); diff --git a/runtime/include/mod_texture_banks.h b/runtime/include/mod_texture_banks.h new file mode 100644 index 00000000..2e79ac25 --- /dev/null +++ b/runtime/include/mod_texture_banks.h @@ -0,0 +1,11 @@ +#pragma once +#include <stdint.h> +#ifdef __cplusplus +extern "C" { +#endif +const uint16_t* mod_texture_bank_pixels(uint16_t id, uint32_t* width, uint32_t* height); +uint16_t mod_texture_packet_bank(uint32_t source, const uint32_t* words, uint32_t count); +int mod_texture_packet_precision(uint32_t source, float q[3], float xy[6]); +#ifdef __cplusplus +} +#endif diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index db6d8c48..5b92e4e4 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -416,6 +416,7 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/mod_builtin_bezel.c ${PSXRECOMP_ROOT}/runtime/src/mod_packages.cpp ${PSXRECOMP_ROOT}/runtime/src/mod_runtime.cpp + ${PSXRECOMP_ROOT}/runtime/src/mod_texture_banks.c ${PSXRECOMP_ROOT}/runtime/src/psx_keybinds.c ${PSXRECOMP_ROOT}/runtime/src/psx_bios_backend.c ${PSXRECOMP_ROOT}/runtime/src/psx_netplay.c diff --git a/runtime/src/boot_state.c b/runtime/src/boot_state.c index c68e14ad..0a644f4d 100644 --- a/runtime/src/boot_state.c +++ b/runtime/src/boot_state.c @@ -1,4 +1,5 @@ #include "boot_state.h" +#include "mod_memory.h" #include "overlay_api.h" /* PSX_OVERLAY_CODEGEN_HASH / _ABI_TAG / _CODEGEN_VER */ #include "dirty_ram_interp.h" #include "gpu.h" /* gpu_get_vram — CPU-auth mirror under dual-raster */ @@ -364,19 +365,23 @@ static int boot_state_save_to(BsOut* o, const CPUState* cpu, int ok; memset(&h, 0, sizeof h); h.magic = BOOT_STATE_MAGIC; - h.version = BOOT_STATE_VERSION; + h.version = psx_mod_memory_snapshot_bytes() ? BOOT_STATE_VERSION : 7u; + h.reserved = psx_mod_memory_layout_cookie(); h.bios_checksum = bios_checksum; h.entry_pc = entry_pc; h.codegen_hash = (uint32_t)PSX_OVERLAY_CODEGEN_HASH; h.abi_tag = (int32_t)PSX_OVERLAY_ABI_TAG; h.codegen_ver = (uint32_t)PSX_OVERLAY_CODEGEN_VER; - h.section_count = 16; + h.section_count = 16 + (psx_mod_memory_snapshot_bytes() ? 1u : 0u); ok = write_header_le(o, &h); if (ok) ok = write_cpu_section(o, cpu); if (ok) ok = write_section(o, BS_SEC_RAM, memory_get_ram_ptr(), RAM_SIZE); if (ok) ok = write_section(o, BS_SEC_SPAD, memory_get_scratchpad_ptr(), SPAD_SIZE); + if (ok && psx_mod_memory_snapshot_bytes()) + ok = write_module_section(o, BS_SEC_MODMEM, psx_mod_memory_snapshot_bytes, + psx_mod_memory_snapshot_write); if (ok) { /* 12B: i_stat, i_mask, cycles_since_vblank. Zeroing csv on warm load * rebased every tip to phase 0 and forked MotK wait-loop resim @@ -668,6 +673,8 @@ static int apply_section(uint32_t tag, const uint8_t* p, uint32_t len, free(words); return 1; } + case BS_SEC_MODMEM: + return psx_mod_memory_snapshot_read(p, len); case BS_SEC_ICACHE: { PstR r; if (len != 1024u * 4u) return 0; @@ -752,6 +759,10 @@ int boot_state_check_buffer(const uint8_t* file, size_t file_len, (unsigned)h.magic, (unsigned)BOOT_STATE_MAGIC); boot_state_append_reason(reason, reason_cap, part); } + if (h.reserved != psx_mod_memory_layout_cookie()) { + boot_state_append_reason(reason, reason_cap, "enhancement_memory_layout"); + return 0; + } if (h.version < BOOT_STATE_VERSION_MIN_READ || h.version > BOOT_STATE_VERSION) { snprintf(part, sizeof(part), "version=%u(want %u..%u)", @@ -802,7 +813,8 @@ int boot_state_load_buffer(const uint8_t* file, size_t file_len, (1u<<BS_SEC_CPU)|(1u<<BS_SEC_RAM)|(1u<<BS_SEC_SPAD)|(1u<<BS_SEC_IRQ)| (1u<<BS_SEC_TIMER)|(1u<<BS_SEC_CLOCK)|(1u<<BS_SEC_GPU)|(1u<<BS_SEC_VRAM)| (1u<<BS_SEC_SPU)|(1u<<BS_SEC_SPURAM)|(1u<<BS_SEC_CDROM)|(1u<<BS_SEC_DMA)| - (1u<<BS_SEC_SIO)|(1u<<BS_SEC_MDEC)|(1u<<BS_SEC_DIRTY); + (1u<<BS_SEC_SIO)|(1u<<BS_SEC_MDEC)|(1u<<BS_SEC_DIRTY)| + (psx_mod_memory_snapshot_bytes() ? (1u<<BS_SEC_MODMEM) : 0u); uint32_t seen = 0; int ok = 1; const double t0 = boot_state_mono_ms(); diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index cb9dda37..fb095afe 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -7892,6 +7892,16 @@ static void handle_ws_aspect(int id, const char *json) send_fmt("{\"id\":%d,\"ok\":true,\"num\":%d,\"den\":%d}", id, num, den); } +extern int psx_debug_display_aspect(int num, int den, int adaptive); +static void handle_display_aspect(int id, const char *json) { + int num=json_get_int(json,"num",-1), den=json_get_int(json,"den",-1); + int adaptive=json_get_int(json,"adaptive",0); + if (!psx_debug_display_aspect(num,den,adaptive)) { + send_err(id,"invalid display aspect (4:3 through 32:9)");return; + } + send_fmt("{\"id\":%d,\"ok\":true,\"num\":%d,\"den\":%d,\"adaptive\":%d}",id,num,den,adaptive!=0); +} + /* Live native-wide vs squash toggle (A/B): ws_nw on=<0|1> re-engages the wide * path in the chosen mode without a relaunch. 2 = native-wide, 1 = squash. */ extern void psx_ws_set_native_wide(int on); @@ -13609,6 +13619,7 @@ static const CmdEntry s_commands[] = { { "ws_hud_mode", handle_ws_hud_mode }, { "kernel_bless", handle_kernel_bless }, { "ws_aspect", handle_ws_aspect }, + { "display_aspect", handle_display_aspect }, { "ws_nw", handle_ws_nw }, { "scanline", handle_scanline }, { "ws_backdrop_ring", handle_ws_backdrop_ring }, diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index ea3c79d6..1d671cd0 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -11,6 +11,8 @@ */ #include "gpu.h" +#include "gpu_gl_renderer.h" +#include "mod_texture_banks.h" #include "display_scanout.h" #include "pgxp.h" #include "mod_memory.h" @@ -4122,14 +4124,36 @@ static void gp0_exec_shaded_textured_tri(void) { } if (draw_area_out_bbox(vx, vy, 3)) return; + uint16_t host_bank = mod_texture_packet_bank(gp0_cmd_source_addr, gp0_cmd_buf, 9u); + if (host_bank && (gr_backend() != GR_BACKEND_OPENGL || + !gl_renderer_select_texture_bank(host_bank))) return; + gr_set_semi_transparency(semi_trans, (int)semi_transparency); prepare_precise_triangle(1, 4, 7, vx, vy); prepare_texture_triangle(1, 4, 7); + if (host_bank) { + float q[3], xy[6]; + if (mod_texture_packet_precision(gp0_cmd_source_addr, q, xy)) { + gr_set_precise_triangle(1, + (int32_t)((xy[0] + draw_offset_x) * 65536.0f), + (int32_t)((xy[1] + draw_offset_y) * 65536.0f), + (int32_t)((xy[2] + draw_offset_x) * 65536.0f), + (int32_t)((xy[3] + draw_offset_y) * 65536.0f), + (int32_t)((xy[4] + draw_offset_x) * 65536.0f), + (int32_t)((xy[5] + draw_offset_y) * 65536.0f)); + gr_set_perspective_triangle(1, q[0], q[1], q[2]); + } + } gr_draw_shaded_textured_triangle(vx[0], vy[0], u[0], v[0], c[0], vx[1], vy[1], u[1], v[1], c[1], vx[2], vy[2], u[2], v[2], c[2], clut_x, clut_y, tpage, raw_texture); + if (host_bank) (void)gl_renderer_select_texture_bank(0); +} + +int psx_mod_texture_banks_supported(void) { + return gr_backend() == GR_BACKEND_OPENGL && gl_renderer_texture_banks_supported(); } /* Execute shaded textured quad (GP0 0x3C-0x3F) */ diff --git a/runtime/src/gpu_gl_renderer.c b/runtime/src/gpu_gl_renderer.c index e2befd6f..d78b44f4 100644 --- a/runtime/src/gpu_gl_renderer.c +++ b/runtime/src/gpu_gl_renderer.c @@ -67,6 +67,7 @@ #include "gpu_render.h" #include "gpu_sw_renderer.h" #include "gpu_gl_renderer.h" +#include "mod_texture_banks.h" #include "frame_interpolation.h" #include "host_osd.h" #include "psx_savestate_menu.h" @@ -359,6 +360,8 @@ static void interp_draw_quad(float alpha, int lx, int ly, int lw, int lh); static void present_bezel(int ww, int wh, int lx, int ly, int lw, int lh); static int s_raster_ok = 0; /* full GPU pipeline available */ +static GLuint s_bank_tex[65536]; +static GLuint s_selected_bank_tex; /* Authoritative VRAM: hr color texture + stencil (mask bit) FBO. */ static GLuint s_hr_tex = 0, s_hr_fbo = 0, s_hr_rb = 0; @@ -1107,7 +1110,9 @@ static const char *TEX_FS = "uniform int u_filter; /* 1 = bilinear */\n" "uniform float u_shift;\n" "int vram_at(int x, int y){\n" - " return int(texelFetch(u_vram, ivec2(x & 1023, y & 511), 0).r);\n" + " ivec2 p = ivec2(x & 1023, y & 511);\n" + " if (any(greaterThanEqual(p, textureSize(u_vram, 0)))) return 0;\n" + " return int(texelFetch(u_vram, p, 0).r);\n" "}\n" "int fetch_texel(int u, int v){\n" " u &= 255; v &= 255;\n" @@ -1746,6 +1751,7 @@ static float s_tb[TEXBATCH_MAXV * TEXV]; static int s_tb_n = 0; /* verts queued */ static int s_tb_semi = -2; static int s_tb_mask = 0, s_tb_filter = 0; +static GLuint s_tb_bank_tex; static int s_tb_twin[4] = {0, 0, 0, 0}; static uint64_t s_batch_total = 0, s_batch_reason[7]; @@ -1857,7 +1863,7 @@ static void flush_tex_batch(void) { hr_begin(1); p_glUseProgram(s_tex_prog); p_glActiveTexture(PSXGL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, s_raw_tex); + glBindTexture(GL_TEXTURE_2D, s_tb_bank_tex ? s_tb_bank_tex : s_raw_tex); p_glUniform1i(s_uVram, 0); p_glUniform4i(s_uTwin, s_tb_twin[0], s_tb_twin[1], s_tb_twin[2], s_tb_twin[3]); p_glUniform1i(s_uMaskset, s_tb_mask); @@ -2053,7 +2059,8 @@ static void gpu_textured_triangle(const int *xs, const int *ys, int depth = (texpage >> 7) & 3; if (depth > 2) depth = 2; flush_cpu_upload(); /* if a CPU->VRAM upload is pending it flushes the batch first */ - flush_pack_if_sampling(base_x, base_y, depth, clut_x, clut_y); /* flushes batch iff it must pack */ + if (!s_selected_bank_tex) + flush_pack_if_sampling(base_x, base_y, depth, clut_x, clut_y); mark_prim_dirty(xs, ys, 3, 1 /* textured */); /* Append to the textured batch. Flush first if this prim's blend/mask/twin/ @@ -2089,7 +2096,8 @@ static void gpu_textured_triangle(const int *xs, const int *ys, int isolate = (semi >= 0); int reason = -1; if (s_tb_n > 0) { - if (isolate) reason = 0; + if (s_tb_bank_tex != s_selected_bank_tex) reason = 0; + else if (isolate) reason = 0; else if (batch_semi != s_tb_semi) reason = 1; else if (s_mask_set != s_tb_mask) reason = 2; else if (s_tex_filter != s_tb_filter) reason = 3; @@ -2104,6 +2112,7 @@ static void gpu_textured_triangle(const int *xs, const int *ys, if (s_tb_n + 3 > TEXBATCH_MAXV) { s_batch_reason[6]++; flush_tex_batch(); } if (s_tb_n == 0) { /* opening a batch: capture its keyed state */ s_tb_semi = batch_semi; s_tb_mask = s_mask_set; s_tb_filter = s_tex_filter; s_tb_gate = gate; + s_tb_bank_tex = s_selected_bank_tex; s_tb_twin[0] = twx; s_tb_twin[1] = twy; s_tb_twin[2] = tox; s_tb_twin[3] = toy; } float *vp = &s_tb[s_tb_n * TEXV]; @@ -2984,6 +2993,41 @@ static int init_gpu_raster(void) { return 1; } +int gl_renderer_texture_banks_supported(void) { return s_raster_ok && !s_cpu_auth_dual; } + +int gl_renderer_select_texture_bank(uint16_t id) { + uint32_t width, height; + const uint16_t* pixels; + GLint alignment, row_length; + if (!id) { s_selected_bank_tex = 0; return 1; } + if (!gl_renderer_texture_banks_supported()) return 0; + if (!s_bank_tex[id]) { + pixels = mod_texture_bank_pixels(id, &width, &height); + if (!pixels) return 0; + glGenTextures(1, &s_bank_tex[id]); + p_glActiveTexture(PSXGL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, s_bank_tex[id]); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glGetIntegerv(GL_UNPACK_ALIGNMENT, &alignment); + glGetIntegerv(PSXGL_UNPACK_ROW_LENGTH, &row_length); + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + glPixelStorei(PSXGL_UNPACK_ROW_LENGTH, 0); + glTexImage2D(GL_TEXTURE_2D, 0, PSXGL_R16UI, (GLsizei)width, + (GLsizei)height, 0, PSXGL_RED_INTEGER, GL_UNSIGNED_SHORT, pixels); + glPixelStorei(GL_UNPACK_ALIGNMENT, alignment); + glPixelStorei(PSXGL_UNPACK_ROW_LENGTH, row_length); + if (glGetError() != GL_NO_ERROR) { + glDeleteTextures(1, &s_bank_tex[id]); s_bank_tex[id] = 0; + return 0; + } + } + s_selected_bank_tex = s_bank_tex[id]; + return 1; +} + int gl_renderer_init_context(SDL_Window *win) { s_win = win; s_present_w = 0; @@ -3084,6 +3128,12 @@ void gl_renderer_set_swap_interval(int interval) { } void gl_renderer_shutdown(void) { + if (s_ctx) { + for (unsigned i = 1; i < 65536u; ++i) + if (s_bank_tex[i]) glDeleteTextures(1, &s_bank_tex[i]); + } + memset(s_bank_tex, 0, sizeof s_bank_tex); + s_selected_bank_tex = s_tb_bank_tex = 0; if (s_ctx) { ensure_cpu(); SDL_GL_DeleteContext(s_ctx); s_ctx = NULL; diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index e19f8d7d..4dd39e78 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -1774,6 +1774,22 @@ extern "C" void psx_ws_set_native_wide(int on) { } extern "C" int psx_ws_get_native_wide(void) { return g_ws_native_wide; } +/* TCP diagnostics: change the rendered view without moving, resizing, raising + * or focusing the user's window. Transient; never writes settings.toml. */ +extern "C" int psx_debug_display_aspect(int num, int den, int adaptive) { + if (adaptive) return psx_mod_set_adaptive_display_aspect(num, den); + if (!psx_mod_set_fixed_display_aspect(num, den)) return 0; + gl_renderer_set_display_aspect(num, den); + vk_renderer_set_display_aspect(num, den); + if (sdl_renderer) { + g_logical_w = 480 * num * g_video_scale / den; + SDL_RenderSetLogicalSize(sdl_renderer, g_logical_w, 480 * g_video_scale); + } + g_ws_projection_mode = -1; + refresh_widescreen_projection(); + return 1; +} + static bool g_gl_active = false; /* GL context live -> GL present path */ static bool g_vk_active = false; /* Vulkan context live -> VK present path */ diff --git a/runtime/src/memory.c b/runtime/src/memory.c index f4b3cb78..fc199c93 100644 --- a/runtime/src/memory.c +++ b/runtime/src/memory.c @@ -16,6 +16,7 @@ #include "gpu.h" #include "mdec.h" #include "mod_memory.h" +#include "pst_wire.h" #include "sio.h" #include "spu.h" #include "timers.h" @@ -43,6 +44,45 @@ static uint32_t mod_memory_used; static uint8_t mod_gpu_dma_memory[PSX_MOD_GPU_DMA_APERTURE_SIZE]; static uint32_t mod_gpu_dma_memory_used; +uint32_t psx_mod_memory_snapshot_bytes(void) { + return mod_memory_used || mod_gpu_dma_memory_used + ? 16u + mod_memory_used + mod_gpu_dma_memory_used : 0u; +} + +uint32_t psx_mod_memory_layout_cookie(void) { + /* Compatibility guard, not a cryptographic identity. Include aperture + * revision; old enabled-mod saves must be rejected before applying RAM. */ + if (!psx_mod_memory_snapshot_bytes()) return 0u; + return 0x4D4F4402u ^ (mod_memory_used * 16777619u) ^ mod_gpu_dma_memory_used; +} + +void psx_mod_memory_snapshot_write(uint8_t* out) { + PstW w; + if (!out || !psx_mod_memory_snapshot_bytes()) return; + pst_w_init(&w, out, psx_mod_memory_snapshot_bytes()); + pst_w_u32(&w, 1u); + pst_w_u32(&w, PSX_MOD_GPU_DMA_APERTURE_BASE); + pst_w_u32(&w, mod_memory_used); + pst_w_u32(&w, mod_gpu_dma_memory_used); + memcpy(out + 16u, mod_memory, mod_memory_used); + memcpy(out + 16u + mod_memory_used, mod_gpu_dma_memory, mod_gpu_dma_memory_used); +} + +int psx_mod_memory_snapshot_read(const uint8_t* data, uint32_t size) { + PstR r; + uint32_t version, base, cpu_bytes, dma_bytes; + if (!data || size < 16u || size != psx_mod_memory_snapshot_bytes()) return 0; + pst_r_init(&r, data, size); + if (!pst_r_u32(&r, &version) || !pst_r_u32(&r, &base) || + !pst_r_u32(&r, &cpu_bytes) || !pst_r_u32(&r, &dma_bytes) || + version != 1u || base != PSX_MOD_GPU_DMA_APERTURE_BASE || + cpu_bytes != mod_memory_used || dma_bytes != mod_gpu_dma_memory_used) + return 0; + memcpy(mod_memory, data + 16u, cpu_bytes); + memcpy(mod_gpu_dma_memory, data + 16u + cpu_bytes, dma_bytes); + return 1; +} + /* * Trusted mods may opt into host-backed guest memory in Expansion 1. Before * the first allocation this entire region retains hardware open-bus behavior. diff --git a/runtime/src/mod_runtime.cpp b/runtime/src/mod_runtime.cpp index 3f8b5765..af31e895 100644 --- a/runtime/src/mod_runtime.cpp +++ b/runtime/src/mod_runtime.cpp @@ -1294,6 +1294,43 @@ extern "C" void mod_runtime_enable_disc_patches(void) { PSXRecompV4::state().disc_enabled = true; } +extern "C" int psx_mod_read_disc_file(const char* path, void* buffer, + uint32_t capacity, uint32_t* size) { + using namespace PSXRecompV4; + if (size) *size = 0; + if (!path || !*path || !size || (!buffer && capacity)) return 0; + try { + const auto& s = state(); + const auto& mount = s.effective_disc_path.empty() ? s.disc_path : s.effective_disc_path; + if (mount.empty()) return 0; + PS1::ISOReader reader; + PS1::ISOFileEntry entry; + if (!reader.Open(mount.string()) || !reader.FindFile(path, entry) || + entry.is_directory || !entry.size || entry.size > 64u * 1024u * 1024u) + return 0; + if (!buffer) { *size = entry.size; return 1; } + if (capacity < entry.size) return 0; + uint8_t sector[2048], raw[2352]; + for (uint32_t offset = 0; offset < entry.size; offset += 2048u) { + const uint32_t lba = entry.lba + offset / 2048u; + if (reader.ReadRawSector(lba, raw)) { + if (raw[15] != 1 && (raw[15] != 2 || (raw[18] & 0x20u))) return 0; + mod_runtime_patch_disc_sector(lba, 1, raw, sizeof raw); + std::memcpy(sector, raw + (raw[15] == 1 ? 16 : 24), sizeof sector); + if (raw[15] == 1) mod_runtime_patch_disc_sector(lba, 0, sector, sizeof sector); + } else { + if (!reader.ReadSector(lba, sector)) return 0; + mod_runtime_patch_disc_sector(lba, 0, sector, sizeof sector); + } + if (state().disc_guard_failed) return 0; + const uint32_t count = std::min(2048u, entry.size - offset); + std::memcpy(static_cast<uint8_t*>(buffer) + offset, sector, count); + } + *size = entry.size; + return 1; + } catch (...) { return 0; } +} + extern "C" void mod_runtime_activate_plugins(void) { using namespace PSXRecompV4; RuntimeMods& s = state(); diff --git a/runtime/src/mod_texture_banks.c b/runtime/src/mod_texture_banks.c new file mode 100644 index 00000000..5eba304d --- /dev/null +++ b/runtime/src/mod_texture_banks.c @@ -0,0 +1,94 @@ +#include "mod_plugins.h" +#include "mod_memory.h" +#include "mod_texture_banks.h" +#include <stdlib.h> +#include <string.h> +#include <math.h> + +typedef struct { uint16_t* pixels; uint32_t width, height; } Bank; +typedef struct { uint32_t begin, size; } PacketArena; +static Bank banks[65536]; +static PacketArena arenas[16]; +static unsigned arena_count; +static uint32_t bank_bytes; +static PSXModTextureBankResolver resolve_bank; +static int resolving; + +void psx_mod_set_texture_bank_resolver(PSXModTextureBankResolver resolver) { + resolve_bank = resolver; +} + +int psx_mod_define_texture_bank(uint16_t id, uint32_t w, uint32_t h, + const uint16_t* pixels) { + uint32_t bytes; + Bank* bank = &banks[id]; + if (!id || !pixels || !w || !h || w > 1024u || h > 512u) return 0; + bytes = w * h * 2u; + if (bank->pixels) + return bank->width == w && bank->height == h && + memcmp(bank->pixels, pixels, bytes) == 0; + if (bytes > 256u * 1024u * 1024u - bank_bytes) return 0; + bank->pixels = (uint16_t*)malloc(bytes); + if (!bank->pixels) return 0; + memcpy(bank->pixels, pixels, bytes); + bank->width = w; bank->height = h; + bank_bytes += bytes; + return 1; +} + +const uint16_t* mod_texture_bank_pixels(uint16_t id, uint32_t* w, uint32_t* h) { + if (!id || !w || !h) return NULL; + if (!banks[id].pixels && resolve_bank && !resolving) { + resolving = 1; + (void)resolve_bank(id); + resolving = 0; + } + *w = banks[id].width; *h = banks[id].height; + return banks[id].pixels; +} + +uint32_t psx_mod_alloc_texture_packet_memory(uint32_t size, uint32_t alignment) { + uint32_t addr; + if (arena_count == 16u || size < 40u) return 0; + addr = psx_mod_gpu_dma_memory_alloc(size, alignment); + if (!addr) return 0; + arenas[arena_count].begin = addr & 0x1FFFFFFFu; + arenas[arena_count++].size = size; + return addr; +} + +uint16_t mod_texture_packet_bank(uint32_t source, const uint32_t* words, uint32_t count) { + unsigned i; + uint32_t address = source & 0x1FFFFFFFu; + if (!words || count != 9u || (words[0] >> 26) != 0xDu) return 0; + for (i = 0; i < arena_count; ++i) { + if (address >= arenas[i].begin && address - arenas[i].begin <= arenas[i].size - 36u) + return (uint16_t)((words[3] >> 24) | ((words[6] >> 24) << 8)); + } + return 0; +} + +int mod_texture_packet_precision(uint32_t source, float q[3], float xy[6]) { + unsigned i, j; + uint32_t address = source & 0x1FFFFFFFu; + for (i = 0; i < arena_count; ++i) { + if (arenas[i].size < 80u || address < arenas[i].begin || + address - arenas[i].begin > arenas[i].size - 76u) continue; + if (psx_mod_read_word(source + 36u) != 0x48545031u) return 0; + for (j = 0; j < 9u; ++j) { + uint32_t bits = psx_mod_read_word(source + 40u + j * 4u); + float f; + memcpy(&f, &bits, sizeof f); + if (!isfinite(f)) return 0; + if (j < 3u) { + if (f <= 0.0f || f > 1.0f) return 0; + q[j] = f; + } else { + if (f < -2048.0f || f > 2048.0f) return 0; + xy[j - 3u] = f; + } + } + return 1; + } + return 0; +} diff --git a/runtime/tests/run_gl_readback_region.py b/runtime/tests/run_gl_readback_region.py index 5dccb731..5f4e2a4b 100644 --- a/runtime/tests/run_gl_readback_region.py +++ b/runtime/tests/run_gl_readback_region.py @@ -26,6 +26,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--compiler-bin", required=True) parser.add_argument("--sdl-root", required=True) + parser.add_argument("--sdl-library", help="Explicit SDL3 library (e.g. a system DLL import library)") parser.add_argument("--output", required=True) parser.add_argument("--gl-source") parser.add_argument("--fixture", type=pathlib.Path) @@ -39,8 +40,9 @@ def main(): sdl = pathlib.Path(args.sdl_root).resolve() if not (sdl / "sdl3-src/include").is_dir(): parser.error(f"--sdl-root must contain the sdl3-src/include directory: {sdl}") - if not (sdl / "sdl3-build/libSDL3.a").is_file(): - parser.error(f"--sdl-root must contain sdl3-build/libSDL3.a: {sdl}") + sdl_library = pathlib.Path(args.sdl_library).resolve() if args.sdl_library else sdl / "sdl3-build/libSDL3.a" + if not sdl_library.is_file(): + parser.error(f"SDL3 library does not exist: {sdl_library}") output = pathlib.Path(args.output).resolve() output.mkdir(parents=True, exist_ok=True) dest = pathlib.Path(tempfile.mkdtemp(prefix="readback-", dir=output)) @@ -73,7 +75,7 @@ def main(): "oleaut32", "version", "uuid", "advapi32", "setupapi", "shell32", "dinput8", "opengl32"] if run([compiler / "g++.exe", "-O2", "-flto", dest / "probe.o", dest / "sw.o", - sdl / "sdl3-build/libSDL3.a", *["-l" + name for name in libraries], + sdl_library, *["-l" + name for name in libraries], "-o", dest / "probe.exe"]).returncode: return 2 for scale in (1, 4): diff --git a/runtime/tests/test_gl_readback_region.c b/runtime/tests/test_gl_readback_region.c index 355eebab..bdc71b5d 100644 --- a/runtime/tests/test_gl_readback_region.c +++ b/runtime/tests/test_gl_readback_region.c @@ -1,5 +1,8 @@ /* Original source-owned GL readback-coherence regression. No retail payload. */ #include "gpu_gl_renderer.c" +#include "mod_texture_banks.c" +uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t n,uint32_t a){(void)n;(void)a;return 0;} +uint32_t psx_mod_read_word(uint32_t a){(void)a;return 0;} static uint16_t image[1024*512], oracle[1024*512]; int g_psx_vram_dirty_tracking=0; uint64_t s_frame_count=0; @@ -73,6 +76,24 @@ int main(int argc,char **argv){ check(glb_vram_read(40,40)==0,"depth24 cleared band immediate CPU read"); check(glb_vram_read(33,33)==0x3210,"newer overlapping texture survives clear"); verify("depth24 leave coherence without subsequent primitive"); + /* Retained banks use their own texels AND CLUT, and bank/VRAM transitions + * must split batches without changing painter order. No retail assets. */ + static uint16_t bank[256*128]; + bank[0]=0x001f; bank[1]=0x03e0; bank[2]=0x7c00; + bank[16]=0x1111; /* 4-bit indices, CLUT at (0,0) */ + check(psx_mod_define_texture_bank(7,256,128,bank),"define retained bank"); + check(gl_renderer_select_texture_bank(7),"select retained bank"); + glb_draw_shaded_textured_triangle(100,250,64,0,0x808080,132,250,64,0,0x808080,100,282,64,0,0x808080,0,0,0,1); + check(gl_renderer_select_texture_bank(0),"select original VRAM"); + glb_vram_write(512,0,0x7c00); + glb_draw_shaded_textured_triangle(116,250,0,0,0x808080,148,250,0,0,0x808080,116,282,0,0,0x808080,0,0,0x108,1); + check(glb_vram_read(102,252)==0x03e0,"retained 4-bit CLUT independent of guest VRAM"); + check(glb_vram_read(118,252)==0x7c00,"following stock texture wins overlap"); + check(gl_renderer_select_texture_bank(7),"reselect retained bank"); + glb_draw_shaded_textured_triangle(300,250,0,0,0x808080,332,250,0,0,0x808080,300,282,0,0,0x808080,0,0,0x100,1); + check(gl_renderer_select_texture_bank(0),"reset bank after direct texture"); + check(glb_vram_read(302,252)==0x001f,"retained 16-bit texel"); + verify("retained banks and original VRAM ordered together"); printf("checks=%d failures=%d\n",checks,failures); gl_renderer_shutdown();SDL_DestroyWindow(win);SDL_Quit();return failures?1:0; } diff --git a/runtime/tests/test_mod_gpu_dma_aperture.c b/runtime/tests/test_mod_gpu_dma_aperture.c index 2033c64e..7829075c 100644 --- a/runtime/tests/test_mod_gpu_dma_aperture.c +++ b/runtime/tests/test_mod_gpu_dma_aperture.c @@ -50,7 +50,7 @@ int main(void) { check(psx_mod_gpu_dma_aperture_offset_for(0xA0000000u + base, 4u, size, &off) && off == 0u, "KSEG1 CPU aperture alias resolves correctly"); check(psx_mod_gpu_dma_aperture_offset_for(base + size - 4u, 4u, size, &off) - && off == size - 4u, "last word of full 4 MiB aperture is accessible"); + && off == size - 4u, "last word of full enhancement aperture is accessible"); check(!psx_mod_gpu_dma_aperture_offset_for(base - 4u, 4u, size, &off), "word before aperture is rejected"); check(!psx_mod_gpu_dma_aperture_offset_for(base + size, 4u, size, &off), diff --git a/runtime/tests/test_mod_memory_snapshot.c b/runtime/tests/test_mod_memory_snapshot.c new file mode 100644 index 00000000..71344d5b --- /dev/null +++ b/runtime/tests/test_mod_memory_snapshot.c @@ -0,0 +1,29 @@ +/* Include implementation to inspect owned backing storage without pulling in + * hardware routing. Section GC discards unrelated memory access functions. */ +#include "memory.c" +static void check(int ok,const char* text){if(!ok){fprintf(stderr,"FAIL %s\n",text);exit(1);}} +int main(void){ + check(psx_mod_memory_snapshot_bytes()==0 && psx_mod_memory_layout_cookie()==0,"vanilla unchanged"); + psx_mod_memory_snapshot_write(NULL); + check(psx_mod_memory_alloc(13,16)==0x9f000000u,"CPU allocation"); + check(psx_mod_gpu_dma_memory_alloc(5u*1024u*1024u,16)==PSX_MOD_GPU_DMA_GUEST_BASE,"beyond old four MiB"); + check(psx_mod_gpu_dma_resolve_address(0xd00000u)==0x100000u,"unallocated tag retains retail folding"); + uint32_t size=psx_mod_memory_snapshot_bytes(),cookie=psx_mod_memory_layout_cookie(); + uint8_t* saved=(uint8_t*)malloc(size);check(saved!=NULL,"test buffer"); + memset(mod_memory,0x12,mod_memory_used);memset(mod_gpu_dma_memory,0x34,mod_gpu_dma_memory_used); + psx_mod_memory_snapshot_write(saved); + memset(mod_memory,0x56,mod_memory_used);memset(mod_gpu_dma_memory,0x78,mod_gpu_dma_memory_used); + check(!psx_mod_memory_snapshot_read(saved,size-1),"truncation rejected"); + saved[0]=99; + check(!psx_mod_memory_snapshot_read(saved,size) && mod_memory[0]==0x56,"bad version no mutation"); + saved[0]=1;saved[8]++; + check(!psx_mod_memory_snapshot_read(saved,size) && mod_gpu_dma_memory[0]==0x78,"bad layout no mutation"); + saved[8]--; + check(psx_mod_memory_snapshot_read(saved,size),"restore"); + for(uint32_t i=0;i<mod_memory_used;++i)check(mod_memory[i]==0x12,"CPU bytes restored"); + for(uint32_t i=0;i<mod_gpu_dma_memory_used;++i)check(mod_gpu_dma_memory[i]==0x34,"DMA bytes restored"); + check(psx_mod_memory_layout_cookie()==cookie,"stable layout identity"); + check(psx_mod_gpu_dma_memory_alloc(16,16)!=0 && psx_mod_memory_layout_cookie()!=cookie,"layout change detected"); + check(!psx_mod_memory_snapshot_read(saved,size),"old layout rejected"); + free(saved);puts("enhancement snapshot checks passed");return 0; +} diff --git a/runtime/tests/test_mod_runtime.cpp b/runtime/tests/test_mod_runtime.cpp index 10778201..9b66052f 100644 --- a/runtime/tests/test_mod_runtime.cpp +++ b/runtime/tests/test_mod_runtime.cpp @@ -6,6 +6,7 @@ #include "gpu.h" #include <array> +#include <algorithm> #include <cstdint> #include <filesystem> #include <fstream> @@ -491,6 +492,44 @@ int main() { "unestablished display geometry must report zero so callers skip " "drawing instead of guessing"); + /* Source-owned ISO: nested file spanning two sectors, without a derived + * disc. Host reads must choose the original mount and leave sizes honest. */ + std::vector<uint8_t> iso(24*2048); + auto le32 = [&](size_t at,uint32_t n) { for(unsigned i=0;i<4;++i) iso[at+i]=(uint8_t)(n>>(i*8)); }; + auto record = [&](size_t at,uint32_t lba,uint32_t bytes,bool directory,const std::string& name) { + iso[at]=(uint8_t)((33+name.size()+1)&~size_t(1)); + le32(at+2,lba); le32(at+10,bytes); iso[at+25]=directory?2:0; + iso[at+28]=1;iso[at+31]=1;iso[at+32]=(uint8_t)name.size(); + std::copy(name.begin(),name.end(),iso.begin()+at+33); + }; + iso[16*2048]=1;std::copy_n("CD001",5,iso.begin()+16*2048+1);iso[16*2048+6]=1; + record(16*2048+156,20,2048,true,std::string(1,'\0')); + record(20*2048,21,2048,true,"S0"); + record(21*2048,22,3000,false,"LEVEL.NSF;1"); + for(unsigned i=0;i<3000;++i)iso[22*2048+i]=(uint8_t)(i*7); + const auto disc_root=root/"host-reader"; + const auto iso_path=disc_root/"original.iso"; + write_bytes(iso_path,iso); + check(PSXRecompV4::mod_runtime_initialize(disc_root,"READER",0,{},&error),"reader initialize"); + check(PSXRecompV4::mod_runtime_commit(iso_path,&error),"reader mount original ISO"); + uint32_t bytes=0; + check(psx_mod_read_disc_file("S0/LEVEL.NSF",nullptr,0,&bytes) && bytes==3000,"query original nested file size"); + std::vector<uint8_t> result(3000); + check(!psx_mod_read_disc_file("S0/LEVEL.NSF",result.data(),2999,&bytes) && bytes==0,"undersized destination rejected"); + check(psx_mod_read_disc_file("S0/LEVEL.NSF",result.data(),(uint32_t)result.size(),&bytes) && + bytes==3000 && std::equal(result.begin(),result.end(),iso.begin()+22*2048),"complete original file bytes"); + check(!psx_mod_read_disc_file("S0/MISSING.NSF",nullptr,0,&bytes) && bytes==0,"missing file explicit"); + check(!psx_mod_read_disc_file("S0",nullptr,0,&bytes),"directory rejected"); + std::vector<uint8_t> raw_iso(24*2352); + for(unsigned i=0;i<24;++i) { + raw_iso[i*2352+15]=2; + std::copy_n(iso.begin()+i*2048,2048,raw_iso.begin()+i*2352+24); + } + const auto raw_path=disc_root/"original.bin"; + write_bytes(raw_path,raw_iso); + check(PSXRecompV4::mod_runtime_commit(raw_path,&error),"reader mount raw disc"); + check(psx_mod_read_disc_file("S0/LEVEL.NSF",result.data(),(uint32_t)result.size(),&bytes) && + std::equal(result.begin(),result.end(),iso.begin()+22*2048),"raw and ISO reads identical"); fs::remove_all(root, ec); if (failures) return 1; std::cout << "mod runtime tests passed\n"; diff --git a/runtime/tests/test_mod_texture_banks.c b/runtime/tests/test_mod_texture_banks.c new file mode 100644 index 00000000..9b39ed7c --- /dev/null +++ b/runtime/tests/test_mod_texture_banks.c @@ -0,0 +1,53 @@ +#include "mod_plugins.h" +#include "mod_memory.h" +#include "mod_texture_banks.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <math.h> +static uint32_t memory[256], used, reads, resolves; +uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t n,uint32_t a) { + (void)a; if(n>sizeof memory-used)return 0; + uint32_t address=PSX_MOD_GPU_DMA_GUEST_BASE+used;used+=n;return address; +} +uint32_t psx_mod_read_word(uint32_t address) { + uint32_t offset=(address&0x1fffffffu)-PSX_MOD_GPU_DMA_APERTURE_BASE; + if(offset>sizeof memory-4)abort(); ++reads;return memory[offset/4]; +} +static void check(int ok,const char* text){if(!ok){fprintf(stderr,"FAIL %s\n",text);exit(1);}} +static int resolve(uint16_t id){uint16_t data=0x4567;++resolves;return psx_mod_define_texture_bank(id,1,1,&data);} +static uint32_t bits(float f){uint32_t n;memcpy(&n,&f,4);return n;} +int main(void){ + uint16_t data[4]={1,2,3,4};uint32_t w,h; + check(!psx_mod_define_texture_bank(0,2,2,data),"bank zero reserved"); + check(!psx_mod_define_texture_bank(1,1025,2,data),"dimensions bounded"); + check(psx_mod_define_texture_bank(1,2,2,data),"define"); + check(psx_mod_define_texture_bank(1,2,2,data),"identical definition idempotent"); + data[0]=99; + check(!psx_mod_define_texture_bank(1,2,2,data),"immutable id cannot change"); + check(mod_texture_bank_pixels(1,&w,&h)[0]==1 && w==2 && h==2,"owned deep copy"); + check(!mod_texture_bank_pixels(2,&w,&h),"missing bank explicit"); + psx_mod_set_texture_bank_resolver(resolve); + check(mod_texture_bank_pixels(2,&w,&h)[0]==0x4567 && resolves==1,"lazy restored-bank resolver"); + check(mod_texture_bank_pixels(2,&w,&h)!=NULL && resolves==1,"resident bank no reload"); + const uint32_t arena=psx_mod_alloc_texture_packet_memory(80,16); + uint32_t words[9]={0x34000000,0,0,0x23000000,0,0,0x01000000,0,0}; + check(!mod_texture_packet_bank(0x80001004,words,9),"stock packet cannot opt in"); + check(mod_texture_packet_bank(arena+4,words,9)==0x123,"bank id only in trusted arena"); + check(!mod_texture_packet_bank(arena+48,words,9),"whole command must fit"); + check(!mod_texture_packet_bank(arena+4,words,8),"command size checked"); + words[0]=0x30000000; + check(!mod_texture_packet_bank(arena+4,words,9),"GT3 opcode checked"); + float q[3],xy[6]; + check(!mod_texture_packet_precision(0x80001004,q,xy) && reads==0,"stock precision does not read suffix"); + memory[10]=0x48545031; + for(unsigned i=0;i<3;++i)memory[11+i]=bits(.01f); + for(unsigned i=0;i<6;++i)memory[14+i]=bits(15.25f+i); + check(mod_texture_packet_precision(arena+4,q,xy) && q[1]==.01f && xy[5]==20.25f,"fractional projection and reciprocal depth"); + check(!mod_texture_packet_precision(arena+8,q,xy),"suffix bounds"); + memory[11]=bits(NAN); + check(!mod_texture_packet_precision(arena+4,q,xy),"NaN rejected"); + memory[11]=bits(0); + check(!mod_texture_packet_precision(arena+4,q,xy),"zero reciprocal depth rejected"); + puts("texture bank checks passed");return 0; +} From 693c7be0fcdbaa812517a906a2c25428c58b63eb Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Mon, 14 Sep 2026 03:01:55 +0300 Subject: [PATCH 28/56] Fix GPU upload history overflow after retention fills (#360) Co-authored-by: Pegasus <pegasus@psxrecomp.local> --- docs/GPU_UPLOAD_HISTORY.md | 21 +++++++++ recompiler/CMakeLists.txt | 7 +++ runtime/src/gpu.c | 4 +- runtime/tests/test_gpu_upload_history.py | 59 ++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 docs/GPU_UPLOAD_HISTORY.md create mode 100644 runtime/tests/test_gpu_upload_history.py diff --git a/docs/GPU_UPLOAD_HISTORY.md b/docs/GPU_UPLOAD_HISTORY.md new file mode 100644 index 00000000..5e5dee32 --- /dev/null +++ b/docs/GPU_UPLOAD_HISTORY.md @@ -0,0 +1,21 @@ +# GPU upload diagnostic retention + +GPU A0 upload diagnostics retain the first 128 uploads. Once the history is +full, subsequent uploads continue normally without changing retained entries. +Reset the active capture slot before every upload so a full history cannot +keep incrementing the last retained entry's signed word counter. + +The authored regression compiles the production capture code at O0 and O2, +checks retention at capacity, and exercises the largest legal upload. It +requires a GCC-compatible compiler and no BIOS or game assets: + +```sh +python runtime/tests/test_gpu_upload_history.py --cc gcc +``` + +The defect caused a long Biohazard TAS replay to crash after return 214410. +The campaign fix passed that boundary and executed all 239202 returns. That +run still first differed from its reference at return 233568 (one clock early, +nine terminal RAM bytes different). This diagnostic fix is not a claim of +Biohazard replay fidelity. It requires rebuilding the runtime, not regenerating +game code; game framework pins are not changed by this submission. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index d44933fe..4dd33e72 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -754,6 +754,13 @@ target_include_directories(reachable_discovery_test PRIVATE target_link_libraries(reachable_discovery_test PRIVATE fmt) # ---- test-registration completeness guard ----------------------------------- +if(BUILD_TESTING AND Python3_Interpreter_FOUND AND NOT MSVC) + add_test(NAME gpu_upload_history + COMMAND ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/../runtime/tests/test_gpu_upload_history.py" + --cc ${CMAKE_C_COMPILER}) +endif() + # Reads both CMakeLists.txt files as text, so it catches an orphan in either # project from whichever one is being configured. Placed before the collision # guard, which must stay last because it needs every target declared; this one diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index 5d880cdc..e7d20c41 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -4632,7 +4632,9 @@ static void gp0_exec_cpu_to_vram(void) { vram_write_w = (w == 0) ? 0x400 : (uint16_t)w; vram_write_h = (h == 0) ? 0x200 : (uint16_t)h; - /* Record for debug */ + /* A full history retains old uploads; later transfers must not append + * to the last slot and overflow its diagnostic word counter. */ + a0_capture_slot = -1; if (a0_history_count < A0_HISTORY_CAP) { int slot = a0_history_count++; a0_history[slot].x = vram_write_x; diff --git a/runtime/tests/test_gpu_upload_history.py b/runtime/tests/test_gpu_upload_history.py new file mode 100644 index 00000000..2e31767d --- /dev/null +++ b/runtime/tests/test_gpu_upload_history.py @@ -0,0 +1,59 @@ +"""Exercise production upload-history capture across its retention limit.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +p=argparse.ArgumentParser() +p.add_argument('--cc',default='gcc') +p.add_argument('--source',type=Path,default=Path(__file__).resolve().parents[1]/'src/gpu.c') +a=p.parse_args() +s=a.source.read_text() +start=s.index('/* A0 upload history for debug inspection */') +end=s.index('/* C0 (VRAM',start) +production=s[start:end] +start=s.index(' /* Capture first few data words for debug */') +end=s.index(' /* Each word contains two',start) +capture=s[start:end] +harness=r""" +#include <stdint.h> +#include <string.h> +#include <assert.h> +#include <stdio.h> +static uint32_t gp0_cmd_buf[16],g_debug_current_func_addr; +static uint16_t vram_write_x,vram_write_y,vram_write_w,vram_write_h,vram_write_col,vram_write_row; +static unsigned vram_write_remaining,s_frame_count; +static int gp0_state; +enum {GP0_VRAM_WRITE=2}; +static struct {uint32_t gpr[32];} *debug_cpu_ptr; +uint32_t psx_read_word(uint32_t addr){(void)addr;assert(0);return 0;} +"""+production+'\nstatic void capture(uint32_t val) {\n'+capture+'}\n'+r""" +int main(void) { + gp0_cmd_buf[1]=7|(9u<<16);gp0_cmd_buf[2]=4|(1u<<16); + for(int i=0;i<A0_HISTORY_CAP;i++) { + gp0_exec_cpu_to_vram();assert(a0_capture_slot==i); + capture(0x12345678);capture(0xabcdef01); + assert(a0_history[i].word_count==2); + } + A0HistEntry retained[A0_HISTORY_CAP];memcpy(retained,a0_history,sizeof(retained)); + for(int upload=0;upload<16;upload++) { + gp0_exec_cpu_to_vram();assert(a0_capture_slot==-1); + assert(vram_write_remaining==2 && gp0_state==GP0_VRAM_WRITE); + capture(0xdeadbeef);capture(0xdeadbeef); + } + assert(a0_history_count==A0_HISTORY_CAP && !memcmp(retained,a0_history,sizeof(retained))); + /* Largest legal upload is bounded to 262144 data words in a fresh slot. */ + a0_history_count=0;gp0_cmd_buf[2]=0;gp0_exec_cpu_to_vram(); + assert(vram_write_remaining==262144); + for(unsigned i=0;i<vram_write_remaining;i++)capture(i); + assert(a0_history[0].word_count==262144); + for(unsigned i=0;i<4;i++)assert(a0_history[0].first_words[i]==i); + puts("PASS retained history freezes at capacity; uploads remain admitted; maximum upload count fits"); +} +""" +with tempfile.TemporaryDirectory() as d: + root=Path(d);c=root/'history.c';c.write_text(harness) + for opt in ('-O0','-O2'): + exe=root/(opt+'.exe') + subprocess.run([a.cc,opt,'-std=c11','-Wall','-Wextra','-Werror',str(c),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True) From e3beb365ba2be376784b8fb47cefac15c53581c4 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 19:11:21 -0700 Subject: [PATCH 29/56] Add opt-in native-wide presentation hold for retained loading scenes --- docs/MOD_PACKAGES.md | 22 ++++++++++++++++++++++ runtime/CMakeLists.txt | 3 +++ runtime/include/mod_plugins.h | 10 ++++++++++ runtime/include/ws_scene_hold.h | 19 +++++++++++++++++++ runtime/src/gpu.c | 19 +++++++++++++++++-- runtime/tests/test_ws_scene_hold.c | 23 +++++++++++++++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 runtime/include/ws_scene_hold.h create mode 100644 runtime/tests/test_ws_scene_hold.c diff --git a/docs/MOD_PACKAGES.md b/docs/MOD_PACKAGES.md index de06eb4b..75d0a607 100644 --- a/docs/MOD_PACKAGES.md +++ b/docs/MOD_PACKAGES.md @@ -602,3 +602,25 @@ off, so the default presentation is unchanged: letterbox and pillarbox margins remain black. Enabling the feature without choosing artwork is also a no-op. The package supplies only the declaration and trusted plugin selection; archives still cannot load native code. + +### Retained-scene loading presentation (native-wide opt-in) + +A trusted game plugin can register +`psx_mod_set_retained_scene_predicate(predicate)` from `mod_plugins.h` when the +game keeps displaying its previous framebuffer while loading. The cheap, pure +emulation-thread callback must return nonzero only while that same scene is +retained. It must not call presentation APIs recursively. Passing NULL removes +the opt-in. Without registration, existing presentation behavior is unchanged. + +In native-wide mode this holds the previous wide/4:3 classification even if a +game-state flag or absent GTE activity would normally classify loading as 2D. +It does not force menus wide, stretch artwork, change guest rendering or memory, +or override the FMV veto. Release the predicate when a new scene replaces the +retained image, so ordinary classification resumes. Crash's experimental +adaptive feature uses its pending level transition and draw-skip globals for +this; these game-specific addresses do not belong in the framework. + +GPU reset and savestate restore discard this host-only history. A save loaded +directly into a frozen loading frame cannot recreate wide reveal strips absent +from the canonical saved framebuffer. `ws_scene_hold_test` covers long holds, +menu release, retained 4:3 scenes, FMV priority and timeline reset. diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 0c41349c..1e033f30 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -23,6 +23,9 @@ psxrecomp_add_runtime_target(psx-runtime # (docs/ENHANCEMENTS.md G1.10). Titles get their <exe>_pgxp the same way. if(BUILD_TESTING) + add_executable(ws_scene_hold_test tests/test_ws_scene_hold.c) + target_include_directories(ws_scene_hold_test PRIVATE include) + add_test(NAME ws_scene_hold_test COMMAND ws_scene_hold_test) add_executable(mod_texture_banks_test tests/test_mod_texture_banks.c src/mod_texture_banks.c) target_include_directories(mod_texture_banks_test PRIVATE include) add_test(NAME mod_texture_banks_test COMMAND mod_texture_banks_test) diff --git a/runtime/include/mod_plugins.h b/runtime/include/mod_plugins.h index bacae0c2..d22c2edc 100644 --- a/runtime/include/mod_plugins.h +++ b/runtime/include/mod_plugins.h @@ -104,6 +104,16 @@ uint32_t psx_mod_display_width(void); /* Height companion to psx_mod_display_width(); same conventions. */ uint32_t psx_mod_display_height(void); +/* Opt-in presentation hold for a game that retains its previous framebuffer + * while loading. The pure, cheap emulation-thread predicate returns nonzero + * only while that SAME scene remains displayed; no GPU/API recursion allowed. + * Native-wide retains its prior wide/4:3 classification, never stretches art + * and never overrides FMV. NULL removes the opt-in. Host history is discarded + * on GPU reset/savestate restore, so loading a frozen scene cannot recreate + * missing widescreen strips. This does not change guest rendering or memory. */ +typedef int (*PSXModRetainedScenePredicate)(void); +void psx_mod_set_retained_scene_predicate(PSXModRetainedScenePredicate predicate); + /* * Read the committed value of one of this package's declared options, as the * player left it in the launcher (or the manifest default when untouched). diff --git a/runtime/include/ws_scene_hold.h b/runtime/include/ws_scene_hold.h new file mode 100644 index 00000000..a3307917 --- /dev/null +++ b/runtime/include/ws_scene_hold.h @@ -0,0 +1,19 @@ +#pragma once + +/* Host presentation history, not guest hardware state. Never serialize it: + * a restored canonical framebuffer cannot reconstruct discarded wide strips. */ +typedef struct WsSceneHold { int valid, native_43; } WsSceneHold; +static inline void ws_scene_hold_reset(WsSceneHold* state) { + state->valid = 0; state->native_43 = 1; +} +static inline int ws_scene_hold_classify(WsSceneHold* state, int hold, + int native_43, int fmv) { + /* FMV always wins, and cannot seed a later wide retained scene. */ + if (fmv) { + state->valid = 1; state->native_43 = 1; + return 1; + } + if (hold && state->valid) return state->native_43; + state->valid = 1; state->native_43 = native_43 != 0; + return state->native_43; +} diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index 1d671cd0..719d33f0 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -27,6 +27,8 @@ #include "event_ring.h" #include "color_lut.h" #include "mod_runtime.h" +#include "mod_plugins.h" +#include "ws_scene_hold.h" #include "sio.h" #include "ws_cull_detect.h" #include "ws_aspect_cone_math.h" @@ -374,17 +376,28 @@ static int ws_2d_only_scene(void) { static uint32_t s_ws_fmv_frame_cache = 0xFFFFFFFFu; static int s_ws_fmv_cached = 0; +static PSXModRetainedScenePredicate s_ws_retained_scene_predicate; +static WsSceneHold s_ws_scene_hold; + +void psx_mod_set_retained_scene_predicate(PSXModRetainedScenePredicate predicate) { + s_ws_retained_scene_predicate = predicate; + ws_scene_hold_reset(&s_ws_scene_hold); +} int gpu_ws_present_native_43(void) { if (!ws_engaged()) return 0; - if (!ws_game_mode()) return 1; /* full-2D screen */ - if (ws_2d_only_scene()) return 1; /* 2D-only gameplay scene */ + int native_43 = !ws_game_mode() || ws_2d_only_scene(); + int hold_enabled = ws_mode == 2 && s_ws_retained_scene_predicate != NULL; + if (!hold_enabled && native_43) return 1; /* unchanged default */ uint32_t f = (uint32_t)s_frame_count; if (f != s_ws_fmv_frame_cache) { s_ws_fmv_frame_cache = f; GpuDisplayInfo di; gpu_get_display_info(&di); s_ws_fmv_cached = di.depth24 || mdec_recently_active(WS_FMV_HYSTERESIS); } + if (hold_enabled) + return ws_scene_hold_classify(&s_ws_scene_hold, + s_ws_retained_scene_predicate(), native_43, s_ws_fmv_cached); return s_ws_fmv_cached; } @@ -2853,6 +2866,7 @@ static void gpu_reset_state(int clear_vram) { gpustat_poll_count = 0; s_ws_fmv_frame_cache = 0xFFFFFFFFu; s_ws_fmv_cached = 0; + ws_scene_hold_reset(&s_ws_scene_hold); s_d24_upload_x1 = 0; s_d24_present_hold = 0; s_d24_prev_disp_h = 0; @@ -6123,6 +6137,7 @@ int gpu_snapshot_read(const uint8_t *p, uint32_t len) { if (len != gpu_snapshot_bytes()) return 0; pst_r_init(&r, p, len); if (!gpu_snap_parse(&r)) return 0; + ws_scene_hold_reset(&s_ws_scene_hold); ws_hud_anchor_clear(ws_hud_anchor_tags, WS_HUD_ANCHOR_TABLE_SIZE); ws_hud_anchor_clear(ws_reveal_clear_tags, WS_HUD_ANCHOR_TABLE_SIZE); ws_repeat_rect_tag_clear(ws_repeat_rect_tags); diff --git a/runtime/tests/test_ws_scene_hold.c b/runtime/tests/test_ws_scene_hold.c new file mode 100644 index 00000000..dd228c62 --- /dev/null +++ b/runtime/tests/test_ws_scene_hold.c @@ -0,0 +1,23 @@ +#include "ws_scene_hold.h" +#undef NDEBUG /* the Release test target must execute its assertions */ +#include <assert.h> +#include <stdio.h> + +int main(void) { + WsSceneHold s; + ws_scene_hold_reset(&s); + assert(ws_scene_hold_classify(&s,0,0,0)==0); /* world */ + for (int i=0;i<300;++i) + assert(ws_scene_hold_classify(&s,1,1,0)==0); /* no GTE during load */ + assert(ws_scene_hold_classify(&s,0,1,0)==1); /* actual menu replaces it */ + assert(ws_scene_hold_classify(&s,1,0,0)==1); /* never force a 2D scene wide */ + assert(ws_scene_hold_classify(&s,0,0,0)==0); + assert(ws_scene_hold_classify(&s,1,0,1)==1); /* FMV veto */ + assert(ws_scene_hold_classify(&s,1,0,0)==1); + assert(ws_scene_hold_classify(&s,0,0,0)==0); + ws_scene_hold_reset(&s); /* cannot reuse unrelated pre-restore margins */ + assert(ws_scene_hold_classify(&s,1,1,0)==1); + assert(ws_scene_hold_classify(&s,0,0,0)==0); + puts("PASS retained scene, menu release, FMV veto and timeline reset"); + return 0; +} From f2352e3d32043c6644b5afca01732f95267521f4 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 19:27:03 -0700 Subject: [PATCH 30/56] Keep retained scene aspect until the next displayed buffer flip --- docs/MOD_PACKAGES.md | 12 ++++++++---- runtime/include/mod_plugins.h | 11 ++++++++++- runtime/include/ws_scene_hold.h | 22 +++++++++++++++++++--- runtime/src/gpu.c | 7 ++++++- runtime/tests/test_ws_scene_hold.c | 30 ++++++++++++++++++------------ 5 files changed, 61 insertions(+), 21 deletions(-) diff --git a/docs/MOD_PACKAGES.md b/docs/MOD_PACKAGES.md index 75d0a607..655f39cf 100644 --- a/docs/MOD_PACKAGES.md +++ b/docs/MOD_PACKAGES.md @@ -608,19 +608,23 @@ still cannot load native code. A trusted game plugin can register `psx_mod_set_retained_scene_predicate(predicate)` from `mod_plugins.h` when the game keeps displaying its previous framebuffer while loading. The cheap, pure -emulation-thread callback must return nonzero only while that same scene is +emulation-thread callback returns `PSX_MOD_SCENE_HOLD` while that same scene is retained. It must not call presentation APIs recursively. Passing NULL removes the opt-in. Without registration, existing presentation behavior is unchanged. In native-wide mode this holds the previous wide/4:3 classification even if a game-state flag or absent GTE activity would normally classify loading as 2D. It does not force menus wide, stretch artwork, change guest rendering or memory, -or override the FMV veto. Release the predicate when a new scene replaces the -retained image, so ordinary classification resumes. Crash's experimental +or override the FMV veto. Return `PSX_MOD_SCENE_RELEASE` when a new scene replaces +the retained image, so ordinary classification resumes. For double-buffered +games whose draw-ready signal precedes the actual display flip, return +`PSX_MOD_SCENE_UNTIL_FLIP`: the prior hold ends only when the displayed VRAM +origin changes. Without a prior HOLD it behaves like RELEASE. Do not use +UNTIL_FLIP for in-place image replacement. Crash's experimental adaptive feature uses its pending level transition and draw-skip globals for this; these game-specific addresses do not belong in the framework. GPU reset and savestate restore discard this host-only history. A save loaded directly into a frozen loading frame cannot recreate wide reveal strips absent from the canonical saved framebuffer. `ws_scene_hold_test` covers long holds, -menu release, retained 4:3 scenes, FMV priority and timeline reset. +menu release, delayed buffer flips, retained 4:3 scenes, FMV and timeline reset. diff --git a/runtime/include/mod_plugins.h b/runtime/include/mod_plugins.h index d22c2edc..ccae3060 100644 --- a/runtime/include/mod_plugins.h +++ b/runtime/include/mod_plugins.h @@ -105,13 +105,22 @@ uint32_t psx_mod_display_width(void); uint32_t psx_mod_display_height(void); /* Opt-in presentation hold for a game that retains its previous framebuffer - * while loading. The pure, cheap emulation-thread predicate returns nonzero + * while loading. The pure, cheap emulation-thread predicate returns HOLD * only while that SAME scene remains displayed; no GPU/API recursion allowed. + * RELEASE resumes normal classification immediately. UNTIL_FLIP releases a + * prior HOLD only once the displayed VRAM origin changes: useful when drawing + * the next backbuffer finishes before it becomes visible. UNTIL_FLIP without + * a prior HOLD does nothing. Do not use it for in-place scene replacements. * Native-wide retains its prior wide/4:3 classification, never stretches art * and never overrides FMV. NULL removes the opt-in. Host history is discarded * on GPU reset/savestate restore, so loading a frozen scene cannot recreate * missing widescreen strips. This does not change guest rendering or memory. */ typedef int (*PSXModRetainedScenePredicate)(void); +enum { + PSX_MOD_SCENE_RELEASE = 0, + PSX_MOD_SCENE_HOLD = 1, + PSX_MOD_SCENE_UNTIL_FLIP = 2 +}; void psx_mod_set_retained_scene_predicate(PSXModRetainedScenePredicate predicate); /* diff --git a/runtime/include/ws_scene_hold.h b/runtime/include/ws_scene_hold.h index a3307917..1ad55744 100644 --- a/runtime/include/ws_scene_hold.h +++ b/runtime/include/ws_scene_hold.h @@ -1,18 +1,34 @@ #pragma once +#include <stdint.h> /* Host presentation history, not guest hardware state. Never serialize it: * a restored canonical framebuffer cannot reconstruct discarded wide strips. */ -typedef struct WsSceneHold { int valid, native_43; } WsSceneHold; +typedef struct WsSceneHold { + int valid, native_43, awaiting_flip; + uint32_t held_origin; +} WsSceneHold; static inline void ws_scene_hold_reset(WsSceneHold* state) { - state->valid = 0; state->native_43 = 1; + state->valid = 0; state->native_43 = 1; state->awaiting_flip = 0; + state->held_origin = 0; } static inline int ws_scene_hold_classify(WsSceneHold* state, int hold, - int native_43, int fmv) { + int native_43, int fmv, + uint32_t display_origin) { /* FMV always wins, and cannot seed a later wide retained scene. */ if (fmv) { state->valid = 1; state->native_43 = 1; + state->awaiting_flip = 0; return 1; } + if (hold == 1) { + state->held_origin = display_origin; + state->awaiting_flip = 1; + } else if (hold != 1) { + /* 2: the backbuffer is ready, but not necessarily on screen yet. */ + hold = hold == 2 && state->awaiting_flip && + display_origin == state->held_origin; + if (!hold) state->awaiting_flip = 0; + } if (hold && state->valid) return state->native_43; state->valid = 1; state->native_43 = native_43 != 0; return state->native_43; diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index 719d33f0..1deb41b6 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -378,6 +378,7 @@ static uint32_t s_ws_fmv_frame_cache = 0xFFFFFFFFu; static int s_ws_fmv_cached = 0; static PSXModRetainedScenePredicate s_ws_retained_scene_predicate; static WsSceneHold s_ws_scene_hold; +static uint32_t ws_display_origin(void); void psx_mod_set_retained_scene_predicate(PSXModRetainedScenePredicate predicate) { s_ws_retained_scene_predicate = predicate; @@ -397,7 +398,8 @@ int gpu_ws_present_native_43(void) { } if (hold_enabled) return ws_scene_hold_classify(&s_ws_scene_hold, - s_ws_retained_scene_predicate(), native_43, s_ws_fmv_cached); + s_ws_retained_scene_predicate(), native_43, s_ws_fmv_cached, + ws_display_origin()); return s_ws_fmv_cached; } @@ -2585,6 +2587,9 @@ extern void psx_irq_raise(uint32_t bit, uint32_t detail); /* Display area start (GP1(05h)) */ static uint32_t display_area_x; static uint32_t display_area_y; +static uint32_t ws_display_origin(void) { + return display_area_x | (display_area_y << 10); +} /* ----- Native-wide compositor driving (see runtime/src/gpu_sw_renderer.c) ---- * The renderer keeps a separate wide surface per framebuffer; we tell it which diff --git a/runtime/tests/test_ws_scene_hold.c b/runtime/tests/test_ws_scene_hold.c index dd228c62..27dd4991 100644 --- a/runtime/tests/test_ws_scene_hold.c +++ b/runtime/tests/test_ws_scene_hold.c @@ -6,18 +6,24 @@ int main(void) { WsSceneHold s; ws_scene_hold_reset(&s); - assert(ws_scene_hold_classify(&s,0,0,0)==0); /* world */ + assert(ws_scene_hold_classify(&s,2,0,0,512)==0); /* normal world */ for (int i=0;i<300;++i) - assert(ws_scene_hold_classify(&s,1,1,0)==0); /* no GTE during load */ - assert(ws_scene_hold_classify(&s,0,1,0)==1); /* actual menu replaces it */ - assert(ws_scene_hold_classify(&s,1,0,0)==1); /* never force a 2D scene wide */ - assert(ws_scene_hold_classify(&s,0,0,0)==0); - assert(ws_scene_hold_classify(&s,1,0,1)==1); /* FMV veto */ - assert(ws_scene_hold_classify(&s,1,0,0)==1); - assert(ws_scene_hold_classify(&s,0,0,0)==0); - ws_scene_hold_reset(&s); /* cannot reuse unrelated pre-restore margins */ - assert(ws_scene_hold_classify(&s,1,1,0)==1); - assert(ws_scene_hold_classify(&s,0,0,0)==0); - puts("PASS retained scene, menu release, FMV veto and timeline reset"); + assert(ws_scene_hold_classify(&s,1,1,0,512)==0); /* held loading */ + assert(ws_scene_hold_classify(&s,2,1,0,512)==0); /* backbuffer ready */ + assert(ws_scene_hold_classify(&s,2,1,0,512)==0); /* still same frontbuffer */ + assert(ws_scene_hold_classify(&s,2,1,0,0)==1); /* real title becomes visible */ + assert(ws_scene_hold_classify(&s,2,0,0,512)==0); /* no hold on later flips */ + assert(ws_scene_hold_classify(&s,1,1,0,512)==0); + assert(ws_scene_hold_classify(&s,0,1,0,512)==1); /* immediate in-place release */ + assert(ws_scene_hold_classify(&s,1,0,0,512)==1); /* retained 4:3 stays 4:3 */ + assert(ws_scene_hold_classify(&s,0,0,0,0)==0); + assert(ws_scene_hold_classify(&s,1,0,1,0)==1); /* FMV veto */ + assert(ws_scene_hold_classify(&s,2,0,1,0)==1); + assert(ws_scene_hold_classify(&s,0,0,0,0)==0); + assert(ws_scene_hold_classify(&s,1,1,0,0)==0); + ws_scene_hold_reset(&s); /* unrelated pre-restore margins cannot survive */ + assert(ws_scene_hold_classify(&s,2,1,0,0)==1); + assert(ws_scene_hold_classify(&s,0,0,0,0)==0); + puts("PASS retained scene, delayed flip, immediate release, FMV and reset"); return 0; } From 8b689140ddc31c3158b7bcefe19591059f250978 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 13 Sep 2026 21:06:58 -0700 Subject: [PATCH 31/56] Add opt-in world scene classification and immutable terrain batching --- runtime/CMakeLists.txt | 5 +++ runtime/include/mod_plugins.h | 16 +++++++ runtime/include/mod_texture_banks.h | 1 + runtime/src/gpu.c | 9 ++++ runtime/src/gpu_gl_renderer.c | 8 +++- runtime/src/mod_texture_banks.c | 7 ++++ runtime/tests/test_gl_readback_region.c | 33 +++++++++++++++ runtime/tests/test_mod_texture_banks.c | 7 ++++ runtime/tests/test_ws_world_scene.c | 55 +++++++++++++++++++++++++ 9 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 runtime/tests/test_ws_world_scene.c diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 99db2120..76184bea 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -30,6 +30,11 @@ if(BUILD_TESTING) target_include_directories(mod_texture_banks_test PRIVATE include) add_test(NAME mod_texture_banks_test COMMAND mod_texture_banks_test) if(NOT MSVC) + add_executable(ws_world_scene_test tests/test_ws_world_scene.c) + target_include_directories(ws_world_scene_test PRIVATE include src) + target_compile_options(ws_world_scene_test PRIVATE -O2 -flto -ffunction-sections -fdata-sections) + target_link_options(ws_world_scene_test PRIVATE -flto -Wl,--gc-sections) + add_test(NAME ws_world_scene_test COMMAND ws_world_scene_test) add_executable(mod_memory_snapshot_test tests/test_mod_memory_snapshot.c) target_include_directories(mod_memory_snapshot_test PRIVATE include src) target_compile_options(mod_memory_snapshot_test PRIVATE -O2 -flto -ffunction-sections -fdata-sections) diff --git a/runtime/include/mod_plugins.h b/runtime/include/mod_plugins.h index ccae3060..5320fb2b 100644 --- a/runtime/include/mod_plugins.h +++ b/runtime/include/mod_plugins.h @@ -43,6 +43,11 @@ int psx_mod_texture_banks_supported(void); int psx_mod_define_texture_bank(uint16_t id, uint32_t width, uint32_t height, const uint16_t* pixels); void psx_mod_set_texture_bank_resolver(PSXModTextureBankResolver resolver); +/* Default-off GL optimization: batch immutable-bank semi triangles in painter + * order on the single-pass dual-source path only. Ordinary VRAM, subtractive + * blending and destination-mask checks retain per-primitive isolation. Call + * from activation or an emulation-thread render boundary. */ +void psx_mod_set_texture_bank_batching(int enabled); /* A dedicated GPU-DMA packet arena. Only GT3 commands sourced from this * allocation interpret C1/C2's otherwise-unused high bytes as a bank ID: * id = (C1 >> 24) | ((C2 >> 24) << 8). ID zero uses ordinary VRAM. The rest @@ -123,6 +128,17 @@ enum { }; void psx_mod_set_retained_scene_predicate(PSXModRetainedScenePredicate predicate); +/* Opt-in supplemental native-wide world classifier. A nonzero result marks a + * known, rendered 3D scene (for example a real-time intro) as world content even + * when a game's gameplay-state allowlist excludes it. Zero defers to the normal + * classifier; NULL removes the callback. Only native-wide mode consults it. + * FMV and retained-frame presentation rules still apply. The pure, cheap callback + * runs on the emulation thread: no GPU calls, allocation or guest-state writes. + * Registration is host configuration, not savestate data. No default behavior + * changes and no extra geometry is generated by this service. */ +typedef int (*PSXModWorldScenePredicate)(void); +void psx_mod_set_world_scene_predicate(PSXModWorldScenePredicate predicate); + /* * Read the committed value of one of this package's declared options, as the * player left it in the launcher (or the manifest default when untouched). diff --git a/runtime/include/mod_texture_banks.h b/runtime/include/mod_texture_banks.h index 2e79ac25..ed80c0a3 100644 --- a/runtime/include/mod_texture_banks.h +++ b/runtime/include/mod_texture_banks.h @@ -6,6 +6,7 @@ extern "C" { const uint16_t* mod_texture_bank_pixels(uint16_t id, uint32_t* width, uint32_t* height); uint16_t mod_texture_packet_bank(uint32_t source, const uint32_t* words, uint32_t count); int mod_texture_packet_precision(uint32_t source, float q[3], float xy[6]); +int mod_texture_bank_batchable(int immutable, int mask_check, int semi); #ifdef __cplusplus } #endif diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index b4caf6c3..761dcb06 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -335,7 +335,15 @@ static int ws_full_2d_mode(void) { if (env < 0) { const char *e = getenv("PSX_WS_FORCE_2D"); env = (e && e[0] == '1') ? 1 : 0; } return ws_full_2d || env; } +static PSXModWorldScenePredicate s_ws_world_scene_predicate; +void psx_mod_set_world_scene_predicate(PSXModWorldScenePredicate predicate) { + s_ws_world_scene_predicate = predicate; +} +static int ws_mod_world_scene(void) { + return ws_mode == 2 && s_ws_world_scene_predicate && s_ws_world_scene_predicate(); +} static int ws_game_mode(void) { + if (ws_mod_world_scene()) return 1; int state_match = ws_gameplay_state_matches(); if (state_match >= 0) return state_match; if (ws_full_2d_mode()) return 1; @@ -370,6 +378,7 @@ static int ws_game_mode(void) { * real scene change crosses it in ~0.1 s. */ #define WS_2D_SCENE_HYSTERESIS 6u static int ws_2d_only_scene(void) { + if (ws_mod_world_scene()) return 0; if (ws_full_2d_mode() || ws_gte_game_mode_cfg) return 0; return (uint32_t)s_frame_count - ws_sust_ovh_stamp > WS_2D_SCENE_HYSTERESIS; } diff --git a/runtime/src/gpu_gl_renderer.c b/runtime/src/gpu_gl_renderer.c index d78b44f4..d3cfebcf 100644 --- a/runtime/src/gpu_gl_renderer.c +++ b/runtime/src/gpu_gl_renderer.c @@ -2092,8 +2092,12 @@ static void gpu_textured_triangle(const int *xs, const int *ys, * mis-orders against neighbouring opaque geometry. Isolate EVERY * semi-transparent textured prim: drain the open batch, draw this * prim alone (composited fully before the next), let opaque prims - * keep batching. Cost is one draw per semi prim. */ - int isolate = (semi >= 0); + * keep batching. Cost is one draw per semi prim. A separately opted-in + * immutable bank may batch the single-pass dual-source cases: it + * cannot alias a render target, keeps painter order, and still splits + * on opaque transitions, bank/state changes, masking or subtraction. */ + int isolate = semi >= 0 && + !mod_texture_bank_batchable(s_selected_bank_tex != 0, s_mask_check, semi); int reason = -1; if (s_tb_n > 0) { if (s_tb_bank_tex != s_selected_bank_tex) reason = 0; diff --git a/runtime/src/mod_texture_banks.c b/runtime/src/mod_texture_banks.c index 5eba304d..d5767db1 100644 --- a/runtime/src/mod_texture_banks.c +++ b/runtime/src/mod_texture_banks.c @@ -13,6 +13,13 @@ static unsigned arena_count; static uint32_t bank_bytes; static PSXModTextureBankResolver resolve_bank; static int resolving; +static int bank_batching; + +void psx_mod_set_texture_bank_batching(int enabled) { bank_batching = enabled != 0; } +int mod_texture_bank_batchable(int immutable, int mask_check, int semi) { + return bank_batching && immutable && !mask_check && + (semi == 0 || semi == 1 || semi == 3); +} void psx_mod_set_texture_bank_resolver(PSXModTextureBankResolver resolver) { resolve_bank = resolver; diff --git a/runtime/tests/test_gl_readback_region.c b/runtime/tests/test_gl_readback_region.c index bdc71b5d..9d9f733a 100644 --- a/runtime/tests/test_gl_readback_region.c +++ b/runtime/tests/test_gl_readback_region.c @@ -28,6 +28,38 @@ static void verify(const char *label){ if(n)fprintf(stderr,"%s: %d native words differ\n",label,n); check(n==0,label);check(glGetError()==GL_NO_ERROR,"GL error"); } +static void verify_bank_batching(void) { + static uint16_t bank[256*128], baseline[96*96], result[96*96]; + for(int i=256;i<256*128;++i)bank[i]=0x3210; + bank[0]=0;bank[1]=0x001f;bank[2]=0x83e0;bank[3]=0xfc00; + check(psx_mod_define_texture_bank(8,256,128,bank),"batch fixture bank"); + for(int filter=0;filter<2;++filter) for(int mask=0;mask<2;++mask) { + int counts[2]; + for(int enabled=0;enabled<2;++enabled) { + gl_renderer_select_texture_bank(0); + glb_set_mask_bits(0,0);glb_set_semi_transparency(0,0); + glb_draw_flat_rect(400,300,96,96,0x1234);flush_flat_batch(); + psx_mod_set_texture_bank_batching(enabled); + s_tex_filter=filter;glb_set_mask_bits(0,mask); + gl_renderer_select_texture_bank(8); + const int before=s_cw_batches; + for(int i=0;i<36;++i) { + const int x=404+(i%6)*5,y=304+(i%4)*7; + glb_set_semi_transparency(1,(i/6)%4); + glb_draw_shaded_textured_triangle(x,y,0,2,0x808080, + x+44,y+2,63,2,0x507090,x+3,y+48,0,65,0x907050,0,0,0,0); + } + flush_tex_batch();counts[enabled]=s_cw_batches-before; + gl_renderer_select_texture_bank(0); + gl_renderer_sync_cpu(); + check(gl_renderer_fbo_peek(400,300,96,96,enabled?result:baseline),"batch pixel read"); + } + check(memcmp(baseline,result,sizeof baseline)==0,"ordered semi batching pixel equivalence"); + check(mask?counts[1]==counts[0]:counts[1]<counts[0],"batch reduction only on supported path"); + } + psx_mod_set_texture_bank_batching(0);s_tex_filter=0; + glb_set_mask_bits(0,0);glb_set_semi_transparency(0,0); +} int main(int argc,char **argv){ int scale=argc>1?atoi(argv[1]):1; if(SDL_Init(SDL_INIT_VIDEO)!=0)return 2; @@ -94,6 +126,7 @@ int main(int argc,char **argv){ check(gl_renderer_select_texture_bank(0),"reset bank after direct texture"); check(glb_vram_read(302,252)==0x001f,"retained 16-bit texel"); verify("retained banks and original VRAM ordered together"); + verify_bank_batching(); printf("checks=%d failures=%d\n",checks,failures); gl_renderer_shutdown();SDL_DestroyWindow(win);SDL_Quit();return failures?1:0; } diff --git a/runtime/tests/test_mod_texture_banks.c b/runtime/tests/test_mod_texture_banks.c index 9b39ed7c..2bd255f5 100644 --- a/runtime/tests/test_mod_texture_banks.c +++ b/runtime/tests/test_mod_texture_banks.c @@ -18,6 +18,13 @@ static void check(int ok,const char* text){if(!ok){fprintf(stderr,"FAIL %s\n",te static int resolve(uint16_t id){uint16_t data=0x4567;++resolves;return psx_mod_define_texture_bank(id,1,1,&data);} static uint32_t bits(float f){uint32_t n;memcpy(&n,&f,4);return n;} int main(void){ + check(!mod_texture_bank_batchable(1,0,0),"bank batching defaults off"); + psx_mod_set_texture_bank_batching(1); + for(int mode=-1;mode<=4;++mode) for(int bank=0;bank<=1;++bank) for(int mask=0;mask<=1;++mask) + check(mod_texture_bank_batchable(bank,mask,mode)== + (bank && !mask && (mode==0 || mode==1 || mode==3)),"single-pass opt-in scope"); + psx_mod_set_texture_bank_batching(0); + check(!mod_texture_bank_batchable(1,0,0),"bank batching disable"); uint16_t data[4]={1,2,3,4};uint32_t w,h; check(!psx_mod_define_texture_bank(0,2,2,data),"bank zero reserved"); check(!psx_mod_define_texture_bank(1,1025,2,data),"dimensions bounded"); diff --git a/runtime/tests/test_ws_world_scene.c b/runtime/tests/test_ws_world_scene.c new file mode 100644 index 00000000..d4c9e0a5 --- /dev/null +++ b/runtime/tests/test_ws_world_scene.c @@ -0,0 +1,55 @@ +/* Exercise production classifier; section GC omits unrelated GPU execution. */ +#include "../src/gpu.c" +#undef NDEBUG +#include <assert.h> + +uint64_t s_frame_count; +static uint32_t state_word; +static int world, calls, fmv, hold; +uint32_t psx_read_word(uint32_t address) { (void)address; return state_word; } +int mdec_recently_active(uint32_t frames) { (void)frames; return fmv; } +static int world_scene(void) { ++calls; return world; } +static int retained_scene(void) { return hold; } +static int native_43(void) { ++s_frame_count; return gpu_ws_present_native_43(); } + +int main(void) { + uint32_t allowed = 0x100; + gpu_ws_set_gameplay_state_gate(0x80001000u, &allowed, 1); + world = 1; + psx_mod_set_world_scene_predicate(world_scene); + ws_mode = 0; + assert(native_43() == 0 && calls == 0); /* stock is untouched */ + ws_mode = 1; + assert(native_43() == 1 && calls == 0); /* squash does not consult mod */ + ws_mode = 2; + assert(native_43() == 0 && calls > 0); /* overrides state AND 2D detector */ + world = 0; + assert(native_43() == 1); /* false defers, never forces wide */ + gpu_ws_set_gte_game_mode(1); + state_word = 0x100; + assert(native_43() == 0); /* original allowlist still works */ + state_word = 0; + assert(native_43() == 1); + world = 1; + fmv = 1; + assert(native_43() == 1); /* streamed FMV veto */ + fmv = 0; + display_depth = 1; + assert(native_43() == 1); /* 24-bit FMV veto */ + display_depth = 0; + assert(native_43() == 0); + psx_mod_set_retained_scene_predicate(retained_scene); + hold = 0; + assert(native_43() == 0); + world = 0; + hold = 1; + assert(native_43() == 0); /* old world retained during loading */ + hold = 0; + assert(native_43() == 1); /* actual menu returns to 4:3 */ + psx_mod_set_retained_scene_predicate(NULL); + world = 1; + psx_mod_set_world_scene_predicate(NULL); + assert(native_43() == 1); /* unregistration restores original policy */ + puts("PASS opt-in scope, world scene, defer, FMV veto, retained frame and unregister"); + return 0; +} From d4abc2f05daf8f11c685730c562a91b573e9627b Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Sun, 2 Aug 2026 23:28:36 +0300 Subject: [PATCH 32/56] Fix non-RA JALR interpreter handoff (cherry picked from commit 2f8f627b242ab4bcdf6840729edf3d81158b2525) --- recompiler/CMakeLists.txt | 1 + runtime/src/dirty_ram_interp.c | 18 ++++++++- .../tests/test_jalr_link_register_guards.py | 37 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 runtime/tests/test_jalr_link_register_guards.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 4dd33e72..4099b660 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -680,6 +680,7 @@ if(BUILD_TESTING) frame_interpolation_display_rate hybrid_dev_any_input launcher_pad_mode_wiring + jalr_link_register_guards launcher_vulkan_option mod_controller_override mod_load_acceleration diff --git a/runtime/src/dirty_ram_interp.c b/runtime/src/dirty_ram_interp.c index 922c9fe8..b6ba33fd 100644 --- a/runtime/src/dirty_ram_interp.c +++ b/runtime/src/dirty_ram_interp.c @@ -1648,7 +1648,16 @@ static int exec_one_fetched_inner(CPUState *cpu, uint32_t pc, uint32_t insn, uint32_t target = cpu->gpr[rs]; if (target & 3) return interp_exception(cpu, 4, target, pc); /* LoadAddressError */ uint32_t return_pc = pc + 8; - cpu->gpr[rd ? rd : 31] = return_pc; + /* JALR writes the encoded rd exactly. The one-operand assembler + * form encodes rd=$ra; rd=0 is a real discard, not an implicit + * $ra. More importantly, only rd=$ra gives the call-unit code a + * return contract it can validate. Games also use other link + * registers for data-bearing transfer stubs (for example + * `jalr $a1,$t0`, where the target consumes $a1 and later returns + * through the pre-existing $ra). Treating pc+8 as that transfer's + * mandatory return address fabricates a nested continuation and + * leaks psx_dispatch_call frames. */ + if (rd != 0) cpu->gpr[rd] = return_pc; cpu->gpr[0] = 0; exec_delay_slot(cpu, pc + 4); cosim_exec_one_transfer_hook(pc + 4); @@ -1660,6 +1669,13 @@ static int exec_one_fetched_inner(CPUState *cpu, uint32_t pc, uint32_t insn, uint32_t _cr = callret_begin(cpu, pc, target); /* call-resolution ring */ #define CRET(code, rv) do { callret_end(_cr, cpu, (code)); return (rv); } while (0) if (g_precise_mode || g_ls_replay_active) { cpu->pc = target; CRET(CRES_PLAIN, 1); } /* slice / lockstep-replay: plain transfer, never execute the callee */ + if (rd != 31) { + /* No architectural $ra contract: preserve the transfer as a + * pc-chain and let the callee's eventual JR choose the real + * continuation. This is both faithful and host-stack-flat. */ + cpu->pc = target; + CRET(CRES_PCCHAIN, 1); + } #ifdef PSX_HAS_GAME_DISPATCH cpu->pc = 0; if (interp_enter_compiled(cpu, target)) { diff --git a/runtime/tests/test_jalr_link_register_guards.py b/runtime/tests/test_jalr_link_register_guards.py new file mode 100644 index 00000000..bd28e69e --- /dev/null +++ b/runtime/tests/test_jalr_link_register_guards.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Guard architectural JALR link-register and non-$ra transfer handling.""" + +from pathlib import Path +import sys + + +def main() -> int: + root = Path(__file__).resolve().parents[2] + source = (root / "runtime/src/dirty_ram_interp.c").read_text(encoding="utf-8") + + start = source.index("case 0x09: { /* JALR rd, rs */") + end = source.index("case 0x0C: /* SYSCALL */", start) + jalr = source[start:end] + + if "if (rd != 0) cpu->gpr[rd] = return_pc;" not in jalr: + raise AssertionError("JALR must write the encoded nonzero link register") + if "cpu->gpr[rd ? rd : 31]" in jalr: + raise AssertionError("JALR rd=0 must not be rewritten as an implicit $ra link") + + chain = jalr.index("if (rd != 31)") + compiled = jalr.index("interp_enter_compiled(cpu, target)") + nonlocal_call = jalr.index("dispatch_nonlocal_call(cpu, target") + if not chain < compiled < nonlocal_call: + raise AssertionError( + "non-$ra JALR must pc-chain before any call-unit optimization" + ) + non_ra = jalr[chain:compiled] + if "cpu->pc = target;" not in non_ra or "CRET(CRES_PCCHAIN, 1);" not in non_ra: + raise AssertionError("non-$ra JALR is missing its faithful pc-chain handoff") + + print("JALR link-register guards: ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 5e23de8393a3041a1a038b0241ffe85c302e237e Mon Sep 17 00:00:00 2001 From: Pegasus <pegasus@psxrecomp.local> Date: Fri, 21 Aug 2026 00:08:49 +0300 Subject: [PATCH 33/56] feat: add PS1 conformance harness and fix GTE fidelity (cherry picked from commit 98d617b5f336125726ac1761f753e89abf99d416) --- docs/CONFORMANCE_TESTROM_STATUS.md | 51 +++ recompiler/CMakeLists.txt | 6 + runtime/include/gte.h | 5 +- runtime/include/guest_tty.h | 23 ++ runtime/runtime.cmake | 1 + runtime/src/debug_server.c | 36 ++ runtime/src/gte.cpp | 400 +++++++++++---------- runtime/src/guest_tty.c | 60 ++++ runtime/src/memory.c | 1 + runtime/tests/test_gte_register_access.cpp | 31 +- runtime/tests/test_guest_tty.c | 33 ++ tools/conformance_testrom/CMakeLists.txt | 43 +++ tools/conformance_testrom/README.md | 70 ++++ tools/conformance_testrom/conformance.py | 309 ++++++++++++++++ tools/conformance_testrom/fixtures.json | 28 ++ 15 files changed, 900 insertions(+), 197 deletions(-) create mode 100644 docs/CONFORMANCE_TESTROM_STATUS.md create mode 100644 runtime/include/guest_tty.h create mode 100644 runtime/src/guest_tty.c create mode 100644 runtime/tests/test_guest_tty.c create mode 100644 tools/conformance_testrom/CMakeLists.txt create mode 100644 tools/conformance_testrom/README.md create mode 100644 tools/conformance_testrom/conformance.py create mode 100644 tools/conformance_testrom/fixtures.json diff --git a/docs/CONFORMANCE_TESTROM_STATUS.md b/docs/CONFORMANCE_TESTROM_STATUS.md new file mode 100644 index 00000000..869c4263 --- /dev/null +++ b/docs/CONFORMANCE_TESTROM_STATUS.md @@ -0,0 +1,51 @@ +# External test-ROM conformance status + +Date: 2026-08-21 + +This lane turns small public PS1 hardware tests into deterministic PSXRecomp +regression fixtures. It boots the exact pinned PS-X EXE from a locally built +disc, captures guest TTY bytes through a bounded debug endpoint, and compares +the assertion log with the upstream hardware-derived result. + +## Current result + +| Fixture | Assertions | PSXRecomp result | Meaning | +|---|---:|---|---| +| `gte-test-all` | 1,150 | **1,150 passed, 0 failed** | All valid GTE opcodes and tested register semantics match the pinned oracle. | +| `cpu-cop` | 17 | **Blocked after 2 passes** | The strict AOT compiler replaces functions containing intentional invalid COP instructions with `psx_unknown_dispatch` stubs. | + +The GTE fixture initially passed zero of its 50 register tests. The oracle then +identified and verified title-neutral corrections for register sign extension, +RES1, FLAG writes, 44-bit MAC overflow, projection overflow flags, MVMVA edge +cases, lighting/color staging, and depth-cue pipeline ordering. The completed +fixture passes from a clean `prepare` -> recompile -> build -> headless run. + +The CPU result is not a harness failure. Its first two tests execute and pass; +the next intentionally invalid COP instruction reaches generated function +`0x80010144`, which is a fatal unknown-dispatch stub. A faithful interpreter or +exception-generating fallback for compiler-rejected instructions is required +before this fixture can test the remaining CPU behavior. + +## Provenance and boundaries + +- Fixture source: `JaCzekanski/ps1-tests`, MIT, release `build-158`, commit + `5f73d61b27a8d51269b54dde1c456d2d2f7dc865`. +- Exact binary/log/archive hashes are in + `tools/conformance_testrom/fixtures.json` and each private run's + `prepare-receipt.json`. +- Fixture binaries and locally owned PlayStation license-sector data are not + committed. +- The upstream expected logs are the automated oracle. A matched physical-PS1 + recapture remains the promotion gate for a hardware-qualified receipt. + +## Next factory steps + +1. Add a nonfatal compiler fallback for deliberately invalid instructions and + finish the 17-test CPU/COP fixture. +2. Have Eagle recapture the hashed synthetic GTE disc on hardware. +3. Add focused CPU, cache, DMA, timer, CD, GPU, SPU, and SIO fixtures one at a + time; each must retain exact input hashes and a machine-readable result. +4. Run this lane before whole-game validation so architectural regressions are + found below the game layer. + +Reproduction instructions are in `tools/conformance_testrom/README.md`. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 4099b660..13966f45 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -355,6 +355,12 @@ if(BUILD_TESTING) add_test(NAME emitter_directive_line_test COMMAND emitter_directive_line_test) + add_executable(guest_tty_test + ../runtime/tests/test_guest_tty.c + ../runtime/src/guest_tty.c) + target_include_directories(guest_tty_test PRIVATE ../runtime/include) + add_test(NAME guest_tty_test COMMAND guest_tty_test) + add_executable(debug_trace_ranges_test ../runtime/tests/test_debug_trace_ranges.c ../runtime/src/debug_trace_ranges.c diff --git a/runtime/include/gte.h b/runtime/include/gte.h index 2f075ba3..06d6a79f 100644 --- a/runtime/include/gte.h +++ b/runtime/include/gte.h @@ -37,6 +37,7 @@ struct GTEState { uint16_t SZ[4]; // Screen Z FIFO (unsigned 16-bit) uint32_t RGBC; // Current input color/code register uint32_t RGB[3]; // RGB color FIFO (RGB0..RGB2) + uint32_t RES1; // Undocumented read/write data register 23 // === Control Registers (matrices, vectors, config) === int16_t RT[3][3]; // Rotation matrix (s3.12) @@ -98,8 +99,8 @@ struct GTEState { void check_mac_overflow(int64_t value, int mac_num) { // mac_num: 1,2,3 - if (value > 0x7FFFFFFFFLL) FLAG |= (1u << (31 - mac_num)); // MAC1=30, MAC2=29, MAC3=28 - if (value < -0x800000000LL) FLAG |= (1u << (28 - mac_num)); // MAC1=27, MAC2=26, MAC3=25 + if (value > 0x7FFFFFFFFFFLL) FLAG |= (1u << (31 - mac_num)); // MAC1=30, MAC2=29, MAC3=28 + if (value < -0x80000000000LL) FLAG |= (1u << (28 - mac_num)); // MAC1=27, MAC2=26, MAC3=25 } void check_mac0_overflow(int64_t value) { diff --git a/runtime/include/guest_tty.h b/runtime/include/guest_tty.h new file mode 100644 index 00000000..6bcade1a --- /dev/null +++ b/runtime/include/guest_tty.h @@ -0,0 +1,23 @@ +#ifndef PSXRECOMP_GUEST_TTY_H +#define PSXRECOMP_GUEST_TTY_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/* Bounded observability sink for bytes emitted through the PS1 guest console. + * Writers never block. A snapshot returns the newest complete byte window and + * the monotonically increasing total number of bytes observed. */ +void psx_guest_tty_putchar(uint8_t value); +size_t psx_guest_tty_snapshot(uint8_t *out, size_t capacity, + uint64_t *total_bytes); +void psx_guest_tty_reset(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 5b92e4e4..12c3bc18 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -331,6 +331,7 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/psx_stick.c ${PSXRECOMP_ROOT}/runtime/src/memory.c ${PSXRECOMP_ROOT}/runtime/src/kernel_patch_ranges.c + ${PSXRECOMP_ROOT}/runtime/src/guest_tty.c ${PSXRECOMP_ROOT}/runtime/src/gpu.c ${PSXRECOMP_ROOT}/runtime/src/ws_ui_group.c ${PSXRECOMP_ROOT}/runtime/src/ws_aspect_cone_math.c diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index 430a2b3e..ebaf487e 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -46,6 +46,7 @@ #include "crash_trace.h" #include "gpu_gl_renderer.h" #include "lockstep.h" +#include "guest_tty.h" #include <stdio.h> #include <stdlib.h> @@ -3570,6 +3571,40 @@ static void handle_bioscall_dump(int id, const char *json) debug_server_send_line(out); free(out); } +/* guest_tty_dump — bounded, structured capture of bytes emitted through the + * guest console. Hex avoids JSON escaping ambiguity and preserves arbitrary + * byte values. The command is observational and never consumes the ring. */ +static void handle_guest_tty_dump(int id, const char *json) +{ + int requested = json_get_int(json, "tail", 4096); + if (requested < 0) requested = 0; + if (requested > 65536) requested = 65536; + + size_t cap = (size_t)requested; + uint8_t *bytes = cap ? (uint8_t *)malloc(cap) : NULL; + if (cap && !bytes) { send_err(id, "oom"); return; } + + uint64_t total = 0; + size_t count = psx_guest_tty_snapshot(bytes, cap, &total); + size_t out_cap = 160u + count * 2u; + char *out = (char *)malloc(out_cap); + if (!out) { free(bytes); send_err(id, "oom"); return; } + + size_t pos = (size_t)snprintf( + out, out_cap, + "{\"id\":%d,\"ok\":true,\"total\":%llu,\"tail\":%llu,\"hex\":\"", + id, (unsigned long long)total, (unsigned long long)count); + static const char digits[] = "0123456789abcdef"; + for (size_t i = 0; i < count; ++i) { + out[pos++] = digits[bytes[i] >> 4]; + out[pos++] = digits[bytes[i] & 0x0Fu]; + } + out[pos++] = '"'; out[pos++] = '}'; out[pos++] = '\n'; out[pos] = '\0'; + debug_server_send_line(out); + free(out); + free(bytes); +} + /* bios_info — which recompiled BIOS this build links, and whether the * loaded ROM matches it. Everything static comes from psx_bios_image (the * generated dispatch's self-description, couriered from the BIOS profile); @@ -13707,6 +13742,7 @@ static const CmdEntry s_commands[] = { { "fntrace_dump", handle_fntrace_dump }, { "unknown_dispatch_log", handle_unknown_dispatch_log }, { "bioscall_dump", handle_bioscall_dump }, + { "guest_tty_dump", handle_guest_tty_dump }, { "bios_info", handle_bios_info }, { "hle_dump", handle_hle_dump }, { "card_trace_dump", handle_card_trace_dump }, diff --git a/runtime/src/gte.cpp b/runtime/src/gte.cpp index 211007b9..eddf2c32 100644 --- a/runtime/src/gte.cpp +++ b/runtime/src/gte.cpp @@ -78,100 +78,101 @@ static int32_t gte_divide(uint16_t H, uint16_t SZ3, uint32_t& FLAG) { static inline bool gte_instr_lm(uint32_t instr) { return (instr & (1u << 10)) != 0; } static inline int gte_instr_sf(uint32_t instr) { return (instr & (1u << 19)) ? 12 : 0; } +static int64_t gte_mac44_stage(GTEState* gte, int64_t value, int mac_num) { + constexpr uint64_t mask = (1ull << 44) - 1ull; + constexpr uint64_t sign = 1ull << 43; + gte->check_mac_overflow(value, mac_num); + uint64_t wrapped = static_cast<uint64_t>(value) & mask; + if (wrapped & sign) wrapped |= ~mask; + return static_cast<int64_t>(wrapped); +} + +static void gte_store_mac_ir(GTEState* gte, int mac_num, int64_t value, + int shift, bool lm) { + value = gte_mac44_stage(gte, value, mac_num) >> shift; + int32_t mac = static_cast<int32_t>(value); + if (mac_num == 1) { gte->MAC1 = mac; gte->IR1 = gte->saturate_ir(mac, 1, lm); } + if (mac_num == 2) { gte->MAC2 = mac; gte->IR2 = gte->saturate_ir(mac, 2, lm); } + if (mac_num == 3) { gte->MAC3 = mac; gte->IR3 = gte->saturate_ir(mac, 3, lm); } +} + // Transform a vertex by Light matrix → IR1/IR2/IR3 (step 1 of lighting) static void light_transform(GTEState* gte, int16_t* V, uint32_t instr) { const bool lm = gte_instr_lm(instr); - int64_t mac1 = ((int64_t)gte->L[0][0] * V[0] + - (int64_t)gte->L[0][1] * V[1] + - (int64_t)gte->L[0][2] * V[2]) >> 12; - int64_t mac2 = ((int64_t)gte->L[1][0] * V[0] + - (int64_t)gte->L[1][1] * V[1] + - (int64_t)gte->L[1][2] * V[2]) >> 12; - int64_t mac3 = ((int64_t)gte->L[2][0] * V[0] + - (int64_t)gte->L[2][1] * V[1] + - (int64_t)gte->L[2][2] * V[2]) >> 12; - gte->MAC1 = static_cast<int32_t>(mac1); - gte->MAC2 = static_cast<int32_t>(mac2); - gte->MAC3 = static_cast<int32_t>(mac3); - gte->IR1 = gte->saturate_ir(gte->MAC1, 1, lm); - gte->IR2 = gte->saturate_ir(gte->MAC2, 2, lm); - gte->IR3 = gte->saturate_ir(gte->MAC3, 3, lm); + const int shift = gte_instr_sf(instr); + for (int row = 0; row < 3; ++row) { + int64_t acc = gte_mac44_stage( + gte, (int64_t)gte->L[row][0] * V[0] + + (int64_t)gte->L[row][1] * V[1], row + 1); + gte_store_mac_ir(gte, row + 1, + acc + (int64_t)gte->L[row][2] * V[2], shift, lm); + } } // Apply light color matrix + background color to IR → IR (step 2 of lighting) static void light_color(GTEState* gte, uint32_t instr) { const bool lm = gte_instr_lm(instr); - int64_t mac1 = (int64_t)gte->BK[0] * 4096 + - (int64_t)gte->LC[0][0] * gte->IR1 + - (int64_t)gte->LC[0][1] * gte->IR2 + - (int64_t)gte->LC[0][2] * gte->IR3; - int64_t mac2 = (int64_t)gte->BK[1] * 4096 + - (int64_t)gte->LC[1][0] * gte->IR1 + - (int64_t)gte->LC[1][1] * gte->IR2 + - (int64_t)gte->LC[1][2] * gte->IR3; - int64_t mac3 = (int64_t)gte->BK[2] * 4096 + - (int64_t)gte->LC[2][0] * gte->IR1 + - (int64_t)gte->LC[2][1] * gte->IR2 + - (int64_t)gte->LC[2][2] * gte->IR3; - gte->MAC1 = static_cast<int32_t>(mac1 >> 12); - gte->MAC2 = static_cast<int32_t>(mac2 >> 12); - gte->MAC3 = static_cast<int32_t>(mac3 >> 12); - gte->IR1 = gte->saturate_ir(gte->MAC1, 1, lm); - gte->IR2 = gte->saturate_ir(gte->MAC2, 2, lm); - gte->IR3 = gte->saturate_ir(gte->MAC3, 3, lm); + const int shift = gte_instr_sf(instr); + const int16_t input[3] = {gte->IR1, gte->IR2, gte->IR3}; + for (int row = 0; row < 3; ++row) { + int64_t acc = gte_mac44_stage( + gte, (int64_t)gte->BK[row] * 4096 + + (int64_t)gte->LC[row][0] * input[0], row + 1); + acc = gte_mac44_stage( + gte, acc + (int64_t)gte->LC[row][1] * input[1], row + 1); + gte_store_mac_ir(gte, row + 1, + acc + (int64_t)gte->LC[row][2] * input[2], shift, lm); + } } -// Multiply IR by RGBC color and push to RGB FIFO (step 3 of color output) -static void color_output(GTEState* gte, uint32_t instr) { - const bool lm = gte_instr_lm(instr); - uint8_t r0 = (gte->RGBC >> 0) & 0xFF; - uint8_t g0 = (gte->RGBC >> 8) & 0xFF; - uint8_t b0 = (gte->RGBC >> 16) & 0xFF; - int64_t mac1 = (int64_t)r0 * gte->IR1 * 16; - int64_t mac2 = (int64_t)g0 * gte->IR2 * 16; - int64_t mac3 = (int64_t)b0 * gte->IR3 * 16; - gte->MAC1 = static_cast<int32_t>(mac1 >> 12); - gte->MAC2 = static_cast<int32_t>(mac2 >> 12); - gte->MAC3 = static_cast<int32_t>(mac3 >> 12); - gte->IR1 = gte->saturate_ir(gte->MAC1, 1, lm); - gte->IR2 = gte->saturate_ir(gte->MAC2, 2, lm); - gte->IR3 = gte->saturate_ir(gte->MAC3, 3, lm); +static void push_rgb_from_mac(GTEState* gte) { gte->push_rgb( gte->saturate_color(gte->MAC1 >> 4, 0), gte->saturate_color(gte->MAC2 >> 4, 1), gte->saturate_color(gte->MAC3 >> 4, 2)); } -// Depth cue / interpolate current IR toward the far color using IR0 -// (common tail of DPCS/DPCT/DPCL/INTPL/NCDS/NCDT/CDP). Hardware: -// base = IR << 12 -// step = lim(((FC << 12) - base) >> (sf*12)) ; ±0x8000 clamp, lm FORCED off -// MAC = (base + IR0 * step) >> (sf*12) -// IR = lim(MAC, lm) ; lm from the instruction -static void depth_cue_from_ir(GTEState* gte, uint32_t instr) { +// Multiply IR by RGBC color (step 3 of NCCS/CC color output). +static void color_multiply(GTEState* gte, uint32_t instr) { + const bool lm = gte_instr_lm(instr); + const int shift = gte_instr_sf(instr); + uint8_t r0 = (gte->RGBC >> 0) & 0xFF; + uint8_t g0 = (gte->RGBC >> 8) & 0xFF; + uint8_t b0 = (gte->RGBC >> 16) & 0xFF; + const int16_t input[3] = {gte->IR1, gte->IR2, gte->IR3}; + gte_store_mac_ir(gte, 1, (int64_t)r0 * input[0] * 16, shift, lm); + gte_store_mac_ir(gte, 2, (int64_t)g0 * input[1] * 16, shift, lm); + gte_store_mac_ir(gte, 3, (int64_t)b0 * input[2] * 16, shift, lm); +} + +static void color_output(GTEState* gte, uint32_t instr) { + color_multiply(gte, instr); + push_rgb_from_mac(gte); +} + +// Interpolate an unshifted MAC input toward far color using IR0. The first +// stage always uses lm=0; the command's lm applies only to the final result. +static void depth_cue_from_mac(GTEState* gte, int64_t base1, int64_t base2, + int64_t base3, uint32_t instr) { const int shift = gte_instr_sf(instr); const bool lm = gte_instr_lm(instr); - int64_t base1 = (int64_t)gte->IR1 * 4096; - int64_t base2 = (int64_t)gte->IR2 * 4096; - int64_t base3 = (int64_t)gte->IR3 * 4096; - int16_t step1 = gte->saturate_ir( - (int32_t)((((int64_t)gte->FC[0] * 4096 - base1) >> shift)), 1, false); - int16_t step2 = gte->saturate_ir( - (int32_t)((((int64_t)gte->FC[1] * 4096 - base2) >> shift)), 2, false); - int16_t step3 = gte->saturate_ir( - (int32_t)((((int64_t)gte->FC[2] * 4096 - base3) >> shift)), 3, false); - int64_t mac1 = (base1 + (int64_t)gte->IR0 * step1) >> shift; - int64_t mac2 = (base2 + (int64_t)gte->IR0 * step2) >> shift; - int64_t mac3 = (base3 + (int64_t)gte->IR0 * step3) >> shift; - gte->check_mac_overflow(mac1, 1); - gte->check_mac_overflow(mac2, 2); - gte->check_mac_overflow(mac3, 3); - gte->MAC1 = static_cast<int32_t>(mac1); - gte->MAC2 = static_cast<int32_t>(mac2); - gte->MAC3 = static_cast<int32_t>(mac3); - gte->IR1 = gte->saturate_ir(gte->MAC1, 1, lm); - gte->IR2 = gte->saturate_ir(gte->MAC2, 2, lm); - gte->IR3 = gte->saturate_ir(gte->MAC3, 3, lm); + gte_store_mac_ir(gte, 1, (int64_t)gte->FC[0] * 4096 - base1, shift, false); + gte_store_mac_ir(gte, 2, (int64_t)gte->FC[1] * 4096 - base2, shift, false); + gte_store_mac_ir(gte, 3, (int64_t)gte->FC[2] * 4096 - base3, shift, false); + const int16_t step1 = gte->IR1; + const int16_t step2 = gte->IR2; + const int16_t step3 = gte->IR3; + gte_store_mac_ir(gte, 1, base1 + (int64_t)gte->IR0 * step1, shift, lm); + gte_store_mac_ir(gte, 2, base2 + (int64_t)gte->IR0 * step2, shift, lm); + gte_store_mac_ir(gte, 3, base3 + (int64_t)gte->IR0 * step3, shift, lm); +} + +static void depth_cue_from_ir(GTEState* gte, uint32_t instr) { + depth_cue_from_mac(gte, + (int64_t)gte->IR1 * 4096, + (int64_t)gte->IR2 * 4096, + (int64_t)gte->IR3 * 4096, + instr); } // --------------------------------------------------------------------------- @@ -805,23 +806,22 @@ void gte_rtps_internal(GTEState* gte, int16_t* V, bool setMac0, uint32_t instr) const bool lm = gte_instr_lm(instr); // Step 1: Matrix multiplication + translation - int64_t mac1 = (int64_t)gte->TR[0] * 4096 + - (int64_t)gte->RT[0][0] * V[0] + - (int64_t)gte->RT[0][1] * V[1] + - (int64_t)gte->RT[0][2] * V[2]; - int64_t mac2 = (int64_t)gte->TR[1] * 4096 + - (int64_t)gte->RT[1][0] * V[0] + - (int64_t)gte->RT[1][1] * V[1] + - (int64_t)gte->RT[1][2] * V[2]; - int64_t mac3 = (int64_t)gte->TR[2] * 4096 + - (int64_t)gte->RT[2][0] * V[0] + - (int64_t)gte->RT[2][1] * V[1] + - (int64_t)gte->RT[2][2] * V[2]; + auto dot = [&](int row, int mac_num) { + int64_t acc = (int64_t)gte->TR[row] * 4096 + + (int64_t)gte->RT[row][0] * V[0]; + acc = gte_mac44_stage(gte, acc, mac_num); + acc += (int64_t)gte->RT[row][1] * V[1]; + acc = gte_mac44_stage(gte, acc, mac_num); + return acc + (int64_t)gte->RT[row][2] * V[2]; + }; + int64_t mac1 = dot(0, 1); + int64_t mac2 = dot(1, 2); + int64_t mac3 = dot(2, 3); - // Overflow flags always check the >>12 view (hardware). - gte->check_mac_overflow(mac1 >> 12, 1); - gte->check_mac_overflow(mac2 >> 12, 2); - gte->check_mac_overflow(mac3 >> 12, 3); + // MAC1..3 overflow flags observe the unshifted 44-bit accumulator. + gte->check_mac_overflow(mac1, 1); + gte->check_mac_overflow(mac2, 2); + gte->check_mac_overflow(mac3, 3); gte->MAC1 = static_cast<int32_t>(mac1 >> shift); gte->MAC2 = static_cast<int32_t>(mac2 >> shift); gte->MAC3 = static_cast<int32_t>(mac3 >> shift); @@ -888,6 +888,10 @@ void gte_rtps_internal(GTEState* gte, int16_t* V, bool setMac0, uint32_t instr) dome_probe_note(gte->SZ[3]); /* locate the dome draw fn (far-vertex tally) */ int64_t sx16 = gte->OFX + xterm; int64_t sy16 = gte->OFY + (int64_t)gte->IR2 * h_div_sz; + /* RTPS/RTPT reuse MAC0's overflow flags for each projected X/Y + * accumulator before MAC0 is replaced by the depth-cue result. */ + gte->check_mac0_overflow(sx16); + gte->check_mac0_overflow(sy16); int64_t sx = sx16 >> 16; int64_t sy = sy16 >> 16; gte->push_sxy(sx, sy); @@ -1055,8 +1059,15 @@ void gte_ncct(GTEState* gte, uint32_t instr) { void gte_ncds_internal(GTEState* gte, int16_t* V, uint32_t instr) { light_transform(gte, V, instr); light_color(gte, instr); - depth_cue_from_ir(gte, instr); - color_output(gte, instr); + const uint8_t r = (gte->RGBC >> 0) & 0xFF; + const uint8_t g = (gte->RGBC >> 8) & 0xFF; + const uint8_t b = (gte->RGBC >> 16) & 0xFF; + depth_cue_from_mac(gte, + (int64_t)r * gte->IR1 * 16, + (int64_t)g * gte->IR2 * 16, + (int64_t)b * gte->IR3 * 16, + instr); + push_rgb_from_mac(gte); } void gte_ncds(GTEState* gte, uint32_t instr) { @@ -1114,17 +1125,9 @@ void gte_dpcs(GTEState* gte, uint32_t instr) { uint8_t r = (gte->RGBC >> 0) & 0xFF; uint8_t g = (gte->RGBC >> 8) & 0xFF; uint8_t b = (gte->RGBC >> 16) & 0xFF; - // MAC = RGBC << 4 - gte->IR1 = gte->saturate_ir(r << 4, 1, false); - gte->IR2 = gte->saturate_ir(g << 4, 2, false); - gte->IR3 = gte->saturate_ir(b << 4, 3, false); - // Interpolate toward far color using IR0 - depth_cue_from_ir(gte, instr); - // Output - gte->push_rgb( - gte->saturate_color(gte->MAC1 >> 4, 0), - gte->saturate_color(gte->MAC2 >> 4, 1), - gte->saturate_color(gte->MAC3 >> 4, 2)); + depth_cue_from_mac(gte, (int64_t)r << 16, (int64_t)g << 16, + (int64_t)b << 16, instr); + push_rgb_from_mac(gte); gte->set_error_flag(); } @@ -1137,14 +1140,9 @@ void gte_dpct(GTEState* gte, uint32_t instr) { uint8_t r = (gte->RGB[0] >> 0) & 0xFF; uint8_t g = (gte->RGB[0] >> 8) & 0xFF; uint8_t b = (gte->RGB[0] >> 16) & 0xFF; - gte->IR1 = gte->saturate_ir(r << 4, 1, false); - gte->IR2 = gte->saturate_ir(g << 4, 2, false); - gte->IR3 = gte->saturate_ir(b << 4, 3, false); - depth_cue_from_ir(gte, instr); - gte->push_rgb( - gte->saturate_color(gte->MAC1 >> 4, 0), - gte->saturate_color(gte->MAC2 >> 4, 1), - gte->saturate_color(gte->MAC3 >> 4, 2)); + depth_cue_from_mac(gte, (int64_t)r << 16, (int64_t)g << 16, + (int64_t)b << 16, instr); + push_rgb_from_mac(gte); } gte->set_error_flag(); } @@ -1158,18 +1156,12 @@ void gte_dpcl(GTEState* gte, uint32_t instr) { uint8_t r = (gte->RGBC >> 0) & 0xFF; uint8_t g = (gte->RGBC >> 8) & 0xFF; uint8_t b = (gte->RGBC >> 16) & 0xFF; - gte->MAC1 = (r * gte->IR1) >> 8; - gte->MAC2 = (g * gte->IR2) >> 8; - gte->MAC3 = (b * gte->IR3) >> 8; - gte->IR1 = gte->saturate_ir(gte->MAC1, 1, gte_instr_lm(instr)); - gte->IR2 = gte->saturate_ir(gte->MAC2, 2, gte_instr_lm(instr)); - gte->IR3 = gte->saturate_ir(gte->MAC3, 3, gte_instr_lm(instr)); - // Depth cue toward far color - depth_cue_from_ir(gte, instr); - gte->push_rgb( - gte->saturate_color(gte->MAC1 >> 4, 0), - gte->saturate_color(gte->MAC2 >> 4, 1), - gte->saturate_color(gte->MAC3 >> 4, 2)); + depth_cue_from_mac(gte, + (int64_t)r * gte->IR1 * 16, + (int64_t)g * gte->IR2 * 16, + (int64_t)b * gte->IR3 * 16, + instr); + push_rgb_from_mac(gte); gte->set_error_flag(); } @@ -1179,10 +1171,7 @@ void gte_dpcl(GTEState* gte, uint32_t instr) { void gte_intpl(GTEState* gte, uint32_t instr) { gte->FLAG = 0; depth_cue_from_ir(gte, instr); - gte->push_rgb( - gte->saturate_color(gte->MAC1 >> 4, 0), - gte->saturate_color(gte->MAC2 >> 4, 1), - gte->saturate_color(gte->MAC3 >> 4, 2)); + push_rgb_from_mac(gte); gte->set_error_flag(); } @@ -1193,8 +1182,15 @@ void gte_cdp(GTEState* gte, uint32_t instr) { gte->FLAG = 0; // IR1/IR2/IR3 already set (from previous NCS or similar) light_color(gte, instr); - depth_cue_from_ir(gte, instr); - color_output(gte, instr); + const uint8_t r = (gte->RGBC >> 0) & 0xFF; + const uint8_t g = (gte->RGBC >> 8) & 0xFF; + const uint8_t b = (gte->RGBC >> 16) & 0xFF; + depth_cue_from_mac(gte, + (int64_t)r * gte->IR1 * 16, + (int64_t)g * gte->IR2 * 16, + (int64_t)b * gte->IR3 * 16, + instr); + push_rgb_from_mac(gte); gte->set_error_flag(); } @@ -1217,7 +1213,8 @@ void gte_mvmva(GTEState* gte, uint32_t instr) { uint32_t mx = (instr >> 17) & 3; // Matrix: 0=RT, 1=Light, 2=LightColor, 3=reserved uint32_t vv = (instr >> 15) & 3; // Vector: 0=V0, 1=V1, 2=V2, 3=IR uint32_t tv = (instr >> 13) & 3; // Translation: 0=TR, 1=BK, 2=FC/bugged, 3=none - int sf = (instr >> 19) & 1; // Shift: 0=no shift, 1=shift right 12 + const int shift = gte_instr_sf(instr); + const bool lm = gte_instr_lm(instr); // Select matrix int16_t M[3][3]; @@ -1225,7 +1222,18 @@ void gte_mvmva(GTEState* gte, uint32_t instr) { case 0: std::memcpy(M, gte->RT, sizeof(M)); break; case 1: std::memcpy(M, gte->L, sizeof(M)); break; case 2: std::memcpy(M, gte->LC, sizeof(M)); break; - default: std::memset(M, 0, sizeof(M)); break; // Garbage on real HW + default: { + /* Undocumented matrix selector wiring, verified by hardware + * register vectors (also used by Beetle/DuckStation). */ + const int16_t color = static_cast<int16_t>( + static_cast<uint16_t>(gte->RGBC & 0xFFu) << 4); + M[0][0] = static_cast<int16_t>(-color); + M[0][1] = color; + M[0][2] = gte->IR0; + M[1][0] = M[1][1] = M[1][2] = gte->RT[0][2]; + M[2][0] = M[2][1] = M[2][2] = gte->RT[1][1]; + break; + } } // Select vector @@ -1246,28 +1254,39 @@ void gte_mvmva(GTEState* gte, uint32_t instr) { case 3: T[0] = 0; T[1] = 0; T[2] = 0; break; } - // Multiply - int64_t mac1 = T[0] + (int64_t)M[0][0] * V[0] + (int64_t)M[0][1] * V[1] + (int64_t)M[0][2] * V[2]; - int64_t mac2 = T[1] + (int64_t)M[1][0] * V[0] + (int64_t)M[1][1] * V[1] + (int64_t)M[1][2] * V[2]; - int64_t mac3 = T[2] + (int64_t)M[2][0] * V[0] + (int64_t)M[2][1] * V[1] + (int64_t)M[2][2] * V[2]; + auto set_mac_ir = [&](int row, int64_t value) { + const int mac_num = row + 1; + value = gte_mac44_stage(gte, value, mac_num) >> shift; + int32_t mac = static_cast<int32_t>(value); + if (row == 0) { gte->MAC1 = mac; gte->IR1 = gte->saturate_ir(mac, 1, lm); } + if (row == 1) { gte->MAC2 = mac; gte->IR2 = gte->saturate_ir(mac, 2, lm); } + if (row == 2) { gte->MAC3 = mac; gte->IR3 = gte->saturate_ir(mac, 3, lm); } + }; - if (sf) { - mac1 >>= 12; mac2 >>= 12; mac3 >>= 12; + for (int row = 0; row < 3; ++row) { + const int mac_num = row + 1; + if (tv == 2) { + /* Hardware's FC translation selector is wired incorrectly: its + * translation+X term only feeds a temporary IR saturation, then + * MAC/IR are replaced by the Y/Z dot product. */ + int64_t first = gte_mac44_stage( + gte, T[row] + (int64_t)M[row][0] * V[0], mac_num); + int32_t temporary = static_cast<int32_t>(first >> shift); + if (row == 0) gte->IR1 = gte->saturate_ir(temporary, 1, false); + if (row == 1) gte->IR2 = gte->saturate_ir(temporary, 2, false); + if (row == 2) gte->IR3 = gte->saturate_ir(temporary, 3, false); + int64_t tail = gte_mac44_stage( + gte, (int64_t)M[row][1] * V[1], mac_num); + set_mac_ir(row, tail + (int64_t)M[row][2] * V[2]); + } else { + int64_t acc = gte_mac44_stage( + gte, T[row] + (int64_t)M[row][0] * V[0], mac_num); + acc = gte_mac44_stage( + gte, acc + (int64_t)M[row][1] * V[1], mac_num); + set_mac_ir(row, acc + (int64_t)M[row][2] * V[2]); + } } - gte->check_mac_overflow(mac1, 1); - gte->check_mac_overflow(mac2, 2); - gte->check_mac_overflow(mac3, 3); - - gte->MAC1 = static_cast<int32_t>(mac1); - gte->MAC2 = static_cast<int32_t>(mac2); - gte->MAC3 = static_cast<int32_t>(mac3); - - bool lm = (instr >> 10) & 1; - gte->IR1 = gte->saturate_ir(gte->MAC1, 1, lm); - gte->IR2 = gte->saturate_ir(gte->MAC2, 2, lm); - gte->IR3 = gte->saturate_ir(gte->MAC3, 3, lm); - gte->set_error_flag(); } @@ -1415,7 +1434,7 @@ void gte_mtc2(GTEState* gte, uint8_t reg, uint32_t value) { case 20: gte->RGB[0] = value; break; case 21: gte->RGB[1] = value; break; case 22: gte->RGB[2] = value; break; - case 23: break; // RES1 reserved + case 23: gte->RES1 = value; break; case 24: gte->MAC0 = value; break; case 25: gte->MAC1 = value; break; case 26: gte->MAC2 = value; break; @@ -1448,11 +1467,11 @@ void gte_mtc2(GTEState* gte, uint8_t reg, uint32_t value) { uint32_t gte_mfc2(GTEState* gte, uint8_t reg) { switch (reg) { case 0: return (static_cast<uint16_t>(gte->V0[1]) << 16) | static_cast<uint16_t>(gte->V0[0]); - case 1: return static_cast<uint16_t>(gte->V0[2]); + case 1: return static_cast<uint32_t>(static_cast<int32_t>(gte->V0[2])); case 2: return (static_cast<uint16_t>(gte->V1[1]) << 16) | static_cast<uint16_t>(gte->V1[0]); - case 3: return static_cast<uint16_t>(gte->V1[2]); + case 3: return static_cast<uint32_t>(static_cast<int32_t>(gte->V1[2])); case 4: return (static_cast<uint16_t>(gte->V2[1]) << 16) | static_cast<uint16_t>(gte->V2[0]); - case 5: return static_cast<uint16_t>(gte->V2[2]); + case 5: return static_cast<uint32_t>(static_cast<int32_t>(gte->V2[2])); case 6: return gte->RGBC; case 7: return gte->OTZ; case 8: return static_cast<int32_t>(gte->IR0); // sign-extend @@ -1470,7 +1489,7 @@ uint32_t gte_mfc2(GTEState* gte, uint8_t reg) { case 20: return gte->RGB[0]; case 21: return gte->RGB[1]; case 22: return gte->RGB[2]; - case 23: return 0; // RES1 + case 23: return gte->RES1; case 24: return gte->MAC0; case 25: return gte->MAC1; case 26: return gte->MAC2; @@ -1523,7 +1542,10 @@ void gte_ctc2(GTEState* gte, uint8_t reg, uint32_t value) { case 28: gte->DQB = value; break; case 29: gte->ZSF3 = static_cast<int16_t>(value & 0xFFFF); break; case 30: gte->ZSF4 = static_cast<int16_t>(value & 0xFFFF); break; - case 31: gte->FLAG = value & 0x7FFFF000u; break; + case 31: + gte->FLAG = value & 0x7FFFF000u; + gte->set_error_flag(); + break; default: break; } } @@ -1537,7 +1559,7 @@ uint32_t gte_cfc2(GTEState* gte, uint8_t reg) { case 1: return (static_cast<uint16_t>(gte->RT[1][0]) << 16) | static_cast<uint16_t>(gte->RT[0][2]); case 2: return (static_cast<uint16_t>(gte->RT[1][2]) << 16) | static_cast<uint16_t>(gte->RT[1][1]); case 3: return (static_cast<uint16_t>(gte->RT[2][1]) << 16) | static_cast<uint16_t>(gte->RT[2][0]); - case 4: return static_cast<uint16_t>(gte->RT[2][2]); + case 4: return static_cast<uint32_t>(static_cast<int32_t>(gte->RT[2][2])); case 5: return gte->TR[0]; case 6: return gte->TR[1]; case 7: return gte->TR[2]; @@ -1545,7 +1567,7 @@ uint32_t gte_cfc2(GTEState* gte, uint8_t reg) { case 9: return (static_cast<uint16_t>(gte->L[1][0]) << 16) | static_cast<uint16_t>(gte->L[0][2]); case 10: return (static_cast<uint16_t>(gte->L[1][2]) << 16) | static_cast<uint16_t>(gte->L[1][1]); case 11: return (static_cast<uint16_t>(gte->L[2][1]) << 16) | static_cast<uint16_t>(gte->L[2][0]); - case 12: return static_cast<uint16_t>(gte->L[2][2]); + case 12: return static_cast<uint32_t>(static_cast<int32_t>(gte->L[2][2])); case 13: return gte->BK[0]; case 14: return gte->BK[1]; case 15: return gte->BK[2]; @@ -1553,13 +1575,14 @@ uint32_t gte_cfc2(GTEState* gte, uint8_t reg) { case 17: return (static_cast<uint16_t>(gte->LC[1][0]) << 16) | static_cast<uint16_t>(gte->LC[0][2]); case 18: return (static_cast<uint16_t>(gte->LC[1][2]) << 16) | static_cast<uint16_t>(gte->LC[1][1]); case 19: return (static_cast<uint16_t>(gte->LC[2][1]) << 16) | static_cast<uint16_t>(gte->LC[2][0]); - case 20: return static_cast<uint16_t>(gte->LC[2][2]); + case 20: return static_cast<uint32_t>(static_cast<int32_t>(gte->LC[2][2])); case 21: return gte->FC[0]; case 22: return gte->FC[1]; case 23: return gte->FC[2]; case 24: return gte->OFX; case 25: return gte->OFY; - case 26: return gte->H; + case 26: return static_cast<uint32_t>(static_cast<int32_t>( + static_cast<int16_t>(gte->H))); case 27: return static_cast<int32_t>(gte->DQA); // sign-extend case 28: return gte->DQB; case 29: return static_cast<int32_t>(gte->ZSF3); @@ -1634,6 +1657,7 @@ static void gte_import_cpu_state(PSXRecomp::GTE::GTEState* gte, gte->SXY[3] = static_cast<int32_t>(d[14]); for (int i = 0; i < 4; ++i) gte->SZ[i] = static_cast<uint16_t>(d[16 + i]); for (int i = 0; i < 3; ++i) gte->RGB[i] = d[20 + i]; + gte->RES1 = d[23]; gte->MAC0 = static_cast<int32_t>(d[24]); gte->MAC1 = static_cast<int32_t>(d[25]); gte->MAC2 = static_cast<int32_t>(d[26]); @@ -1700,11 +1724,11 @@ static void gte_export_cpu_state(CPUState* cpu, uint32_t* c = cpu->gte_ctrl; d[0] = gte_pack_s16_pair(gte->V0[0], gte->V0[1]); - d[1] = static_cast<uint16_t>(gte->V0[2]); + d[1] = static_cast<uint32_t>(static_cast<int32_t>(gte->V0[2])); d[2] = gte_pack_s16_pair(gte->V1[0], gte->V1[1]); - d[3] = static_cast<uint16_t>(gte->V1[2]); + d[3] = static_cast<uint32_t>(static_cast<int32_t>(gte->V1[2])); d[4] = gte_pack_s16_pair(gte->V2[0], gte->V2[1]); - d[5] = static_cast<uint16_t>(gte->V2[2]); + d[5] = static_cast<uint32_t>(static_cast<int32_t>(gte->V2[2])); d[6] = gte->RGBC; d[7] = gte->OTZ; d[8] = static_cast<uint32_t>(static_cast<int32_t>(gte->IR0)); @@ -1717,7 +1741,7 @@ static void gte_export_cpu_state(CPUState* cpu, d[15] = static_cast<uint32_t>(gte->SXY[3]); for (int i = 0; i < 4; ++i) d[16 + i] = gte->SZ[i]; for (int i = 0; i < 3; ++i) d[20 + i] = gte->RGB[i]; - d[23] = 0; + d[23] = gte->RES1; d[24] = static_cast<uint32_t>(gte->MAC0); d[25] = static_cast<uint32_t>(gte->MAC1); d[26] = static_cast<uint32_t>(gte->MAC2); @@ -1730,23 +1754,24 @@ static void gte_export_cpu_state(CPUState* cpu, c[1] = gte_pack_s16_pair(gte->RT[0][2], gte->RT[1][0]); c[2] = gte_pack_s16_pair(gte->RT[1][1], gte->RT[1][2]); c[3] = gte_pack_s16_pair(gte->RT[2][0], gte->RT[2][1]); - c[4] = static_cast<uint16_t>(gte->RT[2][2]); + c[4] = static_cast<uint32_t>(static_cast<int32_t>(gte->RT[2][2])); for (int i = 0; i < 3; ++i) c[5 + i] = static_cast<uint32_t>(gte->TR[i]); c[8] = gte_pack_s16_pair(gte->L[0][0], gte->L[0][1]); c[9] = gte_pack_s16_pair(gte->L[0][2], gte->L[1][0]); c[10] = gte_pack_s16_pair(gte->L[1][1], gte->L[1][2]); c[11] = gte_pack_s16_pair(gte->L[2][0], gte->L[2][1]); - c[12] = static_cast<uint16_t>(gte->L[2][2]); + c[12] = static_cast<uint32_t>(static_cast<int32_t>(gte->L[2][2])); for (int i = 0; i < 3; ++i) c[13 + i] = static_cast<uint32_t>(gte->BK[i]); c[16] = gte_pack_s16_pair(gte->LC[0][0], gte->LC[0][1]); c[17] = gte_pack_s16_pair(gte->LC[0][2], gte->LC[1][0]); c[18] = gte_pack_s16_pair(gte->LC[1][1], gte->LC[1][2]); c[19] = gte_pack_s16_pair(gte->LC[2][0], gte->LC[2][1]); - c[20] = static_cast<uint16_t>(gte->LC[2][2]); + c[20] = static_cast<uint32_t>(static_cast<int32_t>(gte->LC[2][2])); for (int i = 0; i < 3; ++i) c[21 + i] = static_cast<uint32_t>(gte->FC[i]); c[24] = static_cast<uint32_t>(gte->OFX); c[25] = static_cast<uint32_t>(gte->OFY); - c[26] = gte->H; + c[26] = static_cast<uint32_t>(static_cast<int32_t>( + static_cast<int16_t>(gte->H))); c[27] = static_cast<uint32_t>(static_cast<int32_t>(gte->DQA)); c[28] = static_cast<uint32_t>(gte->DQB); c[29] = static_cast<uint32_t>(static_cast<int32_t>(gte->ZSF3)); @@ -1982,20 +2007,17 @@ static uint32_t gte_sign_extend_16(uint32_t value) { * invalidate host precision caches: it is also the compatibility boundary for * committed AOT C emitted before all masked/aliased writes used helpers. */ static void gte_cpu_canonicalize_backing(CPUState* cpu) { - static const uint8_t data_u16[] = {1, 3, 5, 7, 16, 17, 18, 19}; - static const uint8_t data_s16[] = {8, 9, 10, 11}; - static const uint8_t ctrl_u16[] = {4, 12, 20, 26}; - static const uint8_t ctrl_s16[] = {27, 29, 30}; + static const uint8_t data_u16[] = {7, 16, 17, 18, 19}; + static const uint8_t data_s16[] = {1, 3, 5, 8, 9, 10, 11}; + static const uint8_t ctrl_s16[] = {4, 12, 20, 26, 27, 29, 30}; for (uint8_t reg : data_u16) cpu->gte_data[reg] &= 0xFFFFu; for (uint8_t reg : data_s16) cpu->gte_data[reg] = gte_sign_extend_16(cpu->gte_data[reg]); cpu->gte_data[15] = cpu->gte_data[14]; - cpu->gte_data[23] = 0; cpu->gte_data[28] = cpu->gte_data[29] = gte_cpu_pack_irgb(cpu); cpu->gte_data[31] = gte_cpu_lzcr(cpu->gte_data[30]); - for (uint8_t reg : ctrl_u16) cpu->gte_ctrl[reg] &= 0xFFFFu; for (uint8_t reg : ctrl_s16) cpu->gte_ctrl[reg] = gte_sign_extend_16(cpu->gte_ctrl[reg]); } @@ -2003,15 +2025,12 @@ static void gte_cpu_canonicalize_backing(CPUState* cpu) { extern "C" uint32_t gte_read_data(CPUState* cpu, uint8_t reg) { if (reg >= 32) return 0; switch (reg) { - case 1: case 3: case 5: case 7: - case 16: case 17: case 18: case 19: + case 7: case 16: case 17: case 18: case 19: return cpu->gte_data[reg] & 0xFFFFu; - case 8: case 9: case 10: case 11: + case 1: case 3: case 5: case 8: case 9: case 10: case 11: return gte_sign_extend_16(cpu->gte_data[reg]); case 15: return cpu->gte_data[14]; - case 23: - return 0; case 28: case 29: return gte_cpu_pack_irgb(cpu); case 31: @@ -2024,9 +2043,7 @@ extern "C" uint32_t gte_read_data(CPUState* cpu, uint8_t reg) { extern "C" uint32_t gte_read_ctrl(CPUState* cpu, uint8_t reg) { if (reg >= 32) return 0; switch (reg) { - case 4: case 12: case 20: case 26: - return cpu->gte_ctrl[reg] & 0xFFFFu; - case 27: case 29: case 30: + case 4: case 12: case 20: case 26: case 27: case 29: case 30: return gte_sign_extend_16(cpu->gte_ctrl[reg]); default: return cpu->gte_ctrl[reg]; @@ -2037,11 +2054,10 @@ extern "C" void gte_write_data(CPUState* cpu, uint8_t reg, uint32_t val) { if (reg >= 32) return; gte_cpu_canonicalize_backing(cpu); switch (reg) { - case 1: case 3: case 5: case 7: - case 16: case 17: case 18: case 19: + case 7: case 16: case 17: case 18: case 19: cpu->gte_data[reg] = val & 0xFFFFu; break; - case 8: case 9: case 10: case 11: + case 1: case 3: case 5: case 8: case 9: case 10: case 11: cpu->gte_data[reg] = gte_sign_extend_16(val); if (reg >= 9) { const uint32_t packed = gte_cpu_pack_irgb(cpu); @@ -2076,9 +2092,6 @@ extern "C" void gte_write_data(CPUState* cpu, uint8_t reg, uint32_t val) { pgxp_gte_reg_written(15, val); } break; - case 23: - cpu->gte_data[23] = 0; - break; case 28: { cpu->gte_data[9] = (val & 0x1Fu) << 7; cpu->gte_data[10] = ((val >> 5) & 0x1Fu) << 7; @@ -2111,14 +2124,13 @@ extern "C" void gte_write_ctrl(CPUState* cpu, uint8_t reg, uint32_t val) { if (reg >= 32) return; gte_cpu_canonicalize_backing(cpu); switch (reg) { - case 4: case 12: case 20: case 26: - cpu->gte_ctrl[reg] = val & 0xFFFFu; - break; - case 27: case 29: case 30: + case 4: case 12: case 20: case 26: case 27: case 29: case 30: cpu->gte_ctrl[reg] = gte_sign_extend_16(val); break; case 31: cpu->gte_ctrl[31] = val & 0x7FFFF000u; + if (cpu->gte_ctrl[31] & PSXRecomp::GTE::FLAG_ERROR_MASK) + cpu->gte_ctrl[31] |= PSXRecomp::GTE::FLAG_ERROR_BIT; break; default: cpu->gte_ctrl[reg] = val; diff --git a/runtime/src/guest_tty.c b/runtime/src/guest_tty.c new file mode 100644 index 00000000..87375ca0 --- /dev/null +++ b/runtime/src/guest_tty.c @@ -0,0 +1,60 @@ +#include "guest_tty.h" + +#include <stdatomic.h> +#include <stdio.h> + +#define GUEST_TTY_RING_CAP 65536u +#define GUEST_TTY_RING_MASK (GUEST_TTY_RING_CAP - 1u) + +static _Atomic unsigned char s_guest_tty_ring[GUEST_TTY_RING_CAP]; +static _Atomic uint64_t s_guest_tty_total; +static char s_guest_tty_line[257]; +static unsigned s_guest_tty_line_len; + +void psx_guest_tty_putchar(uint8_t value) +{ + uint64_t seq = atomic_load_explicit(&s_guest_tty_total, + memory_order_relaxed); + atomic_store_explicit(&s_guest_tty_ring[seq & GUEST_TTY_RING_MASK], value, + memory_order_relaxed); + atomic_store_explicit(&s_guest_tty_total, seq + 1u, memory_order_release); + + /* Preserve the existing human-readable runtime log alongside the + * structured ring used by automation. */ + if (value == '\n' || s_guest_tty_line_len >= 256u) { + s_guest_tty_line[s_guest_tty_line_len] = '\0'; + fprintf(stderr, "tty: %s\n", s_guest_tty_line); + s_guest_tty_line_len = 0; + if (value != '\n' && value != '\r') + s_guest_tty_line[s_guest_tty_line_len++] = (char)value; + } else if (value != '\r') { + s_guest_tty_line[s_guest_tty_line_len++] = (char)value; + } +} + +size_t psx_guest_tty_snapshot(uint8_t *out, size_t capacity, + uint64_t *total_bytes) +{ + uint64_t total = atomic_load_explicit(&s_guest_tty_total, + memory_order_acquire); + size_t available = (total < GUEST_TTY_RING_CAP) + ? (size_t)total : (size_t)GUEST_TTY_RING_CAP; + size_t count = (capacity < available) ? capacity : available; + uint64_t start = total - count; + + if (total_bytes) *total_bytes = total; + if (!out) return count; + + for (size_t i = 0; i < count; ++i) { + out[i] = atomic_load_explicit( + &s_guest_tty_ring[(start + i) & GUEST_TTY_RING_MASK], + memory_order_relaxed); + } + return count; +} + +void psx_guest_tty_reset(void) +{ + atomic_store_explicit(&s_guest_tty_total, 0u, memory_order_release); + s_guest_tty_line_len = 0; +} diff --git a/runtime/src/memory.c b/runtime/src/memory.c index 9124ebb1..cb1c70a8 100644 --- a/runtime/src/memory.c +++ b/runtime/src/memory.c @@ -23,6 +23,7 @@ #include "lockstep.h" #include "data_shards.h" #include "dirty_ram_interp.h" +#include "guest_tty.h" #include "psx_cycles.h" #include "starvation_ring.h" #include <stdint.h> diff --git a/runtime/tests/test_gte_register_access.cpp b/runtime/tests/test_gte_register_access.cpp index 872d9a67..8fd223de 100644 --- a/runtime/tests/test_gte_register_access.cpp +++ b/runtime/tests/test_gte_register_access.cpp @@ -275,6 +275,35 @@ int test_reads() { return 0; } +int test_hardware_register_semantics() { + CPUState cpu{}; + gte_write_data(&cpu, 1, 0x00008001u); + if (gte_read_data(&cpu, 1) != 0xFFFF8001u) + return fail_value("VZ sign extension", 0, 1, 0x00008001u, + 0xFFFF8001u, gte_read_data(&cpu, 1)); + + gte_write_data(&cpu, 23, 0xDEADBEEFu); + if (gte_read_data(&cpu, 23) != 0xDEADBEEFu) + return fail_value("RES1 round trip", 0, 23, 0xDEADBEEFu, + 0xDEADBEEFu, gte_read_data(&cpu, 23)); + + gte_write_ctrl(&cpu, 4, 0x00008002u); + if (gte_read_ctrl(&cpu, 4) != 0xFFFF8002u) + return fail_value("matrix tail sign extension", 0, 4, 0x00008002u, + 0xFFFF8002u, gte_read_ctrl(&cpu, 4)); + + gte_write_ctrl(&cpu, 26, 0x0000E810u); + if (gte_read_ctrl(&cpu, 26) != 0xFFFFE810u) + return fail_value("H read sign extension", 0, 26, 0x0000E810u, + 0xFFFFE810u, gte_read_ctrl(&cpu, 26)); + + gte_write_ctrl(&cpu, 31, 0x00800000u); + if (gte_read_ctrl(&cpu, 31) != 0x80800000u) + return fail_value("FLAG error summary", 0, 31, 0x00800000u, + 0x80800000u, gte_read_ctrl(&cpu, 31)); + return 0; +} + int test_writes() { for (unsigned iteration = 0; iteration < 64; ++iteration) { CPUState seed; @@ -397,7 +426,6 @@ int test_command_marshaling() { return fail_state("command marshal", iteration, function, cmd, expected, actual); if (actual.gte_data[15] != actual.gte_data[14] || - actual.gte_data[23] != 0u || actual.gte_data[28] != actual.gte_data[29] || actual.gte_data[31] != gte_read_data(&actual, 31)) return fail_value("command canonical aliases", iteration, @@ -703,6 +731,7 @@ int test_precision_speculative_transaction() { } // namespace int main() { + if (int rc = test_hardware_register_semantics()) return rc; if (int rc = test_canonicalizer()) return rc; if (int rc = test_reads()) return rc; if (int rc = test_writes()) return rc; diff --git a/runtime/tests/test_guest_tty.c b/runtime/tests/test_guest_tty.c new file mode 100644 index 00000000..c37e4524 --- /dev/null +++ b/runtime/tests/test_guest_tty.c @@ -0,0 +1,33 @@ +#include "guest_tty.h" + +#include <assert.h> +#include <stdint.h> +#include <string.h> + +int main(void) +{ + uint8_t out[8] = {0}; + uint64_t total = 99; + + psx_guest_tty_reset(); + assert(psx_guest_tty_snapshot(out, sizeof(out), &total) == 0); + assert(total == 0); + + psx_guest_tty_putchar('A'); + psx_guest_tty_putchar('B'); + psx_guest_tty_putchar(0); + psx_guest_tty_putchar('\n'); + assert(psx_guest_tty_snapshot(out, sizeof(out), &total) == 4); + assert(total == 4); + assert(memcmp(out, "AB\0\n", 4) == 0); + + memset(out, 0, sizeof(out)); + assert(psx_guest_tty_snapshot(out, 2, &total) == 2); + assert(total == 4); + assert(out[0] == 0 && out[1] == '\n'); + + psx_guest_tty_reset(); + assert(psx_guest_tty_snapshot(out, sizeof(out), &total) == 0); + assert(total == 0); + return 0; +} diff --git a/tools/conformance_testrom/CMakeLists.txt b/tools/conformance_testrom/CMakeLists.txt new file mode 100644 index 00000000..a68b2c71 --- /dev/null +++ b/tools/conformance_testrom/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.20) +project(PSXConformanceTestROM C CXX) + +set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) + +get_filename_component(_default_root "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) +set(PSXRECOMP_ROOT "${_default_root}" CACHE PATH "Path to PSXRecomp") +set(CONFORMANCE_RUN_DIR "" CACHE PATH "Prepared conformance fixture directory") +set(CONFORMANCE_DEBUG_PORT "4601" CACHE STRING "Runtime debug port") + +# The headless conformance binary does not expose product UI features. Keep +# optional runtime integrations out so a clean fixture build needs no unrelated +# private/submodule state. +set(PSX_REWIND OFF CACHE BOOL "Disable rewind in conformance fixture" FORCE) +set(PSX_NETPLAY OFF CACHE BOOL "Disable netplay in conformance fixture" FORCE) +set(PSX_RECOMP_UI OFF CACHE BOOL "Disable launcher UI in conformance fixture" FORCE) +set(PSX_DEBUG_TOOLS ON CACHE BOOL "Enable the structured oracle endpoint" FORCE) + +if(NOT CONFORMANCE_RUN_DIR) + message(FATAL_ERROR "Set -DCONFORMANCE_RUN_DIR=<prepared fixture directory>") +endif() +get_filename_component(CONFORMANCE_RUN_DIR "${CONFORMANCE_RUN_DIR}" ABSOLUTE) + +file(GLOB _game_full "${CONFORMANCE_RUN_DIR}/generated/test.exe_full_*.c") +if(NOT _game_full AND EXISTS "${CONFORMANCE_RUN_DIR}/generated/test.exe_full.c") + set(_game_full "${CONFORMANCE_RUN_DIR}/generated/test.exe_full.c") +endif() +set(_game_dispatch "${CONFORMANCE_RUN_DIR}/generated/test.exe_dispatch.c") +if(NOT _game_full OR NOT EXISTS "${_game_dispatch}") + message(FATAL_ERROR "Fixture has not been recompiled: ${CONFORMANCE_RUN_DIR}") +endif() + +include("${PSXRECOMP_ROOT}/runtime/runtime.cmake") + +psxrecomp_add_runtime_target(psx-conformance + GAME_GENERATED_FULL_C ${_game_full} + GAME_GENERATED_DISPATCH_C "${_game_dispatch}" + DEBUG_PORT "${CONFORMANCE_DEBUG_PORT}" + WINDOW_TITLE "PSX Conformance" + DEFAULT_GAME_CONFIG_PATH "${CONFORMANCE_RUN_DIR}/game.toml" + EXE_NAME "PSXConformance" +) diff --git a/tools/conformance_testrom/README.md b/tools/conformance_testrom/README.md new file mode 100644 index 00000000..2a36b0eb --- /dev/null +++ b/tools/conformance_testrom/README.md @@ -0,0 +1,70 @@ +# External PS1 conformance fixtures + +This harness runs exact, pinned public PS-X EXE tests through PSXRecomp and +compares the guest console output with the fixture's expected PS1 log. It is a +small regression oracle, not a replacement for matched hardware capture. + +Initial fixtures from `JaCzekanski/ps1-tests` release `build-158`: + +- `cpu-cop`: 17 CPU exception/coprocessor-access assertions. +- `gte-test-all`: 1,150 GTE register/opcode vectors. + +The MIT fixture binaries are not vendored. `fixtures.json` records their exact +release, paths, and SHA-256 identities. The synthetic boot disc additionally +requires `license_data.dat` extracted locally from a PlayStation disc owned by +the operator; that file and every prepared run remain outside Git. + +## Prepare + +Build `psxrecomp-toml`, `psxrecomp-game`, and an OpenBIOS backend first. Then: + +```powershell +python .\tools\conformance_testrom\conformance.py prepare ` + --fixture cpu-cop ` + --source-root D:\private\ps1-tests-build-158 ` + --output D:\validation\cpu-cop-v1 ` + --framework-root D:\src\psxrecomp ` + --recompiler-build D:\build\psxrecomp-recompiler ` + --mkpsxiso D:\tools\mkpsxiso.exe ` + --license-data D:\private\license_data.dat +``` + +The command refuses an occupied output directory, verifies every private input, +generates a synthetic licensed disc, recompiles the test, and writes a +hash-bound `prepare-receipt.json`. + +## Build and run + +```powershell +cmake -S .\tools\conformance_testrom -B D:\build\cpu-cop ` + -G Ninja ` + -DCMAKE_BUILD_TYPE=Release ` + -DCMAKE_C_COMPILER=C:\mingw64\bin\gcc.exe ` + -DCMAKE_CXX_COMPILER=C:\mingw64\bin\g++.exe ` + -DCONFORMANCE_RUN_DIR=D:\validation\cpu-cop-v1 ` + -DCONFORMANCE_DEBUG_PORT=4601 +cmake --build D:\build\cpu-cop --target psx-conformance + +python .\tools\conformance_testrom\conformance.py run ` + --run-dir D:\validation\cpu-cop-v1 ` + --executable D:\build\cpu-cop\PSXConformance.exe ` + --bios .\bios\openbios.bin ` + --port 4601 +``` + +Use the same MinGW toolchain as the runtime. A default Windows Clang/MSVC +configuration may select MASM for libchdr's optional assembly and fail when +`ml64.exe` is not installed. + +`run` starts the runtime headlessly, retrieves the bounded guest-TTY ring over +the existing JSON debug protocol, performs an exact normalized comparison, and +writes `result.json`. Normalization removes capture-format `% ` prefixes, blank +outer lines, CR/LF differences, and the derived `Total tests:` line which some +binaries print but their pinned expected logs omit. Assertion text and pass/fail +counts remain exact; the complete normalized capture is retained in the result. + +## Oracle status + +The release's expected `psx.log` is the first automated oracle. Promotion to a +hardware-qualified result requires Eagle to run the same hashed `test.bin` and +capture the same normalized console result or another unambiguous pass surface. diff --git a/tools/conformance_testrom/conformance.py b/tools/conformance_testrom/conformance.py new file mode 100644 index 00000000..7484efbe --- /dev/null +++ b/tools/conformance_testrom/conformance.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Prepare and run pinned PS1 test-ROM fixtures against PSXRecomp.""" + +from __future__ import annotations + +import argparse +import difflib +import hashlib +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import sys +import time + + +HERE = Path(__file__).resolve().parent +FIXTURES_PATH = HERE / "fixtures.json" + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest().upper() + + +def require_hash(path: Path, expected: str) -> None: + actual = sha256(path) + if actual != expected.upper(): + raise RuntimeError(f"hash mismatch for {path}: {actual} != {expected}") + + +def run_checked(argv: list[str], cwd: Path) -> None: + print("+", subprocess.list2cmdline(argv)) + subprocess.run(argv, cwd=cwd, check=True) + + +def normalized_log(data: bytes) -> str: + text = data.decode("utf-8", errors="replace").replace("\r\n", "\n").replace("\r", "\n") + lines = text.splitlines() + while lines and not lines[0]: + lines.pop(0) + while lines and not lines[-1]: + lines.pop() + lines = [line[2:] if line.startswith("% ") else line for line in lines] + return "\n".join(lines) + ("\n" if lines else "") + + +def comparable_log(text: str) -> str: + # Some ps1-tests binaries print a derived Total count that their pinned + # expected log omits. Preserve it in actual_raw_normalized/result.json, + # but do not let this non-assertion summary line decide conformance. + lines = [line for line in text.splitlines() + if not (line.startswith("Total tests: ") and line[13:].isdigit())] + return "\n".join(lines) + ("\n" if lines else "") + + +def load_catalog() -> dict: + return json.loads(FIXTURES_PATH.read_text(encoding="utf-8")) + + +def git_head(root: Path) -> str: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + + +def prepare(args: argparse.Namespace) -> int: + catalog = load_catalog() + spec = catalog["fixtures"].get(args.fixture) + if spec is None: + raise RuntimeError(f"unknown fixture: {args.fixture}") + + source_root = args.source_root.resolve() + output = args.output.resolve() + framework = args.framework_root.resolve() + recompiler_build = args.recompiler_build.resolve() + mkpsxiso = args.mkpsxiso.resolve() + license_data = args.license_data.resolve() + if output.exists(): + raise RuntimeError(f"output already exists; preserve it and choose a new path: {output}") + + source_exe = source_root / spec["exe"] + source_expected = source_root / spec["expected"] + require_hash(source_exe, spec["exe_sha256"]) + require_hash(source_expected, spec["expected_sha256"]) + for required in ( + recompiler_build / "psxrecomp-toml.exe", + recompiler_build / "psxrecomp-game.exe", + framework / "bios" / "OpenBIOS.toml", + mkpsxiso, + license_data, + ): + if not required.is_file(): + raise RuntimeError(f"missing required input: {required}") + + output.mkdir(parents=True) + disc = output / "disc" + disc.mkdir() + shutil.copy2(source_exe, output / "test.exe") + shutil.copy2(source_expected, output / "expected.log") + shutil.copy2(license_data, disc / "license_data.dat") + + run_checked([ + str(recompiler_build / "psxrecomp-toml.exe"), "test.exe", + "--output", "game.toml", "--seeds", "seeds.txt", + "--name", args.fixture, "--id", "PSXT-00001", + ], output) + game_toml = output / "game.toml" + text = game_toml.read_text(encoding="utf-8") + bios_profile = (framework / "bios" / "OpenBIOS.toml").as_posix() + text = text.replace( + 'seeds = ""', + 'seeds = "seeds.txt"', + ).replace( + 'out_dir = "generated"', + f'out_dir = "generated"\nbios_config = "{bios_profile}"', + ).replace( + '# debug_port = 4370', + f'debug_port = {args.port}', + ) + game_toml.write_text(text, encoding="utf-8", newline="\n") + + (disc / "SYSTEM.CNF").write_text( + "BOOT = cdrom:\\TEST_000.01;1\nTCB = 4\nEVENT = 10\nSTACK = 801FFFF0\n", + encoding="ascii", newline="\n") + volume = "PSXTCPU" if args.fixture == "cpu-cop" else "PSXTGTE" + (disc / "disc.xml").write_text(f'''<?xml version="1.0" encoding="UTF-8"?> +<iso_project image_name="test.bin" cue_sheet="test.cue" no_xa="0"> + <track type="data" xa_edc="true" new_type="true"> + <identifiers system="PLAYSTATION" application="PLAYSTATION" + volume="{volume}" volume_set="{volume}" publisher="PSXRECOMP" + data_preparer="MKPSXISO" copyright="SYSTEM.CNF;1"/> + <license file="license_data.dat"/> + <directory_tree> + <file name="system.cnf" type="data" source="SYSTEM.CNF"/> + <file name="test_000.01" type="data" source="../test.exe"/> + </directory_tree> + </track> +</iso_project> +''', encoding="utf-8", newline="\n") + + run_checked([str(mkpsxiso), "-y", "disc.xml"], disc) + run_checked([str(recompiler_build / "psxrecomp-game.exe"), "--config", "game.toml"], output) + + receipt = { + "schema_version": 1, + "fixture": args.fixture, + "class": spec["class"], + "expected_assertions": spec["expected_assertions"], + "source": catalog["source"], + "framework": {"path": str(framework), "commit": git_head(framework)}, + "inputs": { + "test.exe": sha256(output / "test.exe"), + "expected.log": sha256(output / "expected.log"), + "license_data.dat": sha256(disc / "license_data.dat"), + "psxrecomp-toml.exe": sha256(recompiler_build / "psxrecomp-toml.exe"), + "psxrecomp-game.exe": sha256(recompiler_build / "psxrecomp-game.exe"), + "mkpsxiso.exe": sha256(mkpsxiso), + }, + "outputs": { + "disc/test.bin": sha256(disc / "test.bin"), + "disc/test.cue": sha256(disc / "test.cue"), + "generated/test.exe_dispatch.c": sha256(output / "generated" / "test.exe_dispatch.c"), + }, + "oracle": { + "upstream_expected_log": "available", + "matched_hardware_recapture": "pending", + }, + "debug_port": args.port, + } + (output / "prepare-receipt.json").write_text( + json.dumps(receipt, indent=2) + "\n", encoding="utf-8", newline="\n") + print(f"prepared {args.fixture}: {output}") + return 0 + + +def tcp_query(port: int, payload: dict, timeout: float = 5.0) -> dict: + wire = (json.dumps(payload, separators=(",", ":")) + "\n").encode("ascii") + with socket.create_connection(("127.0.0.1", port), timeout=timeout) as sock: + sock.sendall(wire) + stream = sock.makefile("rb") + line = stream.readline() + if not line: + raise RuntimeError("debug server closed without a response") + return json.loads(line) + + +def capture_tty(port: int, timeout: float) -> bytes: + deadline = time.monotonic() + timeout + last = b"" + while time.monotonic() < deadline: + try: + response = tcp_query(port, {"cmd": "guest_tty_dump", "tail": 65536}) + if not response.get("ok"): + raise RuntimeError(f"guest_tty_dump failed: {response}") + last = bytes.fromhex(response.get("hex", "")) + if "Done.\n" in normalized_log(last): + return last + except (ConnectionError, OSError): + pass + time.sleep(0.1) + return last + + +def compare_capture(run_dir: Path, actual: bytes, result_path: Path) -> bool: + expected_path = run_dir / "expected.log" + expected = expected_path.read_bytes() + expected_norm = comparable_log(normalized_log(expected)) + actual_raw_norm = normalized_log(actual) + actual_norm = actual_raw_norm + expected_lines = expected_norm.splitlines() + if expected_lines: + actual_lines = actual_raw_norm.splitlines() + try: + start = actual_lines.index(expected_lines[0]) + except ValueError: + pass + else: + actual_norm = comparable_log("\n".join(actual_lines[start:]) + "\n") + passed = actual_norm == expected_norm + diff = "" if passed else "\n".join(difflib.unified_diff( + expected_norm.splitlines(), actual_norm.splitlines(), + fromfile="expected", tofile="actual", lineterm="")) + result = { + "schema_version": 1, + "status": "pass" if passed else "fail", + "expected_sha256": sha256(expected_path), + "actual_sha256": hashlib.sha256(actual).hexdigest().upper(), + "expected_normalized": expected_norm, + "actual_raw_normalized": actual_raw_norm, + "actual_normalized": actual_norm, + "diff": diff, + } + result_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8", newline="\n") + return passed + + +def run_fixture(args: argparse.Namespace) -> int: + run_dir = args.run_dir.resolve() + executable = args.executable.resolve() + bios = args.bios.resolve() + for path in (run_dir / "game.toml", run_dir / "disc" / "test.cue", executable, bios): + if not path.is_file(): + raise RuntimeError(f"missing required run input: {path}") + + argv = [str(executable), "--no-launcher", "--headless", + "--game", str(run_dir / "game.toml"), + "--bios", str(bios), "--disc", str(run_dir / "disc" / "test.cue")] + creationflags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0 + proc = subprocess.Popen(argv, cwd=run_dir, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, creationflags=creationflags) + try: + actual = capture_tty(args.port, args.timeout) + finally: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) + + result_path = run_dir / "result.json" + passed = compare_capture(run_dir, actual, result_path) + print(f"{run_dir.name}: {'PASS' if passed else 'FAIL'} ({len(actual)} guest-TTY bytes)") + print(result_path) + return 0 if passed else 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + + prep = sub.add_parser("prepare") + prep.add_argument("--fixture", required=True, choices=sorted(load_catalog()["fixtures"])) + prep.add_argument("--source-root", type=Path, required=True) + prep.add_argument("--output", type=Path, required=True) + prep.add_argument("--framework-root", type=Path, required=True) + prep.add_argument("--recompiler-build", type=Path, required=True) + prep.add_argument("--mkpsxiso", type=Path, required=True) + prep.add_argument("--license-data", type=Path, required=True) + prep.add_argument("--port", type=int, default=4601) + prep.set_defaults(func=prepare) + + run = sub.add_parser("run") + run.add_argument("--run-dir", type=Path, required=True) + run.add_argument("--executable", type=Path, required=True) + run.add_argument("--bios", type=Path, required=True) + run.add_argument("--port", type=int, default=4601) + run.add_argument("--timeout", type=float, default=30.0) + run.set_defaults(func=run_fixture) + return parser + + +def main() -> int: + try: + args = build_parser().parse_args() + return args.func(args) + except (OSError, RuntimeError, subprocess.CalledProcessError) as exc: + print(f"conformance: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/conformance_testrom/fixtures.json b/tools/conformance_testrom/fixtures.json new file mode 100644 index 00000000..0ef920a1 --- /dev/null +++ b/tools/conformance_testrom/fixtures.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "source": { + "repository": "https://github.com/JaCzekanski/ps1-tests", + "license": "MIT", + "release": "build-158", + "release_commit": "5f73d61b27a8d51269b54dde1c456d2d2f7dc865", + "release_archive_sha256": "5DE0E82854C1A511697A7ADE381C2D778EC4B2FFF4A7D7AC79E8098C35EB859B" + }, + "fixtures": { + "cpu-cop": { + "exe": "cpu/cop/cop.exe", + "exe_sha256": "D31D3CEC66ABE8DB7935B51590F206602346284DA2AC20EA8614339924EA9981", + "expected": "cpu/cop/psx.log", + "expected_sha256": "E3B09837989C4FD53E4D192AC869F437D965A4D71342BB80D038B5CC813C729C", + "expected_assertions": 17, + "class": "CPU exception and coprocessor access" + }, + "gte-test-all": { + "exe": "gte/test-all/test-all.exe", + "exe_sha256": "479ADC925DA6CC4844B567A033FABC06B2AB934FFBBB79C5DAD698BE20944B9E", + "expected": "gte/test-all/psx.log", + "expected_sha256": "78DB534A7B02B4BC4C521A20A406743F296DADA95C5220568F2DF6E0AC37B7A0", + "expected_assertions": 1150, + "class": "GTE registers and valid opcodes" + } + } +} From 324690cb970528e1ddf16170883c31103edd79d4 Mon Sep 17 00:00:00 2001 From: Pegasus <pegasus@psxrecomp.local> Date: Wed, 9 Sep 2026 14:40:43 +0300 Subject: [PATCH 34/56] Preserve fully qualified Windows network asset paths (cherry picked from commit 51d5d56a275d00b2341817c1ccfda2841274652e) --- docs/tasreplays/windows-unc-paths.md | 32 +++++++++++++++++ recompiler/CMakeLists.txt | 3 ++ recompiler/include/host_path.h | 42 ++++++++++++++++++++++ recompiler/src/config_loader.cpp | 29 +++++++-------- recompiler/src/main_bios.cpp | 5 +-- recompiler/tests/host_path_test.cpp | 53 ++++++++++++++++++++++++++++ runtime/CMakeLists.txt | 7 ++-- runtime/src/cue_sheet.cpp | 3 +- runtime/src/disc_path.cpp | 3 +- runtime/src/main.cpp | 37 +++++++++---------- 10 files changed, 175 insertions(+), 39 deletions(-) create mode 100644 docs/tasreplays/windows-unc-paths.md create mode 100644 recompiler/include/host_path.h create mode 100644 recompiler/tests/host_path_test.cpp diff --git a/docs/tasreplays/windows-unc-paths.md b/docs/tasreplays/windows-unc-paths.md new file mode 100644 index 00000000..0f519699 --- /dev/null +++ b/docs/tasreplays/windows-unc-paths.md @@ -0,0 +1,32 @@ +# Windows network asset paths + +Pepsiman's first native attempt exited before guest execution because BIOS +selection could not open the path returned by its resolver. An independent +fixture using the same WinLibs GCC 16.1 UCRT library reproduced the boundary: +the original `\\server\share\file` was readable, `path.is_absolute()` returned +false, and `std::filesystem::absolute()` returned `D:\server\share\file` without +an error. Python's argument list had preserved the original argument. + +The handover corpus already records a similar network-path symptom under +PSX-WIN-003. This reproduction isolates a filesystem resolution defect in +addition to that finding's separate PowerShell argument-boundary problem. +Microsoft documents UNC paths as fully qualified in +[Windows file path formats](https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats). + +`host_path.h` preserves fully qualified Windows network and device paths +before invoking the standard library's relative-path resolution. Runtime +BIOS, disc and config resolution, CUE payload resolution, and compiler config +and BIOS CLI paths use that shared rule. Relative config values still resolve +against their config root; ordinary relative CLI paths still resolve against +the current directory. No emulated device or timing behavior changes. + +`host_path_test` is registered with the recompiler CTest suite. It checks both +UNC separator forms, device namespace paths, mapped drives, relative paths, +config-root isolation and error-code clearing. Its optional path argument +also checks that an owned asset remains readable, without changing it. +The campaign additionally compiled the production disc resolver and verified +that both network spellings preserve the Pepsiman CUE and all eight tracks. +The existing disc resolver, CD-DA and SBI fixtures pass with this correction. + +This is a host admission correction, not a TAS pass or PS1 hardware claim. +Full native observations and registered title regressions remain required. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 13966f45..7c64817c 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -352,6 +352,9 @@ if(BUILD_TESTING) target_include_directories(control_flow_metadata_test PRIVATE include) target_link_libraries(control_flow_metadata_test PRIVATE fmt) add_test(NAME control_flow_metadata_test COMMAND control_flow_metadata_test) + add_executable(host_path_test tests/host_path_test.cpp) + target_include_directories(host_path_test PRIVATE include) + add_test(NAME host_path_test COMMAND host_path_test) add_test(NAME emitter_directive_line_test COMMAND emitter_directive_line_test) diff --git a/recompiler/include/host_path.h b/recompiler/include/host_path.h new file mode 100644 index 00000000..a4a665ca --- /dev/null +++ b/recompiler/include/host_path.h @@ -0,0 +1,42 @@ +#pragma once + +#include <filesystem> +#include <system_error> + +namespace PSXRecompV4 { + +// Some MinGW libstdc++ builds treat a readable UNC path as drive-relative: +// absolute("\\\\server\\share\\file") becomes "D:\\server\\share\\file". +// Windows already considers UNC and device namespace paths fully qualified. +// Preserve their spelling before asking std::filesystem to anchor a path. +inline bool host_path_is_absolute(const std::filesystem::path& path) { +#ifdef _WIN32 + const auto& s = path.native(); + const auto separator = [](wchar_t c) { return c == L'\\' || c == L'/'; }; + if (s.size() > 2 && separator(s[0]) && separator(s[1]) && !separator(s[2])) + return true; +#endif + return path.is_absolute(); +} + +inline std::filesystem::path host_absolute(const std::filesystem::path& path, + std::error_code& ec) { + if (host_path_is_absolute(path)) { + ec.clear(); + return path; + } + return std::filesystem::absolute(path, ec); +} + +inline std::filesystem::path host_absolute(const std::filesystem::path& path) { + if (host_path_is_absolute(path)) return path; + return std::filesystem::absolute(path); +} + +// A fully qualified config value must not be prefixed with its config root. +inline std::filesystem::path host_resolve(const std::filesystem::path& root, + const std::filesystem::path& path) { + return host_absolute(host_path_is_absolute(path) ? path : root / path); +} + +} // namespace PSXRecompV4 diff --git a/recompiler/src/config_loader.cpp b/recompiler/src/config_loader.cpp index c82ec734..b4ee4147 100644 --- a/recompiler/src/config_loader.cpp +++ b/recompiler/src/config_loader.cpp @@ -14,6 +14,7 @@ #include <vector> #include "bios_rom_alias.h" +#include "host_path.h" #include "fmt/format.h" #include "ps1_exe_parser.h" @@ -361,7 +362,7 @@ static RuntimeConfig parse_runtime_block(const toml::value& cfg, const fs::path& } if (runtime.contains("memcard_dir")) { const auto rel = toml::find<std::string>(runtime, "memcard_dir"); - rt.memcard_dir = fs::absolute(root / rel); + rt.memcard_dir = PSXRecompV4::host_resolve(root, rel); rt.has_memcard_dir = true; } if (runtime.contains("disc_speed")) { @@ -819,7 +820,7 @@ static RuntimeConfig parse_runtime_block(const toml::value& cfg, const fs::path& } fs::path find_project_root(const fs::path& config_path) { - fs::path cur = fs::absolute(config_path).parent_path(); + fs::path cur = PSXRecompV4::host_absolute(config_path).parent_path(); const fs::path fallback = cur; for (int i = 0; i < 8; ++i) { for (const char* marker : { ".gitignore", ".git", "CMakeLists.txt" }) { @@ -855,7 +856,7 @@ static std::string derive_out_stem(const std::string& rom_basename) { } BiosConfig load_bios_config(const fs::path& config_path_in) { - const fs::path config_path = fs::absolute(config_path_in); + const fs::path config_path = PSXRecompV4::host_absolute(config_path_in); if (!fs::exists(config_path)) { throw std::runtime_error( fmt::format("config file not found: {}", config_path.string())); @@ -920,7 +921,7 @@ BiosConfig load_bios_config(const fs::path& config_path_in) { // Tolerate either BIOS filename convention (bare "SCPH1001.BIN" vs // region-qualified "US-PSX-SCPH1001.BIN") — see bios_rom_alias.h. A no-op // for game [program].exe fields, which never match a BIOS model token. - const fs::path rom_path = resolve_bios_rom(fs::absolute(root / rom_field)); + const fs::path rom_path = resolve_bios_rom(PSXRecompV4::host_resolve(root, rom_field)); const uint32_t load_address = parse_hex(toml::find<std::string>(prog, "load_address"), "program.load_address"); @@ -954,13 +955,13 @@ BiosConfig load_bios_config(const fs::path& config_path_in) { fmt::format("{}: [recompiler] missing 'seeds' field", config_path.string())); } const std::string seeds_field = toml::find<std::string>(recomp, "seeds"); - const fs::path seeds_path = fs::absolute(root / seeds_field); + const fs::path seeds_path = PSXRecompV4::host_resolve(root, seeds_field); const std::string out_dir_field = recomp.contains("out_dir") ? toml::find<std::string>(recomp, "out_dir") : std::string{"generated"}; - const fs::path out_dir = fs::absolute(root / out_dir_field); + const fs::path out_dir = PSXRecompV4::host_resolve(root, out_dir_field); const bool strict = recomp.contains("strict") ? toml::find<bool>(recomp, "strict") @@ -1128,7 +1129,7 @@ BiosConfig load_bios_config(const fs::path& config_path_in) { } GameConfig load_game_config(const fs::path& config_path_in) { - const fs::path config_path = fs::absolute(config_path_in); + const fs::path config_path = PSXRecompV4::host_absolute(config_path_in); if (!fs::exists(config_path)) { throw std::runtime_error( fmt::format("game config not found: {}", config_path.string())); @@ -1176,7 +1177,7 @@ GameConfig load_game_config(const fs::path& config_path_in) { throw std::runtime_error( fmt::format("{}: [game] missing 'exe' or 'rom' field", config_path.string())); } - const fs::path exe_path = fs::absolute(root / exe_field); + const fs::path exe_path = PSXRecompV4::host_resolve(root, exe_field); // Auto-detect EXE header values for any field not explicitly set in TOML. // Parses the PS-X EXE header once and fills in load_address, entry_pc, @@ -1244,10 +1245,10 @@ GameConfig load_game_config(const fs::path& config_path_in) { std::vector<fs::path> discs; if (game.contains("discs")) { const auto& arr = toml::find<std::vector<std::string>>(game, "discs"); - for (const auto& d : arr) discs.push_back(fs::absolute(root / d)); + for (const auto& d : arr) discs.push_back(PSXRecompV4::host_resolve(root, d)); } else if (game.contains("disc")) { const auto& d = toml::find<std::string>(game, "disc"); - discs.push_back(fs::absolute(root / d)); + discs.push_back(PSXRecompV4::host_resolve(root, d)); } /* Per-disc serials, parallel to `discs`. Absent => no per-disc gate. */ std::vector<std::string> disc_serials; @@ -1346,12 +1347,12 @@ GameConfig load_game_config(const fs::path& config_path_in) { fmt::format("{}: [recompiler] missing 'seeds' field", config_path.string())); } const fs::path seeds_path = - fs::absolute(root / toml::find<std::string>(recomp, "seeds")); + PSXRecompV4::host_resolve(root, toml::find<std::string>(recomp, "seeds")); fs::path bios_thunks_path; if (recomp.contains("bios_thunks")) { bios_thunks_path = - fs::absolute(root / toml::find<std::string>(recomp, "bios_thunks")); + PSXRecompV4::host_resolve(root, toml::find<std::string>(recomp, "bios_thunks")); } // [recompiler] bios_config — the BIOS profile this game is built @@ -1360,14 +1361,14 @@ GameConfig load_game_config(const fs::path& config_path_in) { fs::path bios_config_path; if (recomp.contains("bios_config")) { bios_config_path = - fs::absolute(root / toml::find<std::string>(recomp, "bios_config")); + PSXRecompV4::host_resolve(root, toml::find<std::string>(recomp, "bios_config")); } const std::string out_dir_field = recomp.contains("out_dir") ? toml::find<std::string>(recomp, "out_dir") : std::string{"generated"}; - const fs::path out_dir = fs::absolute(root / out_dir_field); + const fs::path out_dir = PSXRecompV4::host_resolve(root, out_dir_field); const bool strict = recomp.contains("strict") ? toml::find<bool>(recomp, "strict") diff --git a/recompiler/src/main_bios.cpp b/recompiler/src/main_bios.cpp index a45c9e9f..69b7b50e 100644 --- a/recompiler/src/main_bios.cpp +++ b/recompiler/src/main_bios.cpp @@ -41,6 +41,7 @@ #include "bios_slice_walker.h" #include "bios_rom_alias.h" +#include "host_path.h" #include "config_loader.h" #include "full_function_emitter.h" #include "function_discovery.h" @@ -969,9 +970,9 @@ int main(int argc, char** argv) { } else if (a.rfind("--config=", 0) == 0) { config_path = a.substr(std::string("--config=").size()); } else if (a == "--rom" && i + 1 < argc) { - config_rom_override = fs::absolute(argv[++i]); + config_rom_override = PSXRecompV4::host_absolute(argv[++i]); } else if (a == "--out-dir" && i + 1 < argc) { - config_out_override = fs::absolute(argv[++i]); + config_out_override = PSXRecompV4::host_absolute(argv[++i]); } } if (config_path) { diff --git a/recompiler/tests/host_path_test.cpp b/recompiler/tests/host_path_test.cpp new file mode 100644 index 00000000..88044cb3 --- /dev/null +++ b/recompiler/tests/host_path_test.cpp @@ -0,0 +1,53 @@ +#include "host_path.h" + +#include <cstdio> +#include <cstdlib> +#include <fstream> + +namespace fs = std::filesystem; +using namespace PSXRecompV4; +static int checks = 0; +static void check(bool value, const char* message) { + ++checks; + if (!value) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); } +} + +int main(int argc, char** argv) { +#ifdef _WIN32 + for (const char* spelling : { + R"(\\server\share\BIOS files\SCPH5500.BIN)", + "//server/share/Game (Japan)/game.cue", + R"(\\?\UNC\server\share\game.cue)", + R"(\\?\C:\Game files\game.cue)", + R"(Z:\Game files\game.cue)"}) { + const fs::path path(spelling); + check(host_path_is_absolute(path), "fully qualified Windows path"); + std::error_code ec = std::make_error_code(std::errc::invalid_argument); + check(host_absolute(path, ec).native() == path.native() && !ec, + "nonthrowing absolute preserves exact Windows path and clears error"); + check(host_absolute(path).native() == path.native(), + "throwing absolute preserves exact Windows path"); + check(host_resolve(R"(D:\unrelated\config)", path).native() == path.native(), + "config root never replaces an absolute Windows path"); + } + check(!host_path_is_absolute(R"(\current-drive\file)"), "drive-rooted path needs current drive"); + check(!host_path_is_absolute(R"(C:drive-relative.bin)"), "drive-relative path needs drive directory"); +#else + check(host_absolute("/tmp/PSX files/game.cue") == fs::path("/tmp/PSX files/game.cue"), + "POSIX absolute path unchanged"); + check(!host_path_is_absolute(R"(\\server\share\file)"), "backslashes remain ordinary on POSIX"); +#endif + const fs::path relative("relative directory/game.cue"); + check(!host_path_is_absolute(relative), "ordinary relative path"); + check(host_absolute(relative) == fs::absolute(relative), "relative path anchors to cwd"); + const auto root = fs::current_path() / "config directory"; + check(host_resolve(root, relative) == root / relative, "relative config value anchors to root"); + // Optional private integration input: inspect only, never change the asset. + if (argc == 2) { + const fs::path path(argv[1]); + check(bool(std::ifstream(path, std::ios::binary)), "original private path readable"); + check(bool(std::ifstream(host_absolute(path), std::ios::binary)), "resolved private path readable"); + check(host_absolute(path).native() == path.native(), "private path remains exact"); + } + std::printf("host_path: %d checks passed\n", checks); +} diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 76184bea..d39d56e1 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -368,7 +368,7 @@ if(BUILD_TESTING) src/disc_path.cpp src/cue_sheet.cpp src/iso_reader.cpp) - target_include_directories(disc_path_resolve_test PRIVATE include) + target_include_directories(disc_path_resolve_test PRIVATE include ${PSXRECOMP_ROOT}/recompiler/include) target_link_libraries(disc_path_resolve_test PRIVATE chdr-static) add_test(NAME disc_path_resolve_test COMMAND disc_path_resolve_test) @@ -384,7 +384,7 @@ if(BUILD_TESTING) tests/test_iso_reader_cdda.cpp src/iso_reader.cpp src/cue_sheet.cpp) - target_include_directories(iso_reader_cdda_test PRIVATE include) + target_include_directories(iso_reader_cdda_test PRIVATE include ${PSXRECOMP_ROOT}/recompiler/include) target_link_libraries(iso_reader_cdda_test PRIVATE chdr-static) add_test(NAME iso_reader_cdda_test COMMAND iso_reader_cdda_test) @@ -392,7 +392,7 @@ if(BUILD_TESTING) tests/test_iso_reader_sbi.cpp src/iso_reader.cpp src/cue_sheet.cpp) - target_include_directories(iso_reader_sbi_test PRIVATE include) + target_include_directories(iso_reader_sbi_test PRIVATE include ${PSXRECOMP_ROOT}/recompiler/include) target_link_libraries(iso_reader_sbi_test PRIVATE chdr-static) add_test(NAME iso_reader_sbi_test COMMAND iso_reader_sbi_test) @@ -425,6 +425,7 @@ if(BUILD_TESTING) src/psx_sha256.c) target_include_directories(mod_runtime_test PRIVATE include + ../recompiler/include ../recompiler/lib/toml11) target_link_libraries(mod_runtime_test PRIVATE chdr-static) # The real disc hasher uses a 1 MiB scratch buffer. Match the runtime's diff --git a/runtime/src/cue_sheet.cpp b/runtime/src/cue_sheet.cpp index bad497dc..bcfd6b54 100644 --- a/runtime/src/cue_sheet.cpp +++ b/runtime/src/cue_sheet.cpp @@ -1,6 +1,7 @@ // cue_sheet.cpp — see cue_sheet.h. #include "cue_sheet.h" +#include "host_path.h" #include <cctype> #include <cstdio> @@ -94,7 +95,7 @@ fs::path resolve_cue_relative_file(const fs::path& dir, const std::string& name) named = fs::path(slashed); } - const fs::path literal = named.is_absolute() ? named : (dir / named); + const fs::path literal = PSXRecompV4::host_path_is_absolute(named) ? named : (dir / named); std::error_code ec; if (fs::exists(literal, ec)) return literal; diff --git a/runtime/src/disc_path.cpp b/runtime/src/disc_path.cpp index a2e208d0..50f466dc 100644 --- a/runtime/src/disc_path.cpp +++ b/runtime/src/disc_path.cpp @@ -1,6 +1,7 @@ // disc_path.cpp — see disc_path.h. #include "disc_path.h" +#include "host_path.h" #include "cue_sheet.h" @@ -39,7 +40,7 @@ bool same_file_path(const fs::path& a, const fs::path& b) { fs::path absolute_or_self(const fs::path& p) { std::error_code ec; - fs::path abs = fs::absolute(p, ec); + fs::path abs = PSXRecompV4::host_absolute(p, ec); return ec ? p : abs; } diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 4dd39e78..ca569218 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -78,6 +78,7 @@ extern "C" void psx_event_step_conservative_env_init(void); #include "freeze_heartbeat.h" #include "config_loader.h" #include "bios_rom_alias.h" +#include "host_path.h" #include "launcher_device.h" #include "game_options.h" #include "mod_plugins.h" @@ -2128,7 +2129,7 @@ static void sdl_drc_callback(void* /*user*/, Uint8* stream, int len) { static std::filesystem::path find_upward(std::filesystem::path start, const std::filesystem::path& marker) { std::error_code ec; - start = std::filesystem::absolute(start, ec); + start = PSXRecompV4::host_absolute(start, ec); if (ec) start = std::filesystem::current_path(); if (!std::filesystem::is_directory(start, ec)) start = start.parent_path(); @@ -2167,12 +2168,12 @@ static std::filesystem::path exe_dir_from_argv(const char* argv0) { // is the .AppImage's own path, so settings.toml anchors next to it. if (exe_dir.empty()) { if (const char* appimg = std::getenv("APPIMAGE"); appimg && appimg[0]) { - exe_dir = fs::absolute(appimg, ec).parent_path(); + exe_dir = PSXRecompV4::host_absolute(appimg, ec).parent_path(); if (ec) exe_dir.clear(); } } if (exe_dir.empty() && argv0 && argv0[0]) { - exe_dir = fs::absolute(argv0, ec).parent_path(); + exe_dir = PSXRecompV4::host_absolute(argv0, ec).parent_path(); if (ec) exe_dir.clear(); } // Last-ditch only (should be unreachable on a normal launch); a bare "." so @@ -2188,15 +2189,15 @@ static std::filesystem::path resolve_existing_runtime_path(const char* requested std::error_code ec; fs::path p(requested); - if (fs::exists(p, ec)) return fs::absolute(p, ec); - if (p.is_absolute()) return {}; + if (fs::exists(p, ec)) return PSXRecompV4::host_absolute(p, ec); + if (PSXRecompV4::host_path_is_absolute(p)) return {}; // Anchor exclusively on the exe directory — never cwd (see exe_dir_from_argv). const fs::path root = exe_dir_from_argv(argv0); fs::path direct = root / p; - if (fs::exists(direct, ec)) return fs::absolute(direct, ec); + if (fs::exists(direct, ec)) return PSXRecompV4::host_absolute(direct, ec); fs::path found = find_upward(root, p); - if (!found.empty()) return fs::absolute(found / p, ec); + if (!found.empty()) return PSXRecompV4::host_absolute(found / p, ec); return {}; } @@ -2685,24 +2686,24 @@ static std::filesystem::path resolve_bios_path(const char* requested, const char if (!requested || !requested[0]) return {}; fs::path p(requested); if (fs::exists(p, ec)) { - fs::path abs = fs::absolute(p, ec); + fs::path abs = PSXRecompV4::host_absolute(p, ec); return ec ? p : abs; } // Either BIOS filename convention is acceptable: a dump folder holding // "US-PSX-SCPH1001.BIN" satisfies a request for "SCPH1001.BIN" and vice // versa (see recompiler/include/bios_rom_alias.h). if (fs::path aliased = PSXRecompV4::resolve_bios_rom(p); aliased != p) { - fs::path abs = fs::absolute(aliased, ec); + fs::path abs = PSXRecompV4::host_absolute(aliased, ec); return ec ? aliased : abs; } - if (p.is_absolute()) return p; + if (PSXRecompV4::host_path_is_absolute(p)) return p; // Anchor on the exe directory — never cwd (see exe_dir_from_argv). fs::path found = find_upward(exe_dir_from_argv(argv0), p); if (!found.empty()) return found / p; // Same walk, accepting the other naming convention at each rung: the // literal name is absent but a region-qualified sibling may be present. - for (fs::path dir = fs::absolute(exe_dir_from_argv(argv0), ec); + for (fs::path dir = PSXRecompV4::host_absolute(exe_dir_from_argv(argv0), ec); !dir.empty(); dir = dir.parent_path()) { const fs::path aliased = PSXRecompV4::resolve_bios_rom(dir / p); if (aliased != dir / p && fs::exists(aliased, ec)) return aliased; @@ -2764,7 +2765,7 @@ static std::filesystem::path discover_retail_bios_near(const char* argv0) { const fs::path cand = dir / name_buf[ni]; if (retail_bios_file_ok(cand)) { auto abs = fs::weakly_canonical(cand, ec); - if (ec) abs = fs::absolute(cand, ec); + if (ec) abs = PSXRecompV4::host_absolute(cand, ec); return abs; } } @@ -3929,7 +3930,7 @@ static std::filesystem::path resolve_overlay_capture_path( auto root_relative = [&](const std::string& raw) { std::filesystem::path p(raw); - return p.is_absolute() ? p : value_base / p; + return PSXRecompV4::host_path_is_absolute(p) ? p : value_base / p; }; std::filesystem::path result; if (!direct.empty()) result = root_relative(direct); @@ -12530,7 +12531,7 @@ int main(int argc, char** argv) { default_game_config_storage = default_game_config.string(); game_config_path = default_game_config_storage.c_str(); } - } else if (!std::filesystem::path(game_config_path).is_absolute()) { + } else if (!PSXRecompV4::host_path_is_absolute(std::filesystem::path(game_config_path))) { // An explicit --game with a relative path must ALSO anchor on the exe // dir, never cwd — otherwise the disc / memcard_dir / game_options.toml // that resolve against this file's parent silently point at cwd. Resolve @@ -13739,7 +13740,7 @@ int main(int argc, char** argv) { std::error_code ec; if (!resolved.empty() && std::filesystem::exists(resolved, ec)) { seed.bios_path = std::filesystem::weakly_canonical(resolved, ec); - if (ec) seed.bios_path = std::filesystem::absolute(resolved, ec); + if (ec) seed.bios_path = PSXRecompV4::host_absolute(resolved, ec); seed.has_bios_path = true; } else { seed.bios_path.clear(); @@ -14035,7 +14036,7 @@ int main(int argc, char** argv) { if (!resolved.empty() && std::filesystem::exists(resolved, ec)) { auto abs = std::filesystem::weakly_canonical(resolved, ec); - if (ec) abs = std::filesystem::absolute(resolved, ec); + if (ec) abs = PSXRecompV4::host_absolute(resolved, ec); std::snprintf(ls.bios_path, sizeof(ls.bios_path), "%s", abs.string().c_str()); } @@ -14223,7 +14224,7 @@ int main(int argc, char** argv) { seed.bios_path = std::filesystem::weakly_canonical(resolved, ec); if (ec) - seed.bios_path = std::filesystem::absolute(resolved, ec); + seed.bios_path = PSXRecompV4::host_absolute(resolved, ec); } else { seed.bios_path = ls.bios_path; } @@ -15961,7 +15962,7 @@ soft_return_lobby: std::error_code ec; if (!resolved.empty() && std::filesystem::exists(resolved, ec)) { auto abs = std::filesystem::weakly_canonical(resolved, ec); - if (ec) abs = std::filesystem::absolute(resolved, ec); + if (ec) abs = PSXRecompV4::host_absolute(resolved, ec); std::snprintf(ls.bios_path, sizeof(ls.bios_path), "%s", abs.string().c_str()); } else { From 77a84803636ebe91e4a312ac2b7aa98715ab80ed Mon Sep 17 00:00:00 2001 From: Pegasus <pegasus@psxrecomp.local> Date: Sat, 12 Sep 2026 17:17:59 +0300 Subject: [PATCH 35/56] Read disc region from the actual SYSTEM.CNF directory entry Prefer the ISO9660 boot file over the bounded raw-image scan. Biohazard stores SYSTEM.CNF outside that scan, leaving fresh builds at the default SCEA response. Add raw and cooked late-file regressions with an early decoy serial. The full-profile prefix again matches the retained native baseline through 5999 returns; the independent-source frontier remains return 464. (cherry picked from commit 82c3de13ef0763b8c58949c99e5c5779899fa1f3) --- recompiler/CMakeLists.txt | 12 +++++++ runtime/include/disc_identity.h | 3 +- runtime/src/disc_identity.cpp | 12 +++++-- runtime/tests/test_disc_identity.cpp | 50 ++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 runtime/tests/test_disc_identity.cpp diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 7c64817c..eb880682 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -352,6 +352,18 @@ if(BUILD_TESTING) target_include_directories(control_flow_metadata_test PRIVATE include) target_link_libraries(control_flow_metadata_test PRIVATE fmt) add_test(NAME control_flow_metadata_test COMMAND control_flow_metadata_test) + if(PSXRECOMP_ENABLE_CHD) + add_executable(disc_identity_test ../runtime/tests/test_disc_identity.cpp + ../runtime/src/disc_identity.cpp ../runtime/src/disc_path.cpp + ../runtime/src/iso_reader.cpp ../runtime/src/cue_sheet.cpp + ../runtime/src/psx_sha256.c ../runtime/src/crc32.c) + target_include_directories(disc_identity_test PRIVATE ../runtime/include include) + target_compile_options(disc_identity_test PRIVATE + $<$<CXX_COMPILER_ID:MSVC>:/UNDEBUG> + $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-UNDEBUG>) + target_link_libraries(disc_identity_test PRIVATE chdr-static) + add_test(NAME disc_identity_test COMMAND disc_identity_test) + endif() add_executable(host_path_test tests/host_path_test.cpp) target_include_directories(host_path_test PRIVATE include) add_test(NAME host_path_test COMMAND host_path_test) diff --git a/runtime/include/disc_identity.h b/runtime/include/disc_identity.h index 241ac19e..c7003eab 100644 --- a/runtime/include/disc_identity.h +++ b/runtime/include/disc_identity.h @@ -2,7 +2,8 @@ // // Single source of truth for "is this the right disc?". Reads a .cue, raw // .bin/.iso/.img/.car, or CHD, checks for the ISO9660 PVD, extracts the volume id and the -// PlayStation boot serial (from SYSTEM.CNF), derives the region from the +// PlayStation boot serial (from the actual SYSTEM.CNF directory entry, with +// an early-image scan as fallback), derives the region from the // serial prefix, and optionally compares against an expected serial / CRC32. // // Also mounts the resolved path via ISOReader to capture TOC geometry diff --git a/runtime/src/disc_identity.cpp b/runtime/src/disc_identity.cpp index 79fe1d0a..63ec5c63 100644 --- a/runtime/src/disc_identity.cpp +++ b/runtime/src/disc_identity.cpp @@ -140,6 +140,14 @@ void fill_toc_from_mount(DiscIdentity& v, const DiscPathResolution& resolved) { return; } v.toc_opened = true; + // SYSTEM.CNF can lie beyond the bounded early-image scan. Read the actual + // directory entry first; unrelated executable strings must not select a region. + uint8_t pvd[2048]; + if (reader.ReadSector(16, pvd) && is_iso_pvd(pvd)) { + std::vector<uint8_t> cnf(4096); + cnf.resize(reader.ReadFile("SYSTEM.CNF", cnf.data(), cnf.size())); + v.detected_serial = scan_boot_serial(cnf); + } v.track_count = reader.TrackCount(); if (v.track_count < 1) v.track_count = 1; v.leadout_lba = reader.GetSectorCount(); @@ -276,7 +284,7 @@ DiscIdentity identify_disc(const fs::path& path, } } if (scan_ok) { - v.detected_serial = scan_boot_serial(scan); + if (v.detected_serial.empty()) v.detected_serial = scan_boot_serial(scan); if (v.expected_serial_given) { const std::string serial_id = uppercase_ascii(expected_serial); std::string exe_id = serial_id; @@ -354,7 +362,7 @@ DiscIdentity identify_disc(const fs::path& path, const size_t scan_len = (size_t)std::min<uint64_t>(size, 16ull * 1024ull * 1024ull); std::vector<uint8_t> scan(scan_len); if (read_at(f, 0, scan.data(), scan.size())) { - v.detected_serial = scan_boot_serial(scan); + if (v.detected_serial.empty()) v.detected_serial = scan_boot_serial(scan); if (v.expected_serial_given) { // Match either the game-id form (SCUS-94236) or the EXE form (SCUS_942.36). diff --git a/runtime/tests/test_disc_identity.cpp b/runtime/tests/test_disc_identity.cpp new file mode 100644 index 00000000..cd10e5c6 --- /dev/null +++ b/runtime/tests/test_disc_identity.cpp @@ -0,0 +1,50 @@ +// SYSTEM.CNF beyond 16 MiB wins over unrelated early cdrom: strings. +#include "disc_identity.h" +#include <cassert> +#include <chrono> +#include <cstring> +#include <filesystem> +#include <fstream> +#include <vector> + +namespace fs=std::filesystem; +static void both32(uint8_t* p,uint32_t value) { + for(unsigned i=0;i<4;i++) {p[i]=(uint8_t)(value>>(8*i));p[7-i]=p[i];} +} +static void fixture(const fs::path& path,bool raw) { + const unsigned unit=raw?2352:2048,offset=raw?24:0; + std::ofstream file(path,std::ios::binary); + file.seekp(10000ull*unit-1);file.put(0); + if(raw) { + uint8_t sync[16]={};std::memset(sync+1,255,10);sync[15]=2; + file.seekp(0);file.write((char*)sync,sizeof sync); + } + auto write=[&](unsigned lba,const std::vector<uint8_t>& data) { + file.seekp((uint64_t)lba*unit+offset); + file.write((const char*)data.data(),data.size()); + }; + std::vector<uint8_t> pvd(2048); + pvd[0]=1;std::memcpy(pvd.data()+1,"CD001",5);pvd[6]=1; + auto* root=pvd.data()+156;root[0]=34;both32(root+2,20);both32(root+10,2048); + root[25]=2;root[28]=root[31]=root[32]=1; + write(16,pvd); + const std::string cnf="BOOT = cdrom:SLPS_009.98;1\r\n"; + std::vector<uint8_t> directory(2048); + directory[0]=46;both32(directory.data()+2,9000);both32(directory.data()+10,(uint32_t)cnf.size()); + directory[28]=directory[31]=1;directory[32]=12; + std::memcpy(directory.data()+33,"SYSTEM.CNF;1",12);write(20,directory); + write(9000,std::vector<uint8_t>(cnf.begin(),cnf.end())); + const std::string decoy="BOOT = cdrom:SLUS_999.99;1\r\n"; + write(40,std::vector<uint8_t>(decoy.begin(),decoy.end())); +} +int main() { + auto root=fs::temp_directory_path()/ ("psx-disc-identity-"+std::to_string( + std::chrono::steady_clock::now().time_since_epoch().count())); + fs::create_directory(root); + for(bool raw:{false,true}) { + auto image=root/(raw?"disc.bin":"disc.iso");fixture(image,raw); + auto id=PSXRecompV4::identify_disc(image,"",0,false,false); + assert(id.opened && id.has_header && id.detected_serial=="SLPS-00998" && id.region=="NTSC-J"); + } + fs::remove(root/"disc.iso");fs::remove(root/"disc.bin");fs::remove(root); +} From 5968162fe999dbe74633f9d8b246e1d226a9db1d Mon Sep 17 00:00:00 2001 From: Pegasus <pegasus@psxrecomp.local> Date: Wed, 9 Sep 2026 18:06:19 +0300 Subject: [PATCH 36/56] fix: publish deferred CPU work before GTE deadline checks (cherry picked from commit 43abd4a5c84a6dc2732e572636dfcca0e2ab1fb1) --- docs/source-gte-deferred-deadlines.md | 39 +++++++++ recompiler/CMakeLists.txt | 15 ++++ runtime/src/psx_cycles.c | 12 +++ runtime/tests/test_gte_deferred.c | 112 ++++++++++++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 docs/source-gte-deferred-deadlines.md create mode 100644 runtime/tests/test_gte_deferred.c diff --git a/docs/source-gte-deferred-deadlines.md b/docs/source-gte-deferred-deadlines.md new file mode 100644 index 00000000..d9d38a42 --- /dev/null +++ b/docs/source-gte-deferred-deadlines.md @@ -0,0 +1,39 @@ +# GTE deadlines and deferred CPU work + +Pepsiman qualification10 first differs at return 1,514: source cycle +856709465, native 856709463, one changed RAM page (1FF000). Both repeats +produce identical RAM/CPU/log captures and later stop after 2,288 returns at +a separate GPU scope guard. The original inputs are unchanged. + +Passive pre-fetch tracing is qualified against each untraced control. The +first 96,369 PC/clock entries agree. The following entry is PC 8001E4B8: +source 856413994, native 856413995. The preceding SWC2 reads the result of a +DPCS command issued while prior load-wait credits still overlap CPU steps. +Pending GPR-load values and kernel scratch registers are not assumed equal +at this internal representation boundary. + +The runtime calculated its GTE stall from psx_cycle_count before publishing +the instruction's pending base cycle. psx_advance_cycles then published that +pending cycle and added the stale stall, counting the cycle twice. The same +omission could arm a GTE command deadline too early or give an MFC2/CFC2 read +the wrong load-overlap credit. The command latency itself is unchanged: +original Octoshock 2.3 GTE_Instruction returns DPCS cost 8 minus 1. + +psx_gte_set, psx_gte_stall and psx_gte_read now publish pending batch and local +charges before reading the clock. This is the same ownership rule already +used by the multiply/divide deadline helpers. It applies to callers using +the shared runtime, including interpreted and generated CPU paths. + +The BIOS/disc-free test_gte_deferred.c links the actual psx_cycles.c. Its +short DPCS/ADDIU/SWC2 timing case fails before the fix. It then exercises all +64 latency-table entries, prior command overlap, batch/local/both pending +charges, elapsed/deadline boundaries, read credit and unchanged store credit. +75,661 checks pass at O0 and O2. This fixture verifies shared timing helpers; +full game replays and generated/interpretation coverage remain separate gates. + +Diagnosis receipts: pepsiman-boundary1514-cpu-comparison.json, +pepsiman-source-boundary1514-11/passivity.json, +pepsiman-native-boundary1514-12/passivity.json and + gte-deferred-regression-01/{before,O0,O2}.log in the private accuracy campaign. +No frame-specific clock offset, changed input, GTE arithmetic change, observer +reduction or performance tuning is part of this correction. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index eb880682..a4ae57b9 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -775,6 +775,21 @@ target_include_directories(reachable_discovery_test PRIVATE ) target_link_libraries(reachable_discovery_test PRIVATE fmt) +# GTE deadline ownership includes pending CPU charges from both host sinks. +if(BUILD_TESTING AND CMAKE_C_COMPILER_ID STREQUAL "GNU") + foreach(_mode IN ITEMS O0 O2) + set(_target "tas_gte_deferred_${_mode}") + add_executable(${_target} + "${CMAKE_CURRENT_SOURCE_DIR}/../runtime/tests/test_gte_deferred.c" + "${CMAKE_CURRENT_SOURCE_DIR}/../runtime/src/psx_cycles.c") + target_include_directories(${_target} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../runtime/include") + target_compile_definitions(${_target} PRIVATE PSX_ENABLE_BLOCK_CYCLES=1) + target_compile_options(${_target} PRIVATE -${_mode} -UNDEBUG -ffunction-sections -fdata-sections) + target_link_options(${_target} PRIVATE -Wl,--gc-sections) + add_test(NAME ${_target} COMMAND ${_target}) + endforeach() +endif() + # ---- test-registration completeness guard ----------------------------------- if(BUILD_TESTING AND Python3_Interpreter_FOUND AND NOT MSVC) add_test(NAME gpu_upload_history diff --git a/runtime/src/psx_cycles.c b/runtime/src/psx_cycles.c index 1057093f..6859c461 100644 --- a/runtime/src/psx_cycles.c +++ b/runtime/src/psx_cycles.c @@ -703,6 +703,10 @@ void psx_muldiv_stall(CPUState* cpu) { * §1+DO_LDS that bracket this ran in the instruction's psx_cyc_step (COP2 is non-load). * MTC2/CTC2 (writes) use psx_gte_stall (stall only, no give-back). */ void psx_gte_read(CPUState* cpu, uint32_t rt) { + /* Include deferred CPU work before computing a deadline or stall. A + * later advance publishes it too, so using the old clock double-counts + * that work in the stall and can arm a command deadline too early. */ + psx_cyc_batch_flush(); if (cpu->gte_ts_done > psx_cycle_count) { uint32_t stall = (uint32_t)(cpu->gte_ts_done - psx_cycle_count); cpu->ld_absorb = stall; @@ -760,6 +764,10 @@ uint32_t psx_gte_cmd_latency(uint32_t cmd) { } void psx_gte_set(CPUState* cpu, uint32_t latency) { + /* Include deferred CPU work before computing a deadline or stall. A + * later advance publishes it too, so using the old clock double-counts + * that work in the stall and can arm a command deadline too early. */ + psx_cyc_batch_flush(); /* Back-to-back GTE ops serialize: finish the prior op first. */ if (cpu->gte_ts_done > psx_cycle_count) { psx_advance_cycles((uint32_t)(cpu->gte_ts_done - psx_cycle_count)); @@ -768,6 +776,10 @@ void psx_gte_set(CPUState* cpu, uint32_t latency) { } void psx_gte_stall(CPUState* cpu) { + /* Include deferred CPU work before computing a deadline or stall. A + * later advance publishes it too, so using the old clock double-counts + * that work in the stall and can arm a command deadline too early. */ + psx_cyc_batch_flush(); if (cpu->gte_ts_done > psx_cycle_count) { psx_advance_cycles((uint32_t)(cpu->gte_ts_done - psx_cycle_count)); } diff --git a/runtime/tests/test_gte_deferred.c b/runtime/tests/test_gte_deferred.c new file mode 100644 index 00000000..05a2d8d3 --- /dev/null +++ b/runtime/tests/test_gte_deferred.c @@ -0,0 +1,112 @@ +/* Production GTE deadline regressions. No BIOS, disc or copied guest code. */ +#include "psx_cyc.h" +#include <assert.h> +#include <stdio.h> +int g_ls_replay_active = 0; +int g_ls_mode = 0; +int g_precise_mode = 0; +int g_psx_call_bail = 0; +uint32_t i_mask = 0; +uint64_t g_guest_store_count = 0; +uint64_t g_mmio_access_count = 0; + +static uint32_t s_cd_cycles_remaining = 5; +static int s_cd_ready = 0; +static uint32_t s_dma_ready_cycles = 0; + +void sio_advance(uint32_t cycles) { (void)cycles; } + +void cdrom_advance(uint32_t cycles) { + if (s_cd_ready) return; + if (cycles >= s_cd_cycles_remaining) { + s_cd_cycles_remaining = 0; + s_cd_ready = 1; + } else { + s_cd_cycles_remaining -= cycles; + } +} + +void dma_advance(uint32_t cycles) { + if (s_cd_ready) s_dma_ready_cycles += cycles; +} + +void timers_advance(uint32_t cycles) { (void)cycles; } +void interrupts_advance_cycles(uint32_t cycles) { (void)cycles; } +void interrupts_service_scheduled_events(void) {} + +uint32_t interrupts_cycles_to_vblank(void) { return UINT32_MAX; } +uint32_t timers_cycles_to_irq(uint32_t mask) { (void)mask; return UINT32_MAX; } +uint32_t cdrom_cycles_to_irq(uint32_t mask) { + (void)mask; + return s_cd_ready ? UINT32_MAX : s_cd_cycles_remaining; +} +uint32_t dma_cycles_to_internal_event(void) { return UINT32_MAX; } +void source_gpu_runtime_advance(void) {} +uint32_t source_gpu_runtime_cycles_to_event(void) { return UINT32_MAX; } +uint32_t dma_cycles_to_deliverable_irq(uint32_t mask) { + (void)mask; + return UINT32_MAX; +} +uint32_t sio_cycles_to_irq(uint32_t mask) { (void)mask; return UINT32_MAX; } +/* SPU sample-event scheduler (golden 1a973806): psx_cycles.c consults it; stub here. */ +static uint32_t s_spu_next_sample = UINT32_MAX; +uint32_t psx_spu_sample_event_cycles_to_next(void) { return s_spu_next_sample; } +void psx_spu_sample_event_service(void) {} +int psx_get_in_exception(void) { return 0; } + +void starvation_watchdog_check(void) {} +void starvation_ring_pc_sample(void) {} + +int psx_netplay_active(void) { return 0; } +int psx_selfcheck_enabled(void) { return 0; } +void dirty_ram_ld_delay_discard(void) {} +void dirty_ram_irq_ambient_resync_after_restore(void) {} + + +static uint32_t local; +static unsigned checks; +#define CHECK(x) do {checks++;if(!(x)){fprintf(stderr,"FAIL line%d: %s\n",__LINE__,#x);return 1;}}while(0) +static void reset(unsigned elapsed,unsigned batch,unsigned local_value) { + psx_cycles_reset_for_boot();s_cd_ready=1;s_spu_next_sample=UINT32_MAX; + psx_advance_cycles(1000+elapsed); + psx_next_service_cycle=UINT64_MAX; + g_psx_cyc_batch=batch;g_psx_cyc_batch_limit=64;g_psx_cyc_bb_defer=1; + local=local_value;g_psx_cyc_local_acc=local_value?&local:NULL; +} +int main(void) { + CPUState cpu={0}; + /* Authored timing equivalent of DPCS / ADDIU / SWC2 with two remaining + * load-overlap credits. Source return(ret-1) gives a seven-cycle deadline. + * SWC2's one pending base cycle must be part of its stall comparison. */ + reset(0,0,0);cpu.read_absorb_which=23;cpu.read_absorb[23]=2; + cpu.ld_which_t=32;cpu.ld_absorb=5; + psx_cyc_step(&cpu,0);psx_gte_set(&cpu,psx_gte_cmd_latency(0x10)); + psx_cyc_step(&cpu,(1u<<2)|(1u<<14));psx_cyc_step(&cpu,0); + psx_gte_stall(&cpu);psx_cyc_batch_flush(); + CHECK(psx_cycle_count==1007); + /* Every command latency, pending source (batch/local/both), and prior + * command overlap. Deadline ownership is max(published time,old deadline). */ + for(unsigned cmd=0;cmd<64;cmd++)for(unsigned pending=0;pending<=64;pending++) + for(unsigned mode=0;mode<3;mode++)for(unsigned lead=0;lead<=43;lead+=43) { + reset(0,mode==1?0:pending,mode==0?0:pending);cpu.gte_ts_done=1000+lead; + unsigned elapsed=(mode==2?2:1)*pending; + uint64_t at=1000+(elapsed>lead?elapsed:lead); + psx_gte_set(&cpu,psx_gte_cmd_latency(cmd)); + CHECK(psx_cycle_count==at && cpu.gte_ts_done==at+psx_gte_cmd_latency(cmd)); + CHECK(g_psx_cyc_batch==0 && local==0); + } + /* Reads additionally return only the actual stall as the new load credit. + * Ordinary writes/stores preserve the existing credit and delayed slot. */ + for(unsigned elapsed=0;elapsed<=64;elapsed++)for(unsigned pending=0;pending<=elapsed;pending++) + for(unsigned mode=0;mode<2;mode++)for(unsigned read=0;read<2;read++) { + memset(&cpu,0,sizeof(cpu));cpu.gte_ts_done=1043;cpu.ld_absorb=19;cpu.ld_which_t=32; + cpu.read_absorb_which=5;cpu.read_absorb[5]=11; + reset(elapsed-pending,mode?0:pending,mode?pending:0); + if(read)psx_gte_read(&cpu,7);else psx_gte_stall(&cpu); + unsigned stall=elapsed<43?43-elapsed:0; + CHECK(psx_cycle_count==1000+elapsed+stall); + CHECK(cpu.ld_absorb==(read?stall:19) && cpu.ld_which_t==(read?7:32)); + CHECK(cpu.read_absorb[5]==11 && g_psx_cyc_batch==0 && local==0); + } + printf("PASS %u production GTE pending-charge/deadline checks\n",checks);return 0; +} From 4a792379a128d9d96173960e86da4786e69ca115 Mon Sep 17 00:00:00 2001 From: Kevin <9866117+kerokline@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:54:28 -0400 Subject: [PATCH 37/56] compile_overlays: only reconcile empty primary scans in DLL mode 3a174fab added reconcile_empty_primary_scans() at the tail of main() and called it unconditionally with cache_dir. cache_dir is bound only on the DLL path; --static never assigns it. So every --static run now ends in UnboundLocalError: cannot access local variable 'cache_dir' where it is not associated with a value after generated/overlays_static.c and all its translation units have already been written. The output is complete, but the process exits 1, and callers that gate on the compiler's exit (axis_b_loop.sh expects 0 or 2) abort before the runtime is rebuilt. Found regenerating Breath of Fire III against master 193a60b8. The reconciliation has nothing to do in static mode: _do_capture hands each capture to static_capture_job and returns before the only pending_empty_primary.append(), and the check it performs reads the per-game DLL cache, which static mode does not have. Guard the call on the non-static path. In static mode, raise instead of silently skipping if anything was ever queued, so a future change that routes static captures through the DLL path fails loudly rather than dropping a root. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- recompiler/CMakeLists.txt | 9 +++ tools/compile_overlays.py | 20 +++++- .../test_compile_overlays_static_tail.py | 67 +++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 tools/tests/test_compile_overlays_static_tail.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index a4ae57b9..60f7f243 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -600,6 +600,15 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_aot_overlay_pipeline.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) endif() + # --static must walk the whole of compile_overlays.main() and exit 0: + # a DLL-only reconciliation at its tail once read an unbound cache_dir + # and failed every static run after its output was written. + add_test( + NAME compile_overlays_static_tail + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_compile_overlays_static_tail.py + --recompiler $<TARGET_FILE:psxrecomp-game> + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) add_test( NAME fmv_cycle_batch_codegen COMMAND ${Python3_EXECUTABLE} diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 1b754c30..09ba2016 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -7619,9 +7619,23 @@ def main(): # that pass, using ABI-valid, current-byte guarded native coverage. Missing # even one primary root remains a failure; toolchain/audit failures above # are never cleared by this reconciliation. - reconcile_empty_primary_scans( - pending_empty_primary, cache_dir, - overlay_abi_tag(args.runtime_include, args.flavor), stats) + # + # DLL mode only. The reconciliation reads the per-game DLL cache, and + # cache_dir is bound only on the non-static path; --static emits one + # self-contained overlays_static.c and _do_capture returns into + # static_capture_job before anything is queued. Calling it unconditionally + # raised UnboundLocalError at the very end of every --static run, after + # the output was already written, and turned a clean compile into exit 1. + if args.static: + if pending_empty_primary: + raise SystemExit( + 'internal error: --static queued %d empty-primary ' + 'reconciliation(s), but static mode has no DLL cache to ' + 'reconcile against' % len(pending_empty_primary)) + else: + reconcile_empty_primary_scans( + pending_empty_primary, cache_dir, + overlay_abi_tag(args.runtime_include, args.flavor), stats) # LOUD summary + machine-readable result line, then a non-zero exit when any # shard that should have built failed. The runtime's autocompile watcher and diff --git a/tools/tests/test_compile_overlays_static_tail.py b/tools/tests/test_compile_overlays_static_tail.py new file mode 100644 index 00000000..656f62e6 --- /dev/null +++ b/tools/tests/test_compile_overlays_static_tail.py @@ -0,0 +1,67 @@ +"""--static must reach the end of main() and exit cleanly. + +Regression for 3a174fab: reconcile_empty_primary_scans() was called at the +tail of main() with cache_dir, a local bound only on the DLL path. Every +--static run then died with UnboundLocalError after its output was already +written, exiting 1, and axis_b_loop.sh (which accepts only 0 or 2) aborted +before the runtime rebuild. + +An empty capture list is the smallest input that walks the whole of main() +in static mode, so no game assets are involved. The recompiler binary is +still required: compile_overlays asks it for --codegen-hash as a staleness +check before doing anything else. CTest passes it the same way the other +overlay codegen tests get it. + +Usage: test_compile_overlays_static_tail.py --recompiler <psxrecomp-game> +""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +TOOLS = Path(__file__).resolve().parents[1] +ROOT = TOOLS.parent +RECOMPILER = None + + +class StaticTailTest(unittest.TestCase): + def test_static_run_with_no_captures_exits_zero(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + caps = tmp / "captures.json" + caps.write_text(json.dumps([]), encoding="utf-8") + toml = tmp / "game.toml" + toml.write_text('[game]\nid = "TEST-00000"\nname = "static tail"\n', + encoding="utf-8") + out = tmp / "out" + proc = subprocess.run( + [sys.executable, str(TOOLS / "compile_overlays.py"), + "--static", + "--captures", str(caps), + "--out-dir", str(out), + "--game-toml", str(toml), + "--recompiler", RECOMPILER, + "--runtime-include", str(ROOT / "runtime" / "include")], + capture_output=True, text=True, timeout=120) + detail = proc.stdout[-1500:] + proc.stderr[-1500:] + # The contract is only that main() runs to its end. With nothing + # captured there is legitimately no overlays_static.c to write, + # so the output file is not part of it. + self.assertNotIn("UnboundLocalError", detail, detail) + self.assertIn("Done.", detail, detail) + self.assertEqual(proc.returncode, 0, detail) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--recompiler", required=True, + help="psxrecomp-game binary (compile_overlays calls it " + "for --codegen-hash)") + args, rest = ap.parse_known_args() + if not Path(args.recompiler).is_file(): + sys.exit("no recompiler at %s" % args.recompiler) + RECOMPILER = args.recompiler + unittest.main(argv=[sys.argv[0]] + rest) From d17676a5c724918037824ec2fc9e3eb861559e0d Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 11:49:59 +0300 Subject: [PATCH 38/56] setup: read .chd through libchdr in the Python disc tools The runtime mounts a .chd natively and the launcher lists it in the disc picker, so a player could select one and see a valid disc identity, then have first-run setup fail: verify-disc, prepare_disc.py and probe_disc.py hash and parse raw 2352-byte track data, which a CHD stores compressed. Nothing on the Python side could read it, and telling players to install MAME for chdman is not an answer. libchdr is already vendored and compiled for the runtime and the native media CLI. Build it once more as a shared library in recompiler/ (CMake target chdr, output next to the emitters, static CRT on Windows so a plain python.exe can load it) and read the CHD from Python through ctypes. tools/psx_chd.py mirrors the layout the runtime's iso_reader.cpp uses -- 2448-byte frames, 4-frame track padding, stored versus virtual pregaps, big-endian audio -- and reproduces the .bin files a Redump dump of the same disc contains, in either of Redump's two layouts (one bin per track, or one bin for the disc). verify-disc hashes those reconstructed track bytes against [prepare_disc] instead of the container. prepare_disc.py writes the tracks into out_dir in whichever layout the recorded digests match and then continues down its normal cue path, so the data track is written once, SBI companions are looked up next to the CHD, and the receipt names the CHD the player chose. probe_disc.py extracts to a scratch directory and probes the cue. The CLI builds target chdr on demand for a kit whose emitters predate it, and falls back to the extract-it-first message when it cannot. Verified against two discs from a Redump library converted with chdman: a single-track MODE2 disc and a four-track disc with stored audio pregaps both reproduce their catalogued track-1 size, MD5 and SHA-1 exactly; verify-disc and prepare_disc.py accept the four-track one end to end. Tests build a synthetic uncompressed v5 CHD around a cooked ISO image, so no retail data is needed; the decode tests skip with a reason when no libchdr is built. --- docs/config_schema.md | 5 +- psxrecomp_cli.py | 116 +++- recompiler/CMakeLists.txt | 21 + tools/new_project_layout/probe_disc.py | 31 +- tools/prepare_disc.py | 58 ++ tools/psx_chd.py | 720 +++++++++++++++++++++++++ tools/tests/test_psx_chd.py | 306 +++++++++++ 7 files changed, 1251 insertions(+), 6 deletions(-) create mode 100644 tools/psx_chd.py create mode 100644 tools/tests/test_psx_chd.py diff --git a/docs/config_schema.md b/docs/config_schema.md index b675789b..4fa6f861 100644 --- a/docs/config_schema.md +++ b/docs/config_schema.md @@ -93,7 +93,10 @@ online Create/Join require `netplay_ok` (and online also a clean verify + non-empty `disc_fp`). Mirror `required_tracks` in the Retro catalog as `rom_identity.track_counts` so the hub library scan rejects Track-01-only dumps. Wizard / Retro / catalog submission accept Redump `.cue` + sibling `.bin` -tracks only — not `.iso`/`.chd` (cannot reliably expand to multi-track). +tracks, or a MAME-compatible `.chd` of the same dump: `tools/psx_chd.py` reads +it through the libchdr the emitters build (CMake target `chdr`) and writes the +Redump-shaped track files back, so `[prepare_disc]` digests, SBI companions and +`required_tracks` apply unchanged. A bare `.iso` cannot expand to multi-track. ## Program / game block diff --git a/psxrecomp_cli.py b/psxrecomp_cli.py index cc593f83..c8049c2c 100755 --- a/psxrecomp_cli.py +++ b/psxrecomp_cli.py @@ -30,6 +30,7 @@ ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(ROOT / "tools")) from sdk_progress import ProgressReporter # noqa: E402 from disc_companion import CompanionError, inspect_companion # noqa: E402 +import psx_chd # noqa: E402 from toolchain_pack import ( # noqa: E402 ensure_toolchain as _ensure_toolchain_pack, resolve_toolchain_bin, @@ -836,16 +837,105 @@ def load_sections(config: Path) -> dict[str, dict[str, Any]]: return parse_toml_simple(config.read_text(encoding="utf-8")) +def ensure_chd_reader( + project_root: Path, progress: ProgressReporter, *, build: bool = True +) -> Optional[Path]: + """Return the shared libchdr the Python tools read .chd through. + + recompiler/ builds it next to the emitters (CMake target ``chdr``). A kit + whose emitters predate that target has no reader; the build attempt is + best-effort and a miss degrades to the extract-it-first message. + """ + lib = psx_chd.find_libchdr(project_root, ROOT) + if lib is not None or not build: + return lib + progress.log("CHD reader (libchdr) not built yet; building target chdr") + try: + _build_recompiler_targets(project_root, progress, ("chdr",)) + except Exception as exc: # noqa: BLE001 + progress.log(f"CHD reader build failed: {exc}") + return None + lib = psx_chd.find_libchdr(project_root, ROOT) + if lib is not None: + progress.log(f"CHD reader ready: {lib}") + return lib + + +def _verify_chd( + disc: Path, + prep: dict[str, Any], + *, + skip_hash: bool, + progress: ProgressReporter, + chd_lib: Optional[Path], +) -> dict[str, Any]: + """verify_disc_path for a .chd: reproduce the Redump track bytes through + libchdr and check those digests, never the compressed container's.""" + lib_path = chd_lib or psx_chd.find_libchdr(None, ROOT) + if lib_path is None: + raise DiscVerifyError(psx_chd.unsupported_message(disc)) + try: + with psx_chd.ChdDisc(disc, psx_chd.LibChdr(lib_path)) as chd: + progress.log( + f"{disc.name}: {len(chd.tracks)} track(s), reading through {lib_path.name}" + ) + chd_digests = psx_chd.digests(chd) + except psx_chd.ChdError as exc: + raise DiscVerifyError(str(exc)) from exc + sizes = [int(s) for s in (prep.get("known_sizes") or [])] + md5s = [str(x).lower() for x in (prep.get("known_md5") or [])] + sha1s = [str(x).lower() for x in (prep.get("known_sha1") or [])] + layout = chd_digests.layout_matching(sizes, md5s, sha1s) + chosen = chd_digests.disc if layout == "single" else chd_digests.first_track + try: + subchannel, _ = inspect_companion(disc, chosen.size, chosen.sha1) + except CompanionError as exc: + raise DiscVerifyError(str(exc)) from exc + identity = { + "path": str(disc), + "md5": chosen.md5, + "sha1": chosen.sha1, + "size": chosen.size, + "verified": False, + "subchannel": subchannel, + "chd": {"layout": layout or "multi", "tracks": len(chd_digests.tracks)}, + } + progress.event("disc", **identity) + if not md5s and not sha1s and not sizes: + identity["verified"] = True + return identity + if skip_hash: + return identity + if layout is None: + first = chd_digests.first_track + whole = chd_digests.disc + raise DiscVerifyError( + f"CHD track digests not in prepare_disc.known_* " + f"(track 1: size={first.size} md5={first.md5} sha1={first.sha1}; " + f"whole disc: size={whole.size} md5={whole.md5} sha1={whole.sha1})" + ) + identity["verified"] = True + return identity + + def verify_disc_path( disc: Path, prep: dict[str, Any], *, skip_hash: bool, progress: ProgressReporter, + chd_lib: Optional[Path] = None, ) -> dict[str, Any]: path = disc.resolve() if path.suffix.lower() == ".cue": path = resolve_cue_bin(path) + elif path.suffix.lower() == ".chd": + # prepare_disc records digests of the uncompressed track data. A CHD + # is a compressed container, so it is read back through libchdr into + # the same bytes a Redump .bin holds and those are what get hashed. + return _verify_chd( + path, prep, skip_hash=skip_hash, progress=progress, chd_lib=chd_lib + ) md5, sha1, size = file_hashes(path) try: subchannel, _ = inspect_companion(disc, size, sha1) @@ -906,9 +996,13 @@ def cmd_verify_disc(args: argparse.Namespace, progress: ProgressReporter) -> int secs = load_sections(config) prep = secs.get("prepare_disc") or {} progress.phase("verify", pct=0.1, message=f"Verifying {disc.name}") + chd_lib = None + if disc.suffix.lower() == ".chd": + chd_lib = ensure_chd_reader(project_root, progress) try: identity = verify_disc_path( - disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress + disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress, + chd_lib=chd_lib, ) except DiscVerifyError as exc: progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True) @@ -924,6 +1018,8 @@ def run_prepare_disc( config: Path, source: Path, progress: ProgressReporter, + *, + chd_lib: Optional[Path] = None, ) -> Path: script = ROOT / "tools" / "prepare_disc.py" if not script.is_file(): @@ -938,7 +1034,13 @@ def run_prepare_disc( str(project_root), str(source), ] - proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True, encoding="utf-8", errors="replace") + env = dict(os.environ) + if chd_lib is not None: + env[psx_chd.LIB_ENV] = str(chd_lib) + proc = subprocess.run( + cmd, cwd=str(project_root), capture_output=True, text=True, + encoding="utf-8", errors="replace", env=env, + ) out = (proc.stdout or "") + (proc.stderr or "") for line in out.splitlines(): if line.strip(): @@ -987,10 +1089,14 @@ def cmd_generate(args: argparse.Namespace, progress: ProgressReporter) -> int: progress.log(f"generate --disc {disc}") progress.phase("verify", pct=0.05, message=f"Checking disc {disc.name}") + chd_lib = None + if disc.suffix.lower() == ".chd" and disc.is_file(): + chd_lib = ensure_chd_reader(project_root, progress) try: if disc.is_file(): verify_disc_path( - disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress + disc, prep, skip_hash=bool(args.skip_hash_check), progress=progress, + chd_lib=chd_lib, ) except DiscVerifyError as exc: progress.error(str(exc), code=EXIT_VERIFY, verify_failed=True) @@ -1007,7 +1113,9 @@ def cmd_generate(args: argparse.Namespace, progress: ProgressReporter) -> int: ) if need_prep or args.force_prepare: try: - working_disc = run_prepare_disc(project_root, config, disc, progress) + working_disc = run_prepare_disc( + project_root, config, disc, progress, chd_lib=chd_lib + ) except Exception as exc: # noqa: BLE001 progress.error(str(exc), code=EXIT_ERROR) return EXIT_ERROR diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index a4ae57b9..6f65c7ed 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -18,7 +18,23 @@ option(PSXRECOMP_ENABLE_CHD "Build the psxrecomp media CLI (requires libchdr via FetchContent)" ON) if(PSXRECOMP_ENABLE_CHD) + # The Python disc tools (psxrecomp_cli.py verify-disc, tools/prepare_disc.py, + # tools/new_project_layout/probe_disc.py) read .chd through this same + # decoder via ctypes, so libchdr is also built as a shared library, placed + # next to the emitters where tools/psx_chd.py looks for it. BUILD_SHARED_LIBS + # is scoped to that include: libchdr's codecs are declared STATIC and + # chdr-static (what psxrecomp links) is unaffected; nothing else in this + # project turns shared. + set(_psx_saved_build_shared "${BUILD_SHARED_LIBS}") + set(BUILD_SHARED_LIBS ON) include("${CMAKE_CURRENT_SOURCE_DIR}/../runtime/chd_dependency.cmake") + set(BUILD_SHARED_LIBS "${_psx_saved_build_shared}") + unset(_psx_saved_build_shared) + if(TARGET chdr) + set_target_properties(chdr PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + endif() endif() add_subdirectory(lib/fmt) @@ -268,6 +284,11 @@ if(PSXRECOMP_STATIC_CLI AND WIN32 AND (MINGW OR CMAKE_CXX_COMPILER_ID MATCHES "C if(PSXRECOMP_ENABLE_CHD) list(APPEND _cli_targets psxrecomp) endif() + if(TARGET chdr) + # The Python tools load this with ctypes from a plain python.exe, with + # no toolchain bin/ on PATH, so it must not pull libgcc or libwinpthread. + list(APPEND _cli_targets chdr) + endif() foreach(_cli_target IN LISTS _cli_targets) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_link_options(${_cli_target} PRIVATE diff --git a/tools/new_project_layout/probe_disc.py b/tools/new_project_layout/probe_disc.py index 234a6a1a..5526fb23 100755 --- a/tools/new_project_layout/probe_disc.py +++ b/tools/new_project_layout/probe_disc.py @@ -724,7 +724,36 @@ def main() -> int: print(f"cue not found: {cue}", file=sys.stderr) return 1 - p = probe(cue, identity_only=args.identity_only) + # A .chd is read back into Redump-shaped tracks (one bin per track) in a + # scratch directory and probed as that cue, so identity, digests and the + # boot EXE come out exactly as they would from the dump it was made from. + scratch = None + if cue.suffix.lower() == ".chd": + import tempfile + + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + import psx_chd + + fw_root = Path(__file__).resolve().parents[2] + lib_path = psx_chd.find_libchdr(None, fw_root) + if lib_path is None: + print(psx_chd.unsupported_message(cue), file=sys.stderr) + return 1 + scratch = tempfile.TemporaryDirectory(prefix="probe-chd-") + try: + with psx_chd.ChdDisc(cue, psx_chd.LibChdr(lib_path)) as disc: + print(f" chd: {len(disc.tracks)} track(s) via {lib_path.name}", file=sys.stderr) + cue = psx_chd.extract(disc, Path(scratch.name), f"{cue.stem}.cue", layout="multi") + except psx_chd.ChdError as exc: + print(str(exc), file=sys.stderr) + scratch.cleanup() + return 1 + + try: + p = probe(cue, identity_only=args.identity_only) + finally: + if scratch is not None: + scratch.cleanup() if args.display_name: p.display_name = args.display_name diff --git a/tools/prepare_disc.py b/tools/prepare_disc.py index 40a7aa72..92a321ca 100644 --- a/tools/prepare_disc.py +++ b/tools/prepare_disc.py @@ -32,6 +32,7 @@ from dataclasses import dataclass, field from pathlib import Path from disc_companion import CompanionError, check_destination, inspect_companion, stage_companion +import psx_chd DST_SEC = 2352 SRC_2448 = 2448 @@ -468,6 +469,55 @@ def raw2448_to_bin(src: Path) -> bytes: return bytes(out) +class ChdStageError(Exception): + """A CHD could not be staged; main() prints it and returns 1.""" + + +def stage_from_chd(chd: Path, cfg: PrepareConfig, fw_root: Path) -> Path: + """Write the Redump-shaped .bin/.cue a CHD compresses into out_dir. + + Returns the cue, which the normal cue path then consumes in place: the + data track is already where prepare would copy it, so nothing is copied + twice. The layout (one bin per track, or one bin for the disc) is the one + whose digests [prepare_disc] recorded; with no digests, one bin per track. + """ + lib_path = psx_chd.find_libchdr(cfg.project_root, fw_root) + if lib_path is None: + raise ChdStageError(psx_chd.unsupported_message(chd)) + try: + lib = psx_chd.LibChdr(lib_path) + except psx_chd.ChdError as exc: + raise ChdStageError(str(exc)) from exc + sizes = [k.size for k in cfg.known if k.size] + md5s = [k.md5 for k in cfg.known if k.md5] + sha1s = [k.sha1 for k in cfg.known if k.sha1] + try: + with psx_chd.ChdDisc(chd, lib) as disc: + print(f"source chd: {chd} ({len(disc.tracks)} track(s), via {lib_path.name})") + for t in disc.tracks: + print(f" track {t.number:02d} {t.type} frames={t.frames}") + digests = psx_chd.digests(disc) + for i, d in enumerate(digests.tracks, 1): + print(f" track {i:02d}: size={d.size} md5={d.md5} sha1={d.sha1}") + layout = digests.layout_matching(sizes, md5s, sha1s) + if layout is None: + if cfg.known and not cfg.skip_hash_check: + raise ChdStageError( + "CHD track digests are not in prepare_disc.known_* " + "(pass --skip-hash-check to force)" + ) + layout = "multi" + print(f" layout: {layout}") + cue = psx_chd.extract( + disc, cfg.out_dir, cfg.cue_name, layout=layout, + bin_name=cfg.bin_name, + ) + except psx_chd.ChdError as exc: + raise ChdStageError(str(exc)) from exc + print(f"wrote {cue}") + return cue + + def matches_known(cfg: PrepareConfig, size: int, md5: str, sha1: str) -> bool: if not cfg.known: return False @@ -553,6 +603,14 @@ def main() -> int: return 1 selected_image = src + if src.suffix.lower() == ".chd": + # The companion lookup and the receipt keep naming the CHD the player + # chose; only the track data is materialised. + try: + src = stage_from_chd(src, cfg, fw_root) + except ChdStageError as exc: + print(str(exc), file=sys.stderr) + return 1 cue_src: Path | None = None cue_bins: list[Path] = [] if src.suffix.lower() == ".cue": diff --git a/tools/psx_chd.py b/tools/psx_chd.py new file mode 100644 index 00000000..dcea945a --- /dev/null +++ b/tools/psx_chd.py @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 +"""Read MAME-compatible .chd disc images from Python through libchdr. + +The runtime mounts .chd natively, so a player can pick one in the launcher and +see a valid disc identity. The Python side of setup -- ``verify-disc``, +``tools/prepare_disc.py`` and ``probe_disc.py`` -- hashes and parses raw +2352-byte track data, which a CHD stores compressed. This module bridges the +two: it loads the libchdr shared library that ``recompiler/`` builds next to +the emitters and reproduces, byte for byte, the ``.bin`` files a Redump dump +of the same disc would contain. Nothing here decompresses on its own; libchdr +carries the LZMA, zlib, FLAC and zstd codecs a real CHD needs, and it is the +same decoder the runtime links. + +Layout facts mirrored from ``runtime/src/iso_reader.cpp`` (the runtime is the +reference for how a CHD maps to disc sectors; keep the two in step): + +* every CD frame is 2448 bytes: 2352 of sector data followed by 96 of subcode; +* a track's frames are stored consecutively, then the next track starts on a + 4-frame boundary; +* a ``PREGAP`` whose ``PGTYPE`` starts with ``V`` is stored inside the track's + ``FRAMES``; any other pregap is virtual and occupies no frames; +* audio frames are stored big-endian and are byte-swapped to the little-endian + PCM a ``.bin`` holds. + +Two output layouts exist because Redump ships both: one ``.bin`` per track +(what the runtime and ``prepare_disc`` expect for CDDA titles) or a single +``.bin`` holding every track. ``[prepare_disc]`` digests identify which one a +title was catalogued from; :func:`digests` computes both in one pass so the +caller can pick. + +Python 3.9 compatible on purpose: the kits run on whatever python3 the +player's distribution ships. +""" + +from __future__ import annotations + +import ctypes +import hashlib +import os +import re +import sys +from array import array +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Iterator, Optional, Sequence + +CD_FRAME_SIZE = 2448 +RAW_SECTOR_SIZE = 2352 +SUBCODE_SIZE = 96 +CD_TRACK_PADDING = 4 +CD_MAX_TRACKS = 99 + +CHD_OPEN_READ = 1 +CHDERR_NONE = 0 +CHDERR_METADATA_NOT_FOUND = 19 # enum _chd_error in libchdr/chd.h + + +def _make_tag(a: str, b: str, c: str, d: str) -> int: + return (ord(a) << 24) | (ord(b) << 16) | (ord(c) << 8) | ord(d) + + +CDROM_TRACK_METADATA_TAG = _make_tag("C", "H", "T", "R") +CDROM_TRACK_METADATA2_TAG = _make_tag("C", "H", "T", "2") + +# Track types chdman writes, mapped to the cue mode that names the same +# 2352-byte frame. Cooked types (MODE1, MODE2, MODE2_FORM1, ...) store fewer +# than 2352 bytes of payload per frame and have no Redump .bin equivalent, so +# they are refused rather than guessed at; PS1 dumps are MODE2_RAW + AUDIO. +_CUE_MODE = { + "MODE1_RAW": "MODE1/2352", + "MODE2_RAW": "MODE2/2352", + "AUDIO": "AUDIO", +} + +_LIB_NAMES = { + "win32": ("libchdr.dll", "chdr.dll"), + "darwin": ("libchdr.dylib", "libchdr.0.dylib", "libchdr.0.3.dylib"), +} +_LIB_NAMES_POSIX = ("libchdr.so", "libchdr.so.0", "libchdr.so.0.3") +LIB_ENV = "PSXRECOMP_LIBCHDR" + + +class ChdError(Exception): + """A CHD could not be opened, is not a CD image, or is not supported.""" + + +# --------------------------------------------------------------------------- +# libchdr binding +# --------------------------------------------------------------------------- + +_CHD_MD5_BYTES = 16 +_CHD_SHA1_BYTES = 20 + + +class _ChdHeader(ctypes.Structure): + """Mirror of ``struct _chd_header`` in ``include/libchdr/chd.h``. + + Only the leading fields are read; the rest exist so the struct has the + right size and the pointer field lands on its natural alignment. + """ + + _fields_ = [ + ("length", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("flags", ctypes.c_uint32), + ("compression", ctypes.c_uint32 * 4), + ("hunkbytes", ctypes.c_uint32), + ("totalhunks", ctypes.c_uint32), + ("logicalbytes", ctypes.c_uint64), + ("metaoffset", ctypes.c_uint64), + ("mapoffset", ctypes.c_uint64), + ("md5", ctypes.c_uint8 * _CHD_MD5_BYTES), + ("parentmd5", ctypes.c_uint8 * _CHD_MD5_BYTES), + ("sha1", ctypes.c_uint8 * _CHD_SHA1_BYTES), + ("rawsha1", ctypes.c_uint8 * _CHD_SHA1_BYTES), + ("parentsha1", ctypes.c_uint8 * _CHD_SHA1_BYTES), + ("unitbytes", ctypes.c_uint32), + ("unitcount", ctypes.c_uint64), + ("hunkcount", ctypes.c_uint32), + ("mapentrybytes", ctypes.c_uint32), + ("rawmap", ctypes.c_void_p), + ("obsolete_cylinders", ctypes.c_uint32), + ("obsolete_sectors", ctypes.c_uint32), + ("obsolete_heads", ctypes.c_uint32), + ("obsolete_hunksize", ctypes.c_uint32), + ] + + +def candidate_dirs(project_root: Optional[Path], framework_root: Optional[Path]) -> list[Path]: + """Directories the shared library may sit in, most specific first. + + Same order as ``psxrecomp_cli._find_recompiler_tool`` so the library is + found wherever the emitters were. + """ + dirs: list[Path] = [] + if project_root is not None: + dirs += [ + project_root / "psxrecomp" / "recompiler" / "build", + project_root / "psxrecomp" / "recompiler" / "build" / "Release", + project_root / "build-recompiler", + ] + if framework_root is not None: + dirs += [ + framework_root / "recompiler" / "build", + framework_root / "recompiler" / "build" / "Release", + ] + return dirs + + +def library_names() -> tuple[str, ...]: + return _LIB_NAMES.get(sys.platform, _LIB_NAMES_POSIX) + + +def find_libchdr( + project_root: Optional[Path] = None, + framework_root: Optional[Path] = None, +) -> Optional[Path]: + """Locate the shared libchdr, honouring ``PSXRECOMP_LIBCHDR`` first.""" + env = os.environ.get(LIB_ENV, "").strip() + if env: + p = Path(env).expanduser() + if p.is_file(): + return p.resolve() + for d in candidate_dirs(project_root, framework_root): + for name in library_names(): + c = d / name + if c.is_file(): + return c.resolve() + if d.is_dir() and sys.platform not in _LIB_NAMES: + # Versioned .so names vary with libchdr's own version. + for c in sorted(d.glob("libchdr.so*")): + if c.is_file(): + return c.resolve() + return None + + +class LibChdr: + """Loaded libchdr with the prototypes this module uses.""" + + def __init__(self, path: Path): + self.path = Path(path) + try: + self._lib = ctypes.CDLL(str(self.path)) + except OSError as exc: + raise ChdError(f"cannot load libchdr from {self.path}: {exc}") from exc + lib = self._lib + try: + lib.chd_open.argtypes = [ + ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), + ] + lib.chd_open.restype = ctypes.c_int + lib.chd_close.argtypes = [ctypes.c_void_p] + lib.chd_close.restype = None + lib.chd_get_header.argtypes = [ctypes.c_void_p] + lib.chd_get_header.restype = ctypes.POINTER(_ChdHeader) + lib.chd_read.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p] + lib.chd_read.restype = ctypes.c_int + lib.chd_get_metadata.argtypes = [ + ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p, + ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint32), + ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint8), + ] + lib.chd_get_metadata.restype = ctypes.c_int + lib.chd_error_string.argtypes = [ctypes.c_int] + lib.chd_error_string.restype = ctypes.c_char_p + except AttributeError as exc: + raise ChdError(f"{self.path} does not export the libchdr API: {exc}") from exc + + def error_string(self, err: int) -> str: + s = self._lib.chd_error_string(err) + return s.decode("ascii", "replace") if s else f"chd error {err}" + + @property + def raw(self) -> ctypes.CDLL: + return self._lib + + +def _open_arg_candidates(path: Path) -> list[bytes]: + """Byte strings to try with ``chd_open``, which ends in ``fopen``. + + On Windows that is the ANSI code page. A path the code page cannot spell + (a title with an accented letter under a Greek locale, say) is retried + through its 8.3 short name, which is always ASCII. + """ + cands: list[bytes] = [] + try: + cands.append(os.fsencode(str(path))) + except UnicodeError: + pass + if sys.platform == "win32": + try: + mbcs = str(path).encode("mbcs") + if mbcs not in cands: + cands.append(mbcs) + except UnicodeError: + pass + try: + buf = ctypes.create_unicode_buffer(32768) + n = ctypes.windll.kernel32.GetShortPathNameW(str(path), buf, len(buf)) # type: ignore[attr-defined] + if 0 < n < len(buf): + short = buf.value.encode("mbcs") + if short not in cands: + cands.append(short) + except (AttributeError, OSError, UnicodeError): + pass + return cands + + +# --------------------------------------------------------------------------- +# Track table +# --------------------------------------------------------------------------- + +@dataclass +class ChdTrack: + number: int + type: str + subtype: str + frames: int + pregap: int = 0 + pgtype: str = "" + pgsub: str = "" + postgap: int = 0 + # Derived layout, filled by build_track_table(). + stored_pregap: int = 0 + virtual_pregap: int = 0 + chd_frame_start: int = 0 + disc_lba: int = 0 # first disc sector, including any virtual pregap + + @property + def is_audio(self) -> bool: + return self.type == "AUDIO" + + @property + def cue_mode(self) -> str: + try: + return _CUE_MODE[self.type] + except KeyError: + raise ChdError( + f"track {self.number} is {self.type}; only MODE1_RAW, MODE2_RAW " + f"and AUDIO tracks have a 2352-byte .bin form" + ) from None + + @property + def data_bytes(self) -> int: + return self.frames * RAW_SECTOR_SIZE + + +_META_V2 = re.compile( + r"TRACK:(\d+) TYPE:(\S+) SUBTYPE:(\S+) FRAMES:(\d+) PREGAP:(\d+) " + r"PGTYPE:(\S+) PGSUB:(\S+) POSTGAP:(\d+)" +) +_META_V1 = re.compile(r"TRACK:(\d+) TYPE:(\S+) SUBTYPE:(\S+) FRAMES:(\d+)") + + +def parse_track_metadata(text: str, *, v2: bool) -> ChdTrack: + """Parse one ``CHT2`` (v2) or ``CHTR`` (v1) metadata string.""" + text = text.rstrip("\0").strip() + if v2: + m = _META_V2.match(text) + if not m: + raise ChdError(f"malformed CHT2 track metadata: {text!r}") + return ChdTrack( + number=int(m.group(1)), type=m.group(2), subtype=m.group(3), + frames=int(m.group(4)), pregap=int(m.group(5)), pgtype=m.group(6), + pgsub=m.group(7), postgap=int(m.group(8)), + ) + m = _META_V1.match(text) + if not m: + raise ChdError(f"malformed CHTR track metadata: {text!r}") + return ChdTrack( + number=int(m.group(1)), type=m.group(2), subtype=m.group(3), + frames=int(m.group(4)), + ) + + +def build_track_table(tracks: Sequence[ChdTrack], *, v2: Sequence[bool]) -> list[ChdTrack]: + """Fill the derived layout fields the way the runtime lays a CHD out. + + ``v2`` says, per track, whether the entry came from CHT2 (only those can + carry a stored pregap). + """ + out: list[ChdTrack] = [] + disc_lba = 0 + chd_frame = 0 + for index, (t, is_v2) in enumerate(zip(tracks, v2)): + if t.number != index + 1: + raise ChdError(f"track metadata out of order: expected {index + 1}, got {t.number}") + if t.frames <= 0: + raise ChdError(f"track {t.number} has no frames") + pregap_stored = is_v2 and t.pregap > 0 and t.pgtype[:1].upper() == "V" + t.stored_pregap = min(t.pregap, t.frames) if pregap_stored else 0 + t.virtual_pregap = t.pregap if (not pregap_stored and t.pregap > 0) else 0 + t.disc_lba = disc_lba + t.chd_frame_start = chd_frame + disc_lba += t.virtual_pregap + t.frames + chd_frame += t.frames + chd_frame = (chd_frame + CD_TRACK_PADDING - 1) & ~(CD_TRACK_PADDING - 1) + out.append(t) + if not out: + raise ChdError("CHD carries no CD track metadata; not a CD image") + return out + + +# --------------------------------------------------------------------------- +# Disc access +# --------------------------------------------------------------------------- + +class ChdDisc: + """An open CD CHD. Use as a context manager.""" + + def __init__(self, path: Path, lib: LibChdr): + self.path = Path(path) + self.lib = lib + self._file = ctypes.c_void_p() + self.hunk_bytes = 0 + self.frames_per_hunk = 0 + self.total_hunks = 0 + self.tracks: list[ChdTrack] = [] + self._hunk_buf: Optional[ctypes.Array] = None + self._cached_hunk = -1 + self._open() + + # -- lifecycle ---------------------------------------------------------- + + def _open(self) -> None: + lib = self.lib.raw + last = CHDERR_NONE + for arg in _open_arg_candidates(self.path): + handle = ctypes.c_void_p() + err = lib.chd_open(arg, CHD_OPEN_READ, None, ctypes.byref(handle)) + if err == CHDERR_NONE and handle.value: + self._file = handle + break + last = err + else: + raise ChdError(f"cannot open {self.path.name}: {self.lib.error_string(last)}") + + hdr_p = lib.chd_get_header(self._file) + if not hdr_p: + self.close() + raise ChdError(f"{self.path.name}: libchdr returned no header") + hdr = hdr_p.contents + if hdr.hunkbytes == 0 or hdr.hunkbytes % CD_FRAME_SIZE != 0: + self.close() + raise ChdError( + f"{self.path.name} is not a CD image: hunk size {hdr.hunkbytes} " + f"is not a multiple of the {CD_FRAME_SIZE}-byte CD frame" + ) + self.hunk_bytes = int(hdr.hunkbytes) + self.frames_per_hunk = self.hunk_bytes // CD_FRAME_SIZE + self.total_hunks = int(hdr.totalhunks) + self.version = int(hdr.version) + self._hunk_buf = (ctypes.c_ubyte * self.hunk_bytes)() + try: + self.tracks = self._read_track_table() + except ChdError: + self.close() + raise + + def close(self) -> None: + if self._file and self._file.value: + self.lib.raw.chd_close(self._file) + self._file = ctypes.c_void_p() + + def __enter__(self) -> "ChdDisc": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + # -- metadata ----------------------------------------------------------- + + def _metadata(self, tag: int, index: int) -> Optional[str]: + lib = self.lib.raw + buf = ctypes.create_string_buffer(256) + result_len = ctypes.c_uint32(0) + err = lib.chd_get_metadata( + self._file, tag, index, buf, len(buf) - 1, + ctypes.byref(result_len), None, None, + ) + if err == CHDERR_METADATA_NOT_FOUND: + return None + if err != CHDERR_NONE: + raise ChdError(f"{self.path.name}: metadata read failed: {self.lib.error_string(err)}") + return buf.raw[: result_len.value].decode("ascii", "replace") + + def _read_track_table(self) -> list[ChdTrack]: + tracks: list[ChdTrack] = [] + v2_flags: list[bool] = [] + for index in range(CD_MAX_TRACKS): + text = self._metadata(CDROM_TRACK_METADATA2_TAG, index) + v2 = text is not None + if not v2: + text = self._metadata(CDROM_TRACK_METADATA_TAG, index) + if text is None: + break + tracks.append(parse_track_metadata(text, v2=v2)) + v2_flags.append(v2) + table = build_track_table(tracks, v2=v2_flags) + last = table[-1] + needed = last.chd_frame_start + last.frames + available = self.total_hunks * self.frames_per_hunk + if needed > available: + raise ChdError( + f"{self.path.name}: track table needs {needed} frames but the " + f"image holds {available}" + ) + return table + + @property + def data_frames(self) -> int: + return sum(t.frames for t in self.tracks) + + # -- reading ------------------------------------------------------------ + + def _hunk(self, index: int) -> bytes: + assert self._hunk_buf is not None + if index != self._cached_hunk: + err = self.lib.raw.chd_read(self._file, index, self._hunk_buf) + if err != CHDERR_NONE: + self._cached_hunk = -1 + raise ChdError( + f"{self.path.name}: hunk {index} failed to decode: " + f"{self.lib.error_string(err)}" + ) + self._cached_hunk = index + return bytes(self._hunk_buf) + + def iter_track_chunks(self, track: ChdTrack) -> Iterator[bytes]: + """Yield the track's 2352-byte sectors, hunk by hunk, as .bin bytes. + + Chunks are whole sectors but of varying length (a hunk's worth at + most); concatenated they are exactly the track's ``.bin``. + """ + if track.type not in _CUE_MODE: + track.cue_mode # raises with the reason + frame = track.chd_frame_start + end = frame + track.frames + swap = track.is_audio + while frame < end: + hunk_index = frame // self.frames_per_hunk + first = frame % self.frames_per_hunk + count = min(self.frames_per_hunk - first, end - frame) + hunk = self._hunk(hunk_index) + start = first * CD_FRAME_SIZE + parts = [ + hunk[off: off + RAW_SECTOR_SIZE] + for off in range(start, start + count * CD_FRAME_SIZE, CD_FRAME_SIZE) + ] + chunk = b"".join(parts) + if swap: + a = array("H") + a.frombytes(chunk) + a.byteswap() + chunk = a.tobytes() + yield chunk + frame += count + + def read_track(self, track: ChdTrack) -> bytes: + return b"".join(self.iter_track_chunks(track)) + + +# --------------------------------------------------------------------------- +# Digests and extraction +# --------------------------------------------------------------------------- + +@dataclass +class Digest: + size: int + md5: str + sha1: str + + +@dataclass +class ChdDigests: + """Digests of every layout a Redump dump of this disc could have.""" + + tracks: list[Digest] = field(default_factory=list) + disc: Digest = field(default_factory=lambda: Digest(0, "", "")) + + @property + def first_track(self) -> Digest: + return self.tracks[0] + + def layout_matching(self, sizes: Sequence[int], md5s: Sequence[str], sha1s: Sequence[str]) -> Optional[str]: + """``"multi"`` if the first track alone matches a known digest, + ``"single"`` if the whole disc does, else ``None``.""" + md5s = [m.lower() for m in md5s] + sha1s = [s.lower() for s in sha1s] + + def hit(d: Digest) -> bool: + if d.md5 in md5s or d.sha1 in sha1s: + return True + return bool(sizes) and not md5s and not sha1s and d.size in sizes + + if hit(self.first_track): + return "multi" + if len(self.tracks) > 1 and hit(self.disc): + return "single" + return None + + +ProgressFn = Callable[[int, int], None] + + +def digests(disc: ChdDisc, progress: Optional[ProgressFn] = None) -> ChdDigests: + """Hash every track and the whole-disc concatenation in one pass.""" + out = ChdDigests() + disc_md5, disc_sha1 = hashlib.md5(), hashlib.sha1() + total = disc.data_frames + done = 0 + for track in disc.tracks: + md5, sha1, size = hashlib.md5(), hashlib.sha1(), 0 + for chunk in disc.iter_track_chunks(track): + md5.update(chunk) + sha1.update(chunk) + disc_md5.update(chunk) + disc_sha1.update(chunk) + size += len(chunk) + done += len(chunk) // RAW_SECTOR_SIZE + if progress: + progress(done, total) + out.tracks.append(Digest(size, md5.hexdigest(), sha1.hexdigest())) + out.disc = Digest( + sum(d.size for d in out.tracks), disc_md5.hexdigest(), disc_sha1.hexdigest() + ) + return out + + +def msf(frames: int) -> str: + m, rem = divmod(frames, 75 * 60) + s, f = divmod(rem, 75) + return f"{m:02d}:{s:02d}:{f:02d}" + + +def track_bin_name(stem: str, track: ChdTrack, track_count: int) -> str: + return f"{stem}.bin" if track_count == 1 else f"{stem} (Track {track.number}).bin" + + +def render_cue(tracks: Sequence[ChdTrack], *, layout: str, stem: str, single_bin: Optional[str] = None) -> str: + """Cue text for the given layout, in Redump's shape.""" + lines: list[str] = [] + if layout == "single": + lines.append(f'FILE "{single_bin or stem + ".bin"}" BINARY') + offset = 0 + for i, t in enumerate(tracks): + lines.append(f" TRACK {t.number:02d} {t.cue_mode}") + if i and t.virtual_pregap: + lines.append(f" PREGAP {msf(t.virtual_pregap)}") + if t.stored_pregap: + lines.append(f" INDEX 00 {msf(offset)}") + lines.append(f" INDEX 01 {msf(offset + t.stored_pregap)}") + offset += t.frames + elif layout == "multi": + for i, t in enumerate(tracks): + name = single_bin if (single_bin and len(tracks) == 1) else track_bin_name(stem, t, len(tracks)) + lines.append(f'FILE "{name}" BINARY') + lines.append(f" TRACK {t.number:02d} {t.cue_mode}") + if i and t.virtual_pregap: + lines.append(f" PREGAP {msf(t.virtual_pregap)}") + if t.stored_pregap: + lines.append(" INDEX 00 00:00:00") + lines.append(f" INDEX 01 {msf(t.stored_pregap)}") + else: + raise ValueError(f"unknown layout {layout!r}") + return "\n".join(lines) + "\n" + + +def extract( + disc: ChdDisc, + out_dir: Path, + cue_name: str, + *, + layout: str = "multi", + bin_name: Optional[str] = None, + progress: Optional[ProgressFn] = None, +) -> Path: + """Write ``.bin`` file(s) and a cue into ``out_dir``; return the cue path. + + ``bin_name`` names the single-layout bin (default: cue stem + ``.bin``). + In the multi layout the bins follow Redump naming off the cue stem. + """ + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + stem = Path(cue_name).stem + total = disc.data_frames + done = 0 + + def pump(fh, track: ChdTrack) -> None: + nonlocal done + for chunk in disc.iter_track_chunks(track): + fh.write(chunk) + done += len(chunk) // RAW_SECTOR_SIZE + if progress: + progress(done, total) + + if layout == "single": + target = out_dir / (bin_name or f"{stem}.bin") + with open(target, "wb") as fh: + for t in disc.tracks: + pump(fh, t) + cue_text = render_cue(disc.tracks, layout="single", stem=stem, single_bin=target.name) + elif layout == "multi": + single_track = len(disc.tracks) == 1 and bin_name + for t in disc.tracks: + name = bin_name if single_track else track_bin_name(stem, t, len(disc.tracks)) + with open(out_dir / name, "wb") as fh: + pump(fh, t) + cue_text = render_cue( + disc.tracks, layout="multi", stem=stem, + single_bin=bin_name if single_track else None, + ) + else: + raise ValueError(f"unknown layout {layout!r}") + + cue_path = out_dir / cue_name + with open(cue_path, "w", encoding="ascii", newline="\n") as fh: + fh.write(cue_text) + return cue_path + + +def multi_layout_bins(tracks: Sequence[ChdTrack], cue_name: str) -> list[str]: + stem = Path(cue_name).stem + return [track_bin_name(stem, t, len(tracks)) for t in tracks] + + +def unsupported_message(path: Path) -> str: + """What to tell a player when no libchdr is available for this CHD.""" + return ( + f"{path.name} is a CHD and the CHD reader (libchdr) is not built for this " + f"kit, so the build tools cannot read it -- this does not mean your dump is " + f"bad. Rebuild the emitters, or extract it first, for example: chdman " + f'extractcd -i "{path.name}" -o "{path.stem}.cue" -ob "{path.stem}.bin", ' + f"then select the .cue." + ) + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + +def _main(argv: Optional[Sequence[str]] = None) -> int: + import argparse + + ap = argparse.ArgumentParser(description="Inspect, hash or extract a CD .chd through libchdr.") + ap.add_argument("chd", help="path to the .chd") + ap.add_argument("--lib", default="", help=f"libchdr shared library (default: ${LIB_ENV} or the recompiler build dir)") + ap.add_argument("--extract", default="", help="write .bin/.cue into this directory") + ap.add_argument("--layout", choices=("multi", "single"), default="multi") + ap.add_argument("--cue-name", default="", help="cue file name for --extract (default: <chd stem>.cue)") + ap.add_argument("--no-hash", action="store_true", help="skip the digest pass") + args = ap.parse_args(argv) + + here = Path(__file__).resolve().parent.parent + lib_path = Path(args.lib) if args.lib else find_libchdr(None, here) + if not lib_path: + print(f"libchdr not found; set {LIB_ENV} or build target chdr", file=sys.stderr) + return 2 + lib = LibChdr(lib_path) + chd_path = Path(args.chd).expanduser().resolve() + with ChdDisc(chd_path, lib) as disc: + print(f"{chd_path.name}: v{disc.version}, {disc.total_hunks} hunks of {disc.hunk_bytes}, {len(disc.tracks)} track(s)") + for t in disc.tracks: + extra = f" pregap={t.pregap}({'stored' if t.stored_pregap else 'virtual' if t.virtual_pregap else 'none'})" + print(f" track {t.number:02d} {t.type:<10} frames={t.frames}{extra} chd_frame={t.chd_frame_start}") + if not args.no_hash: + d = digests(disc) + for i, td in enumerate(d.tracks, 1): + print(f" track {i:02d}: size={td.size} md5={td.md5} sha1={td.sha1}") + if len(d.tracks) > 1: + print(f" disc : size={d.disc.size} md5={d.disc.md5} sha1={d.disc.sha1}") + if args.extract: + cue = extract(disc, Path(args.extract), args.cue_name or f"{chd_path.stem}.cue", layout=args.layout) + print(f"wrote {cue}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/tools/tests/test_psx_chd.py b/tools/tests/test_psx_chd.py new file mode 100644 index 00000000..a8fd729f --- /dev/null +++ b/tools/tests/test_psx_chd.py @@ -0,0 +1,306 @@ +"""Source-owned CHD fixtures: a synthetic uncompressed v5 CHD wrapped around a +cooked ISO 9660 image. No retail sectors, executables or audio. + +The pure-Python parts (metadata parsing, track layout, cue rendering, layout +selection) always run. Everything that decodes a CHD needs the shared libchdr +that ``recompiler/`` builds (CMake target ``chdr``); those tests skip with a +reason when ``PSXRECOMP_LIBCHDR`` is unset and no build directory has it. +""" +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(ROOT), str(ROOT / "tools"), str(ROOT / "tools" / "new_project_layout")] +import prepare_disc as prepare # noqa: E402 +import psx_chd # noqa: E402 +import psxrecomp_cli as cli # noqa: E402 +from sdk_progress import ProgressReporter # noqa: E402 + +BOOT = "SLES_000.00" +FRAME = psx_chd.CD_FRAME_SIZE +RAW = psx_chd.RAW_SECTOR_SIZE +HUNK_FRAMES = 8 + + +def cooked_disc() -> bytes: + """A 24-sector ISO 9660 image with SYSTEM.CNF and a PS-X EXE stub.""" + sectors = [bytearray(2048) for _ in range(24)] + sectors[16][1:6] = b"CD001" + struct.pack_into("<I", sectors[16], 158, 20) + struct.pack_into("<I", sectors[16], 166, 2048) + cursor = 0 + exe = bytearray(b"PS-X EXE" + bytes(2040)) + struct.pack_into("<I", exe, 0x10, 0x80010000) # PC0 + struct.pack_into("<I", exe, 0x18, 0x80010000) # load address + struct.pack_into("<I", exe, 0x1C, 0x800) # text size + for name, lba, data in [("SYSTEM.CNF", 21, b"BOOT = cdrom:\\" + BOOT.encode() + b";1"), + (BOOT, 22, bytes(exe))]: + encoded = (name + ";1").encode() + record = bytearray(33 + len(encoded) + (len(encoded) % 2 == 0)) + record[0] = len(record) + struct.pack_into("<I", record, 2, lba) + struct.pack_into("<I", record, 10, len(data)) + record[32] = len(encoded) + record[33:33 + len(encoded)] = encoded + sectors[20][cursor:cursor + len(record)] = record + cursor += len(record) + sectors[lba][:len(data)] = data + return b"".join(sectors) + + +def audio_track(frames: int, seed: int) -> bytes: + """Deterministic little-endian PCM, the byte order a Redump .bin holds.""" + out = bytearray() + for i in range(frames * RAW // 2): + v = (i * 7919 + seed) & 0xFFFF + out += struct.pack("<H", v) + return bytes(out) + + +def write_uncompressed_chd(path: Path, tracks) -> None: + """Write a v5 CHD with codec NONE. ``tracks`` is a list of + ``(type, data_bytes, pregap, pgtype)``; ``data_bytes`` is the track's + .bin content, including any stored pregap frames.""" + hunk_bytes = FRAME * HUNK_FRAMES + stream = bytearray() + meta_texts = [] + for number, (ttype, data, pregap, pgtype) in enumerate(tracks, 1): + assert len(data) % RAW == 0 + frames = len(data) // RAW + for f in range(frames): + sector = data[f * RAW:(f + 1) * RAW] + if ttype == "AUDIO": + sector = b"".join(sector[i + 1:i + 2] + sector[i:i + 1] for i in range(0, RAW, 2)) + stream += sector + bytes(psx_chd.SUBCODE_SIZE) + total = len(stream) // FRAME + padded = (total + psx_chd.CD_TRACK_PADDING - 1) & ~(psx_chd.CD_TRACK_PADDING - 1) + stream += bytes((padded - total) * FRAME) + meta_texts.append( + f"TRACK:{number} TYPE:{ttype} SUBTYPE:NONE FRAMES:{frames} PREGAP:{pregap} " + f"PGTYPE:{pgtype} PGSUB:NONE POSTGAP:0" + ) + logical = len(stream) + total_hunks = (logical + hunk_bytes - 1) // hunk_bytes + stream += bytes(total_hunks * hunk_bytes - logical) + + header_size = 124 + map_offset = header_size + map_size = total_hunks * 4 + meta_offset = map_offset + map_size + meta_blob = bytearray() + cursor = meta_offset + for i, text in enumerate(meta_texts): + payload = text.encode("ascii") + b"\0" + entry = 16 + len(payload) + nxt = cursor + entry if i + 1 < len(meta_texts) else 0 + meta_blob += struct.pack(">II Q", psx_chd.CDROM_TRACK_METADATA2_TAG, + (1 << 24) | len(payload), nxt) + payload + cursor += entry + data_offset = ((meta_offset + len(meta_blob) + hunk_bytes - 1) // hunk_bytes) * hunk_bytes + first_index = data_offset // hunk_bytes + raw_map = b"".join(struct.pack(">I", first_index + i) for i in range(total_hunks)) + header = (b"MComprHD" + struct.pack(">II", header_size, 5) + struct.pack(">IIII", 0, 0, 0, 0) + + struct.pack(">QQQ", logical, map_offset, meta_offset) + + struct.pack(">II", hunk_bytes, FRAME) + bytes(60)) + assert len(header) == header_size + blob = bytearray(header) + raw_map + meta_blob + blob += bytes(data_offset - len(blob)) + blob += stream + path.write_bytes(bytes(blob)) + + +def find_lib(): + return psx_chd.find_libchdr(None, ROOT) + + +class PureTests(unittest.TestCase): + def test_parse_v2_and_v1_metadata(self): + t = psx_chd.parse_track_metadata( + "TRACK:2 TYPE:AUDIO SUBTYPE:NONE FRAMES:6502 PREGAP:150 PGTYPE:VAUDIO PGSUB:NONE POSTGAP:0\0", + v2=True) + self.assertEqual((t.number, t.type, t.frames, t.pregap, t.pgtype), (2, "AUDIO", 6502, 150, "VAUDIO")) + t1 = psx_chd.parse_track_metadata("TRACK:1 TYPE:MODE2_RAW SUBTYPE:NONE FRAMES:100", v2=False) + self.assertEqual((t1.number, t1.frames, t1.pregap), (1, 100, 0)) + with self.assertRaises(psx_chd.ChdError): + psx_chd.parse_track_metadata("TRACK:1 TYPE:AUDIO", v2=True) + + def test_layout_mirrors_runtime(self): + tracks = [ + psx_chd.ChdTrack(1, "MODE2_RAW", "NONE", 10), + psx_chd.ChdTrack(2, "AUDIO", "NONE", 7, pregap=2, pgtype="VAUDIO"), + psx_chd.ChdTrack(3, "AUDIO", "NONE", 5, pregap=3, pgtype="AUDIO"), + ] + table = psx_chd.build_track_table(tracks, v2=[True, True, True]) + self.assertEqual([t.chd_frame_start for t in table], [0, 12, 20]) + self.assertEqual([t.stored_pregap for t in table], [0, 2, 0]) + self.assertEqual([t.virtual_pregap for t in table], [0, 0, 3]) + self.assertEqual([t.disc_lba for t in table], [0, 10, 17]) + with self.assertRaises(psx_chd.ChdError): + psx_chd.build_track_table([psx_chd.ChdTrack(2, "AUDIO", "NONE", 1)], v2=[True]) + + def test_render_cue_both_layouts(self): + tracks = psx_chd.build_track_table([ + psx_chd.ChdTrack(1, "MODE2_RAW", "NONE", 100), + psx_chd.ChdTrack(2, "AUDIO", "NONE", 300, pregap=150, pgtype="VAUDIO"), + ], v2=[True, True]) + multi = psx_chd.render_cue(tracks, layout="multi", stem="Game") + self.assertEqual(multi.splitlines(), [ + 'FILE "Game (Track 1).bin" BINARY', " TRACK 01 MODE2/2352", " INDEX 01 00:00:00", + 'FILE "Game (Track 2).bin" BINARY', " TRACK 02 AUDIO", " INDEX 00 00:00:00", " INDEX 01 00:02:00", + ]) + single = psx_chd.render_cue(tracks, layout="single", stem="Game") + self.assertEqual(single.splitlines(), [ + 'FILE "Game.bin" BINARY', " TRACK 01 MODE2/2352", " INDEX 01 00:00:00", + " TRACK 02 AUDIO", " INDEX 00 00:01:25", " INDEX 01 00:03:25", + ]) + with self.assertRaises(psx_chd.ChdError): + psx_chd.render_cue([psx_chd.ChdTrack(1, "MODE1", "NONE", 1)], layout="multi", stem="x") + + def test_layout_matching(self): + d = psx_chd.ChdDigests(tracks=[psx_chd.Digest(10, "a", "b"), psx_chd.Digest(5, "c", "d")], + disc=psx_chd.Digest(15, "e", "f")) + self.assertEqual(d.layout_matching([], ["A"], []), "multi") + self.assertEqual(d.layout_matching([], [], ["F"]), "single") + self.assertEqual(d.layout_matching([15], [], []), "single") + self.assertIsNone(d.layout_matching([], ["zz"], ["yy"])) + + +@unittest.skipUnless(find_lib(), f"libchdr not built; set {psx_chd.LIB_ENV} or build target chdr") +class DecodeTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.lib = psx_chd.LibChdr(find_lib()) + self.data = prepare.iso_to_bin(cooked_disc()) + self.audio = audio_track(150 + 33, seed=3) # 150 stored pregap frames + 33 audio frames + self.chd = self.root / "Fixture (Europe).chd" + write_uncompressed_chd(self.chd, [ + ("MODE2_RAW", self.data, 0, "MODE2_RAW"), + ("AUDIO", self.audio, 150, "VAUDIO"), + ]) + self.single = self.root / "Single (USA).chd" + write_uncompressed_chd(self.single, [("MODE2_RAW", self.data, 0, "MODE2_RAW")]) + + def sha1(self, b: bytes) -> str: + return hashlib.sha1(b).hexdigest() + + def test_tracks_read_back_as_redump_bytes(self): + with psx_chd.ChdDisc(self.chd, self.lib) as disc: + self.assertEqual([t.type for t in disc.tracks], ["MODE2_RAW", "AUDIO"]) + self.assertEqual(disc.tracks[1].stored_pregap, 150) + self.assertEqual(disc.read_track(disc.tracks[0]), self.data) + self.assertEqual(disc.read_track(disc.tracks[1]), self.audio) + d = psx_chd.digests(disc) + self.assertEqual(d.first_track.sha1, self.sha1(self.data)) + self.assertEqual(d.tracks[1].sha1, self.sha1(self.audio)) + self.assertEqual(d.disc.sha1, self.sha1(self.data + self.audio)) + self.assertEqual(d.disc.size, len(self.data) + len(self.audio)) + + def test_extract_multi_and_single(self): + with psx_chd.ChdDisc(self.chd, self.lib) as disc: + cue = psx_chd.extract(disc, self.root / "multi", "Fixture (Europe).cue", layout="multi") + single = psx_chd.extract(disc, self.root / "single", "Fixture (Europe).cue", layout="single") + self.assertEqual((self.root / "multi" / "Fixture (Europe) (Track 1).bin").read_bytes(), self.data) + self.assertEqual((self.root / "multi" / "Fixture (Europe) (Track 2).bin").read_bytes(), self.audio) + self.assertIn('FILE "Fixture (Europe) (Track 2).bin" BINARY', cue.read_text()) + self.assertEqual((self.root / "single" / "Fixture (Europe).bin").read_bytes(), self.data + self.audio) + self.assertIn("INDEX 00 00:00:24", single.read_text()) # 24 data sectors, then the stored pregap + + def test_verify_disc_path_hashes_tracks_not_the_container(self): + prep_multi = {"known_sha1": [self.sha1(self.data)]} + identity = cli.verify_disc_path(self.chd, prep_multi, skip_hash=False, progress=ProgressReporter()) + self.assertTrue(identity["verified"]) + self.assertEqual(identity["chd"], {"layout": "multi", "tracks": 2}) + self.assertEqual(identity["size"], len(self.data)) + + prep_single = {"known_md5": [hashlib.md5(self.data + self.audio).hexdigest()]} + identity = cli.verify_disc_path(self.chd, prep_single, skip_hash=False, progress=ProgressReporter()) + self.assertEqual(identity["chd"]["layout"], "single") + self.assertEqual(identity["size"], len(self.data) + len(self.audio)) + + with self.assertRaises(cli.DiscVerifyError) as ctx: + cli.verify_disc_path(self.chd, {"known_sha1": ["0" * 40]}, skip_hash=False, progress=ProgressReporter()) + self.assertIn("CHD track digests", str(ctx.exception)) + self.assertNotIn("wrong dump", str(ctx.exception)) + + def test_verify_without_reader_names_the_reader_not_the_dump(self): + with patch.dict("os.environ", {psx_chd.LIB_ENV: ""}), patch.object(psx_chd, "find_libchdr", return_value=None): + with self.assertRaises(cli.DiscVerifyError) as ctx: + cli.verify_disc_path(self.chd, {"known_sha1": ["x"]}, skip_hash=False, progress=ProgressReporter()) + self.assertIn("libchdr", str(ctx.exception)) + self.assertIn("does not mean your dump is bad", str(ctx.exception)) + + def run_prepare(self, source: Path, config_text: str, out: Path): + config = self.root / "game.toml" + config.write_text(config_text, encoding="utf-8") + output = io.StringIO() + args = ["prepare_disc.py", str(source), "--config", str(config), "--out-dir", str(out)] + with patch.object(sys, "argv", args), contextlib.redirect_stdout(output), \ + contextlib.redirect_stderr(output), patch.object(prepare, "_configure_stdio"): + code = prepare.main() + return code, output.getvalue() + + def test_prepare_disc_stages_a_multitrack_chd_once(self): + out = self.root / "disc" + code, log = self.run_prepare(self.chd, ( + f'[game]\nid = "SLES-00000"\n[prepare_disc]\nboot_exe = "{BOOT}"\n' + f'bin_name = "Fixture (Europe) (Track 1).bin"\ncue_name = "Fixture (Europe).cue"\n' + f'known_sha1 = ["{self.sha1(self.data)}"]\n' + ), out) + self.assertEqual(code, 0, log) + self.assertIn("layout: multi", log) + self.assertIn("preserving multi-track Redump set (2 files)", log) + self.assertEqual((out / "Fixture (Europe) (Track 1).bin").read_bytes(), self.data) + self.assertEqual((out / "Fixture (Europe) (Track 2).bin").read_bytes(), self.audio) + self.assertTrue((out / BOOT).is_file()) + receipt = json.loads((out / "Fixture (Europe).disc-receipt.json").read_text()) + self.assertEqual(receipt["source_image"], str(self.chd)) + self.assertIn('FILE "Fixture (Europe) (Track 2).bin" BINARY', (out / "Fixture (Europe).cue").read_text()) + + def test_prepare_disc_single_track_chd_lands_on_bin_name(self): + out = self.root / "disc" + code, log = self.run_prepare(self.single, ( + f'[game]\nid = "SLUS-00000"\n[prepare_disc]\nboot_exe = "{BOOT}"\n' + f'bin_name = "Single (USA).bin"\ncue_name = "Single (USA).cue"\n' + f'known_md5 = ["{hashlib.md5(self.data).hexdigest()}"]\n' + ), out) + self.assertEqual(code, 0, log) + self.assertIn("source already at", log) + self.assertEqual((out / "Single (USA).bin").read_bytes(), self.data) + self.assertEqual(sorted(p.name for p in out.glob("*.bin")), ["Single (USA).bin"]) + + def test_prepare_disc_refuses_unknown_digests_without_override(self): + code, log = self.run_prepare(self.single, ( + f'[game]\nid = "SLUS-00000"\n[prepare_disc]\nboot_exe = "{BOOT}"\n' + f'known_sha1 = ["{"0" * 40}"]\n' + ), self.root / "disc") + self.assertEqual(code, 1) + self.assertIn("CHD track digests are not in prepare_disc.known_*", log) + + def test_probe_disc_accepts_a_chd(self): + import probe_disc + output = io.StringIO() + args = ["probe_disc.py", str(self.chd)] + with patch.object(sys, "argv", args), contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()): + code = probe_disc.main() + self.assertEqual(code, 0) + payload = json.loads(output.getvalue()) + self.assertEqual(payload["boot_exe"], BOOT) + self.assertEqual(payload["track_count"], 2) + self.assertEqual(payload["data_track_size"], len(self.data)) + self.assertEqual(payload["bin_name"], "Fixture (Europe) (Track 1).bin") + + +if __name__ == "__main__": + unittest.main() From 8cb24ee19ad19cc1d12d7947d7b60411b81c2938 Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 12:00:02 +0300 Subject: [PATCH 39/56] tests: compare resolved paths in the CHD receipt check --- tools/tests/test_psx_chd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/tests/test_psx_chd.py b/tools/tests/test_psx_chd.py index a8fd729f..5dc8316c 100644 --- a/tools/tests/test_psx_chd.py +++ b/tools/tests/test_psx_chd.py @@ -265,7 +265,9 @@ class DecodeTests(unittest.TestCase): self.assertEqual((out / "Fixture (Europe) (Track 2).bin").read_bytes(), self.audio) self.assertTrue((out / BOOT).is_file()) receipt = json.loads((out / "Fixture (Europe).disc-receipt.json").read_text()) - self.assertEqual(receipt["source_image"], str(self.chd)) + # macOS reports the temp dir through /private/var while resolve() may + # not; compare resolved paths. + self.assertEqual(Path(receipt["source_image"]).resolve(), self.chd.resolve()) self.assertIn('FILE "Fixture (Europe) (Track 2).bin" BINARY', (out / "Fixture (Europe).cue").read_text()) def test_prepare_disc_single_track_chd_lands_on_bin_name(self): From 6acf1976d2effa3a6a71244ba3757fbb196e29fc Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 17:44:36 +0300 Subject: [PATCH 40/56] runtime: refuse an over-long overlay autocompile command instead of truncating it s_cmd was 4096 bytes; a 208-address --force-interior list produced a 6392-char command, snprintf cut it mid-argument, every compile failed the same way and autocompile latched degraded for the process lifetime. Buffer is now 8192 (cmd.exe's own ceiling), the cmd.exe wrapper is sized from it, and an over-long command is reported and disabled loudly. (cherry picked from commit b0fd04076ca4b0cc7f8d813bc6f750f94df30bf9) --- runtime/src/autocompile.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/runtime/src/autocompile.c b/runtime/src/autocompile.c index a9b2a79e..e84c51f6 100644 --- a/runtime/src/autocompile.c +++ b/runtime/src/autocompile.c @@ -19,7 +19,12 @@ #define PSX_OVERLAY_FLAVOR 0 #endif -static char s_cmd[4096]; /* large: the runtime-constructed bundled tcc cmd has +/* 8192, not 4096: a 208-address --force-interior list made the runtime-built command + * 6,392 chars, and snprintf's silent truncation cut it mid-argument, so every compile + * failed identically and autocompile latched degraded for the process lifetime + * (Parasite Eve, 2026-09-15). cmd.exe's own line limit is ~8,191 chars, so this is the + * useful ceiling; beyond it the invocation needs an @file, not a bigger buffer. */ +static char s_cmd[8192]; /* large: the runtime-constructed bundled tcc cmd has * many absolute paths (python+script+recompiler+tcc+...) */ static char s_cwd[512]; /* Canonical loader cache dir + captures file (see autocompile_set_cache_paths). @@ -754,6 +759,18 @@ static void autocompile_check_path_args(void) { #endif /* _WIN32 */ void autocompile_configure(const char *cmd, const char *cwd) { + if (cmd && strlen(cmd) >= sizeof(s_cmd)) { + /* Loud, and off: a truncated command line fails argparse on every retry + * with an error that never mentions the real cause. */ + fprintf(stderr, + "psxrecomp: overlay autocompile command is %zu chars, over the %zu-char " + "limit; autocompile DISABLED. Shorten the command (an @file for long " + "--force-interior lists) instead of relying on truncation.\n", + strlen(cmd), sizeof(s_cmd) - 1); + s_cmd[0] = '\0'; + snprintf(s_cwd, sizeof(s_cwd), "%s", cwd ? cwd : ""); + return; + } snprintf(s_cmd, sizeof(s_cmd), "%s", cmd ? cmd : ""); snprintf(s_cwd, sizeof(s_cwd), "%s", cwd ? cwd : ""); #ifdef _WIN32 @@ -879,7 +896,7 @@ int autocompile_request(void) { * is incorrect" — every autocompile run failed and the reshard silently * never happened). With the outer quotes cmd strips exactly those two and * executes the inner command verbatim. */ - char full[4200]; + char full[sizeof(s_cmd) + 32]; /* grows with s_cmd: the wrapper must never be the new truncation point */ snprintf(full, sizeof(full), "cmd.exe /C \"%s\"", s_cmd); STARTUPINFOA si; From 3b541c681f5d2cfb6c3ddf89b6684e5eb03a7ac4 Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 17:58:01 +0300 Subject: [PATCH 41/56] runtime: show launcher warnings through SDL on every platform launcher_warning/launcher_info only ever raised a dialog under _WIN32, so on macOS and Linux a first run with no BIOS printed to a terminal nobody was looking at and quit. SDL3 is already linked; SDL_ShowSimpleMessageBox covers all three platforms and keeps the headless guard. (cherry picked from commit 22f4322eb33d46946f9b953f943bedf070eb75a6) --- runtime/src/main.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index ca569218..43681365 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -2224,18 +2224,16 @@ static void write_cached_path(const char* argv0, const char* filename, static void launcher_warning(const char* title, const std::string& msg) { std::fprintf(stderr, "%s: %s\n", title, msg.c_str()); -#ifdef _WIN32 // Headless (--headless / PSX_HEADLESS): NEVER pop a blocking modal — it would // hang an unattended/CI/scripted run forever waiting for a click. - if (!g_headless) MessageBoxA(NULL, msg.c_str(), title, MB_OK | MB_ICONWARNING); -#endif + // SDL_ShowSimpleMessageBox is cross-platform; the Win32-only MessageBoxA left + // macOS and Linux users with a silent exit (v0.4.0 macOS report, 2026-09-15). + if (!g_headless) SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_WARNING, title, msg.c_str(), NULL); } static void launcher_info(const char* title, const std::string& msg) { std::fprintf(stderr, "%s: %s\n", title, msg.c_str()); -#ifdef _WIN32 - if (!g_headless) MessageBoxA(NULL, msg.c_str(), title, MB_OK | MB_ICONINFORMATION); -#endif + if (!g_headless) SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_INFORMATION, title, msg.c_str(), NULL); } /* Game display name for picker dialogs ("Tomba!"); set after the game From 09a59af86ce35eaf109b503ca8c58deab5c0bc48 Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Tue, 15 Sep 2026 19:57:46 +0300 Subject: [PATCH 42/56] runtime: report frame rate from every product, not just debug builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit s_frame_count is already maintained under PSX_NO_DEBUG_TOOLS, but everything that exposes it — the debug server, the freeze heartbeat, screenshots — is compiled out of production. A shipped binary could not report its own speed, so throughput could only be measured on the diagnostic product, which carries instrumentation of its own and is not what players run. That gap is why a 30% throughput regression reached a release candidate unmeasured, and why the first four attempts to attribute it were all made against the wrong binary. Adds an opt-in periodic line on stdout, outside every debug guard: PSX_FRAME_REPORT_MS=2000 psxrecomp: frames=1234 elapsed_ms=20000 fps=61.7 avg_fps=61.7 Unset, it is one branch on a cached int per vblank. Line-oriented on stdout so a harness can read it without a debug connection. (cherry picked from commit aac4dbf58331a413b4ac8be6890c7ff499c55ca0) (cherry picked from commit c1fbb313d15fe00256bc68a805ba0df255feb249) --- runtime/src/main.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index ca569218..4f425c31 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -3571,6 +3571,44 @@ static void runtime_perf_section_end(uint64_t start, uint64_t *total) { if (end >= start) *total += end - start; } +/* Frame-rate readout available in EVERY product, release included. + * + * The debug server and the freeze heartbeat are compiled out under + * PSX_NO_DEBUG_TOOLS, which left shipped binaries unable to report their own + * speed: a throughput regression could only be measured on the diagnostic + * build, which carries instrumentation of its own. s_frame_count is already + * maintained in production, so this only needs to expose it. + * + * Opt-in via PSX_FRAME_REPORT_MS (milliseconds between lines). When unset this + * is one branch on a cached int per vblank. */ +static void frame_report_tick(uint64_t frames) { + static int interval_ms = -1; + static uint64_t first_ticks = 0, last_ticks = 0, last_frames = 0; + if (interval_ms < 0) { + const char *e = std::getenv("PSX_FRAME_REPORT_MS"); + interval_ms = (e && e[0]) ? std::atoi(e) : 0; + if (interval_ms < 0) interval_ms = 0; + first_ticks = last_ticks = SDL_GetTicks(); + last_frames = frames; + if (interval_ms) + std::fprintf(stdout, "psxrecomp: frame report every %d ms\n", interval_ms); + } + if (!interval_ms) return; + const uint64_t now = SDL_GetTicks(); + if (now - last_ticks < (uint64_t)interval_ms) return; + const double win_s = (double)(now - last_ticks) / 1000.0; + const double all_s = (double)(now - first_ticks) / 1000.0; + std::fprintf(stdout, + "psxrecomp: frames=%llu elapsed_ms=%llu fps=%.1f avg_fps=%.1f\n", + (unsigned long long)frames, + (unsigned long long)(now - first_ticks), + win_s > 0.0 ? (double)(frames - last_frames) / win_s : 0.0, + all_s > 0.0 ? (double)frames / all_s : 0.0); + std::fflush(stdout); + last_ticks = now; + last_frames = frames; +} + static void runtime_perf_diag_tick() { static bool have_last = false; static RuntimePerfSnapshot last; @@ -6496,6 +6534,11 @@ static NetplayVblankEpilogue sdl_vblank_present_body(void) { int override = -1; #endif + { + /* Outside every debug guard on purpose: production must be measurable. */ + extern uint64_t s_frame_count; + frame_report_tick(s_frame_count); + } runtime_perf_frame_begin(); RuntimePerfFrameScope runtime_perf_frame_scope; runtime_perf_diag_tick(); From 6451508633dd3568f46d8cb6acbc77239768079f Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 18:33:19 +0300 Subject: [PATCH 43/56] runtime: spawn overlay autocompile on POSIX hosts autocompile_request() returned 0 on every non-Windows host, so a Linux or macOS player never had a shard compiled at runtime: captures were written, the packager staged a working toolchain, the boot log said "overlay autocompile enabled", and every overlay ran interpreted for the whole session (Parasite Eve on Rocky 9: 1.35e9 interpreted instructions by frame 13k with /usr/bin/gcc beside the exe). Mirror the Windows spawner with fork/exec: /bin/sh -c runs the configured command in its own process group at nice 10, stdout+stderr flow through one pipe into the same tail ring (streaming PSX_SHARD_RESULT parse), a pthread watcher drains the pipe and reaps the child, and the emulation thread's poll turns DONE into the one idempotent cache rescan. No PSX_SHARD_PUBLISHED handoff off Windows: the rescan is the fallback the Windows path already takes when a marker is cut off. Shutdown SIGTERMs the group and joins; PR_SET_PDEATHSIG on Linux plus the dying pipe cover a runtime that never reaches shutdown. The PSX_OVERLAY_* pins are set once at configure time (glibc setenv is not safe against a concurrent getenv). Configure names the interpreter it will run and marks the run degraded when the path is not executable. Link Threads unconditionally on Unix. The child-line buffer, shard_result_values, shard_result_line_locked and publish_parse_locked move out of the Windows block; child_line_locked is per-platform. Both Windows unit tests still pass. Verified on Rocky 9 (PE diagnostic product): 0 -> 68 shards in 150 s, interpreted instructions by frame 14.5k 1.35e9 -> 0.39e9, curve flat from frame ~4k, counters visible over the debug server, quit and SIGKILL mid-compile leave no orphans or zombies. (cherry picked from commit 854540c9c96e86b0cc1eea0dd99ef285169784d9) --- runtime/runtime.cmake | 9 +- runtime/src/autocompile.c | 366 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 353 insertions(+), 22 deletions(-) diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 12c3bc18..109d5c66 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -2061,11 +2061,10 @@ function(psxrecomp_add_runtime_target target) target_link_libraries(${target} PRIVATE "${OPENGL_opengl_LIBRARY}") endif() endif() - # Async lobby connect (psx_lobby_client.c) uses pthread on Unix. - if(PSXRECOMP_HAS_LOBBY_CLIENT) - find_package(Threads REQUIRED) - target_link_libraries(${target} PRIVATE Threads::Threads) - endif() + # The overlay autocompile watcher (autocompile.c), the debug server and + # the lobby client all use pthread on Unix. + find_package(Threads REQUIRED) + target_link_libraries(${target} PRIVATE Threads::Threads) endif() # ---- Vulkan backend (gpu_vk_renderer.c) -------------------------------- diff --git a/runtime/src/autocompile.c b/runtime/src/autocompile.c index e84c51f6..ca31a29d 100644 --- a/runtime/src/autocompile.c +++ b/runtime/src/autocompile.c @@ -1,6 +1,8 @@ -/* autocompile.c — see autocompile.h. Windows-first (the project's dev - * platform); on other hosts the spawn is a graceful no-op and the manual - * compile_overlays.py flow still works. */ +/* autocompile.c — see autocompile.h. Windows spawns through CreateProcess + * with a kill-on-close job and an incremental PSX_SHARD_PUBLISHED handoff; + * Linux/macOS spawn through fork/exec (/bin/sh -c) in their own process group + * and pick every shard up with the batch-end cache rescan. The manual + * compile_overlays.py flow still works everywhere. */ #include "autocompile.h" #include "overlay_loader.h" @@ -13,6 +15,17 @@ #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # include <windows.h> +#else +# include <errno.h> +# include <fcntl.h> +# include <pthread.h> +# include <signal.h> +# include <sys/wait.h> +# include <time.h> +# include <unistd.h> +# ifdef __linux__ +# include <sys/prctl.h> +# endif #endif #ifndef PSX_OVERLAY_FLAVOR @@ -44,10 +57,16 @@ static void ac_state_store(int value) { InterlockedExchange(&s_state, (LONG)value); } #else +/* Written by the watcher thread (DONE) and the emulation thread (RUNNING/IDLE), + * read by both: same seq-cst discipline as the Interlocked pair above. */ static int s_state = AC_IDLE; static int s_exit_code = -1; -static int ac_state_load(void) { return s_state; } -static void ac_state_store(int value) { s_state = value; } +static int ac_state_load(void) { + return __atomic_load_n(&s_state, __ATOMIC_SEQ_CST); +} +static void ac_state_store(int value) { + __atomic_store_n(&s_state, value, __ATOMIC_SEQ_CST); +} #endif static uint32_t s_runs = 0; static uint32_t s_fails = 0; @@ -65,7 +84,9 @@ static uint64_t autocompile_now_ms(void) { #ifdef _WIN32 return (uint64_t)GetTickCount64(); #else - return 0; + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0; + return (uint64_t)ts.tv_sec * 1000ull + (uint64_t)ts.tv_nsec / 1000000ull; #endif } @@ -82,11 +103,6 @@ static uint64_t autocompile_now_ms(void) { static int s_reported_broken = 0; /* Defined below, once the child-output tail ring it reads is declared. */ static void autocompile_report_broken_once(void); -#ifndef _WIN32 -/* The compile spawner and its output ring are Windows-only in this file, so - * there is no child output to quote off-Windows and nothing to report. */ -static void autocompile_report_broken_once(void) { } -#endif static void autocompile_note_failure(void) { s_fails++; @@ -190,9 +206,12 @@ static unsigned s_publish_prepare_count; static unsigned s_publish_prepare_fail; static unsigned s_publish_prepare_retry; static unsigned s_publish_prepare_giveup; +#endif /* _WIN32 */ +/* One child stdout line being assembled by the watcher (both platforms). */ static char s_child_line[1024]; static int s_child_line_len = 0; static int s_child_line_overflow = 0; +#ifdef _WIN32 /* See the forward declaration above for why this is loud. */ static void autocompile_report_broken_once(void) { @@ -231,6 +250,8 @@ static void autocompile_report_broken_once(void) { fflush(stdout); } +#endif /* _WIN32 */ + /* Parse result markers while stdout is streaming, not from s_out after exit. * The configured child command may chain post-processing after * compile_overlays.py (coverage_vault.py is the production example). That @@ -278,6 +299,12 @@ static int shard_result_line_locked(const char *line) { return 1; } +/* Per-platform: Windows also queues PSX_SHARD_PUBLISHED paths for the + * incremental publisher; POSIX only records the result counts and lets the + * batch-end rescan load what the run wrote. Both run under the output lock. */ +static void child_line_locked(void); + +#ifdef _WIN32 static void child_line_locked(void) { if (shard_result_line_locked(s_child_line)) return; static const char marker[] = "PSX_SHARD_PUBLISHED "; @@ -299,6 +326,7 @@ static void child_line_locked(void) { s_publish_raw_tail = item; WakeConditionVariable(&s_publish_cv); } +#endif /* _WIN32 */ static void publish_parse_locked(const char *buf, int n) { for (int i = 0; i < n; i++) { @@ -320,6 +348,7 @@ static void publish_parse_locked(const char *buf, int n) { } } +#ifdef _WIN32 /* The pipe reader only queues paths. A separate worker first-maps them, so a * slow antivirus/disk/loader event cannot stop stdout draining and deadlock the * compiler. There is exactly one mapped handoff at a time. The worker also @@ -608,6 +637,174 @@ static DWORD WINAPI watch_thread(LPVOID arg) { } #endif /* _WIN32 */ +#ifndef _WIN32 +/* ---- POSIX spawner (Linux / macOS) ------------------------------------ + * + * Mirrors the Windows path above with fork/exec: `/bin/sh -c <cmd>` runs the + * configured command in its own process group, stdout+stderr flow through one + * pipe into the same tail ring, a watcher thread drains the pipe and reaps + * the child, and the emulation thread's poll turns DONE into the one + * idempotent cache rescan that makes every shard the run wrote loadable. + * There is no PSX_SHARD_PUBLISHED handoff off Windows; the rescan is the + * same fallback the Windows path takes when a marker line is cut off. + * + * Until 2026-09-16 autocompile_request() was `return 0` on every non-Windows + * host. Captures were written, the packager staged a working toolchain, the + * boot log said "overlay autocompile enabled", and not one shard was ever + * compiled: a Linux or macOS player ran every overlay interpreted for the + * whole session (Parasite Eve on Rocky 9: 1.35e9 interpreted instructions by + * frame 13k with /usr/bin/gcc beside the exe). */ +static pthread_mutex_t s_out_lock = PTHREAD_MUTEX_INITIALIZER; +static int s_out_lock_init = 0; /* autocompile_configure() ran */ +static pid_t s_child = 0; /* live child; guarded by s_out_lock */ +static pthread_t s_watch_thread; +static int s_watch_thread_live = 0; /* emulation thread only */ + +typedef struct { int read_fd; pid_t pid; } PosixWatchCtx; + +static void autocompile_report_broken_once(void) { + if (s_reported_broken || s_consecutive_fails < AC_LOUD_AFTER_FAILS) return; + s_reported_broken = 1; + autocompile_set_degraded( + "overlay autocompile failed %u consecutive runs (last exit %d); " + "nothing is being compiled to native code, so overlay execution stays " + "in the interpreter. Check [runtime] overlay_autocompile_cmd in " + "game.toml: every path in it must resolve from the process working " + "directory.", + s_consecutive_fails, (int)s_exit_code); + char tail[AC_OUT_CAP]; + int n = 0; + if (s_out_lock_init) { + pthread_mutex_lock(&s_out_lock); + n = s_out_len < (int)sizeof tail - 1 ? s_out_len : (int)sizeof tail - 1; + if (n > 0) memcpy(tail, s_out + (s_out_len - n), (size_t)n); + pthread_mutex_unlock(&s_out_lock); + } + tail[n > 0 ? n : 0] = '\0'; + fprintf(stdout, + "psxrecomp: WARNING: overlay autocompile has failed %u consecutive " + "runs (last exit %d).\n" + " Nothing is being compiled to native code, so overlay execution " + "stays in the interpreter and\n" + " frame times will be far worse than this build is capable of.\n" + " Check [runtime] overlay_autocompile_cmd in game.toml: every path in " + "it must resolve from the\n" + " process working directory, and the recompiler and " + "tools/compile_overlays.py it names must exist.\n" + " Last compiler output:\n%s%s", + s_consecutive_fails, (int)s_exit_code, + n > 0 ? tail : " (no output captured)\n", + (n > 0 && tail[n - 1] != '\n') ? "\n" : ""); + fflush(stdout); +} + +static void child_line_locked(void) { + (void)shard_result_line_locked(s_child_line); +} + +static void out_append(const char *buf, int n) { + pthread_mutex_lock(&s_out_lock); + publish_parse_locked(buf, n); + if (n >= AC_OUT_CAP) { + memcpy(s_out, buf + (n - AC_OUT_CAP), AC_OUT_CAP); + s_out_len = AC_OUT_CAP; + } else { + if (s_out_len + n > AC_OUT_CAP) { + int keep = AC_OUT_CAP - n; + memmove(s_out, s_out + (s_out_len - keep), keep); + s_out_len = keep; + } + memcpy(s_out + s_out_len, buf, n); + s_out_len += n; + } + pthread_mutex_unlock(&s_out_lock); +} + +static void *posix_watch_thread(void *arg) { + PosixWatchCtx *ctx = (PosixWatchCtx *)arg; + char buf[1024]; + for (;;) { + ssize_t got = read(ctx->read_fd, buf, sizeof buf); + if (got > 0) { out_append(buf, (int)got); continue; } + if (got < 0 && errno == EINTR) continue; + break; /* EOF: every holder of the write end (sh, python, gcc) is gone */ + } + close(ctx->read_fd); + int status = 0; + pid_t r; + do { r = waitpid(ctx->pid, &status, 0); } while (r < 0 && errno == EINTR); + int code = -1; + if (r == ctx->pid) { + if (WIFEXITED(status)) code = WEXITSTATUS(status); + else if (WIFSIGNALED(status)) code = 128 + WTERMSIG(status); + } + pthread_mutex_lock(&s_out_lock); + /* A marker/result line cut off by child termination is not trustworthy. + * Completion will force the idempotent directory rescan instead. */ + if (s_child_line_len || s_child_line_overflow) { + s_publish_parse_fail_run++; + s_child_line_len = 0; + s_child_line_overflow = 0; + } + if (s_child == ctx->pid) s_child = 0; + pthread_mutex_unlock(&s_out_lock); + free(ctx); + __atomic_store_n(&s_exit_code, code, __ATOMIC_SEQ_CST); + ac_state_store(AC_DONE); + return NULL; +} + +/* Name the interpreter the spawn will run, once, at configure time: the + * command's first token, resolved the way `sh -c` resolves it. A path that + * does not exist or is not executable means every run fails identically, so + * say so before the first compile instead of after three failed spawns. A + * bare name (`python3`) is left to PATH. */ +static void autocompile_report_interpreter(void) { + char tok[1024]; + size_t n = 0; + const char *p = s_cmd; + if (*p == '"' || *p == '\'') { + const char q = *p++; + while (*p && *p != q && n + 1 < sizeof(tok)) tok[n++] = *p++; + } else { + while (*p && *p != ' ' && *p != '\t' && n + 1 < sizeof(tok)) tok[n++] = *p++; + } + tok[n] = '\0'; + if (!tok[0] || !strchr(tok, '/')) return; + if (access(tok, X_OK) == 0) { + fprintf(stdout, "psxrecomp: overlay autocompile interpreter: %s\n", tok); + return; + } + autocompile_set_degraded( + "overlay autocompile interpreter \"%s\" is missing or not executable; " + "every compile run will fail and overlay execution will stay in the " + "interpreter.", tok); + fprintf(stdout, + "psxrecomp: overlay autocompile interpreter '%s' is missing or not " + "executable — every compile run will fail.\n", tok); + fflush(stdout); +} + +/* Kill the whole compile tree (sh -> python -> gcc share the child's process + * group), then reap it through the watcher. SIGTERM first so a compiler can + * finish its current write; SIGKILL if the tree ignores it. */ +static void posix_kill_child_tree(void) { + pid_t child; + pthread_mutex_lock(&s_out_lock); + child = s_child; + pthread_mutex_unlock(&s_out_lock); + if (child <= 0) return; + if (kill(-child, SIGTERM) != 0) (void)kill(child, SIGTERM); + for (int i = 0; i < 30 && ac_state_load() == AC_RUNNING; i++) { + struct timespec ts = { 0, 100000000L }; /* 100 ms */ + nanosleep(&ts, NULL); + } + if (ac_state_load() == AC_RUNNING) { + if (kill(-child, SIGKILL) != 0) (void)kill(child, SIGKILL); + } +} +#endif /* !_WIN32 */ + #ifdef _WIN32 /* Name the interpreter the spawn will actually run, once, at configure time. * The command line's first token resolves through PATH exactly as cmd.exe /C @@ -783,6 +980,23 @@ void autocompile_configure(const char *cmd, const char *cwd) { autocompile_report_interpreter(); autocompile_check_path_args(); } +#else + s_out_lock_init = 1; + /* Pin WRITE cache + READ captures + flavor to the loader's canonical + * values in the environment every child inherits: same contract as the + * SetEnvironmentVariableA block in the Windows request path (see it for + * why each matters). Done once here, on the emulation thread before any + * worker exists, because glibc's setenv is not safe against a concurrent + * getenv on another thread. autocompile_set_cache_paths() runs first. */ + if (s_cache_dir[0]) setenv("PSX_OVERLAY_CACHE_DIR", s_cache_dir, 1); + if (s_captures[0]) setenv("PSX_OVERLAY_CAPTURES", s_captures, 1); + setenv("PSX_OVERLAY_LIVE_AUTOCOMPILE", "1", 1); + { + char flavor_buf[16]; + snprintf(flavor_buf, sizeof flavor_buf, "%d", (int)PSX_OVERLAY_FLAVOR); + setenv("PSX_OVERLAY_FLAVOR", flavor_buf, 1); + } + if (s_cmd[0]) autocompile_report_interpreter(); #endif } @@ -1002,7 +1216,97 @@ int autocompile_request(void) { } return 1; #else - return 0; /* non-Windows hosts: manual compile flow only */ + pthread_mutex_lock(&s_out_lock); + s_publish_drops_run = 0; + s_publish_load_fail_run = s_publish_parse_fail_run = 0; + s_child_line_len = 0; + s_child_line_overflow = 0; + s_out_len = 0; + s_shard_ok = s_shard_fail = s_shard_skipped = 0; + s_shard_result_seen = 0; + pthread_mutex_unlock(&s_out_lock); + __atomic_store_n(&s_exit_code, -1, __ATOMIC_SEQ_CST); + + int fds[2]; + if (pipe(fds) != 0) { + autocompile_note_failure(); + return 0; + } + /* Close-on-exec on both ends: the child's dup2 onto 1/2 makes fresh + * descriptors without the flag, and nothing else may inherit the pipe. */ + (void)fcntl(fds[0], F_SETFD, FD_CLOEXEC); + (void)fcntl(fds[1], F_SETFD, FD_CLOEXEC); + + /* The child inherits the PSX_OVERLAY_* pins set once in configure(). */ + pid_t pid = fork(); + if (pid < 0) { + close(fds[0]); + close(fds[1]); + autocompile_note_failure(); + return 0; + } + if (pid == 0) { + /* Child: async-signal-safe calls only until exec. Own process group so + * shutdown can kill sh -> python -> gcc together; nice(10) keeps the + * compile below the emulation thread (BELOW_NORMAL_PRIORITY_CLASS). */ + (void)setpgid(0, 0); + (void)nice(10); +#ifdef __linux__ + /* Crash-orphan protection for the direct child, the POSIX stand-in + * for the Windows kill-on-close job: if the runtime dies without + * reaching autocompile_shutdown, the kernel signals sh. A grandchild + * (python, gcc) outliving sh dies on its next write to the pipe + * whose read end went away with the runtime (SIGPIPE). */ + (void)prctl(PR_SET_PDEATHSIG, SIGTERM); + if (getppid() == 1) _exit(125); /* parent already gone before prctl */ +#endif + if (s_cwd[0] && chdir(s_cwd) != 0) _exit(126); + int devnull = open("/dev/null", O_RDONLY); + if (devnull >= 0) { + (void)dup2(devnull, 0); + if (devnull > 2) close(devnull); + } + (void)dup2(fds[1], 1); + (void)dup2(fds[1], 2); + execl("/bin/sh", "sh", "-c", s_cmd, (char *)NULL); + _exit(127); + } + (void)setpgid(pid, pid); /* both sides set it: no window for the kill */ + close(fds[1]); + + PosixWatchCtx *ctx = (PosixWatchCtx *)calloc(1, sizeof(*ctx)); + if (!ctx) { + /* The child is already running. Leaving it detached would permit the + * state to remain IDLE and a second writer to enter the same cache. */ + (void)kill(-pid, SIGKILL); + (void)waitpid(pid, NULL, 0); + close(fds[0]); + autocompile_note_failure(); + return 0; + } + ctx->read_fd = fds[0]; + ctx->pid = pid; + pthread_mutex_lock(&s_out_lock); + s_child = pid; + pthread_mutex_unlock(&s_out_lock); + ac_state_store(AC_RUNNING); + s_runs++; + if (pthread_create(&s_watch_thread, NULL, posix_watch_thread, ctx) != 0) { + /* Without the watcher nobody drains stdout, observes completion, or + * reaps the child. Cancel the just-created child fail-closed. */ + (void)kill(-pid, SIGKILL); + (void)waitpid(pid, NULL, 0); + close(fds[0]); + free(ctx); + pthread_mutex_lock(&s_out_lock); + s_child = 0; + pthread_mutex_unlock(&s_out_lock); + ac_state_store(AC_IDLE); + autocompile_note_failure(); + return 0; + } + s_watch_thread_live = 1; + return 1; #endif } @@ -1010,14 +1314,20 @@ int autocompile_request(void) { * skipped=K" line and record the counts. Returns 1 if a result line was found. * Called from autocompile_poll_main (emu thread) after the child exits. */ static int parse_shard_result(void) { -#ifdef _WIN32 char buf[AC_OUT_CAP + 1]; int n = 0; if (s_out_lock_init) { +#ifdef _WIN32 EnterCriticalSection(&s_out_lock); n = s_out_len; memcpy(buf, s_out, (size_t)n); LeaveCriticalSection(&s_out_lock); +#else + pthread_mutex_lock(&s_out_lock); + n = s_out_len; + memcpy(buf, s_out, (size_t)n); + pthread_mutex_unlock(&s_out_lock); +#endif } buf[n] = '\0'; /* Find the LAST occurrence (a run may print more than once over its life). */ @@ -1042,9 +1352,6 @@ static int parse_shard_result(void) { s_shard_fail = failed; s_shard_skipped = skipped; return 1; -#else - return 0; -#endif } void autocompile_poll_main(void) { @@ -1066,9 +1373,18 @@ void autocompile_poll_main(void) { s_rescans++; } if (ac_state_load() == AC_RUNNING) return; +#else + if (!s_out_lock_init) return; #endif if (ac_state_load() != AC_DONE) return; -#ifdef _WIN32 +#ifndef _WIN32 + /* DONE is published by the watcher just before it returns. Join it before + * exposing IDLE so a new run cannot race its exit-code/ring writes. */ + if (s_watch_thread_live) { + pthread_join(s_watch_thread, NULL); + s_watch_thread_live = 0; + } +#else if (publish_pending()) return; /* DONE is published immediately before the preparer returns. Join both * per-run workers before exposing IDLE, so a new run cannot inherit a live @@ -1154,6 +1470,14 @@ void autocompile_shutdown(void) { * preparing or committing: the discard cannot be refused. */ (void)publish_discard_all(); ac_state_store(AC_IDLE); +#else + if (!s_out_lock_init) return; /* never configured — nothing to stop */ + posix_kill_child_tree(); + if (s_watch_thread_live) { + pthread_join(s_watch_thread, NULL); + s_watch_thread_live = 0; + } + ac_state_store(AC_IDLE); #endif } @@ -1206,6 +1530,14 @@ int autocompile_status_json(char *out, int cap) { publish_prepare_last_us = s_publish_prepare_last_us; LeaveCriticalSection(&s_out_lock); } +#else + if (s_out_lock_init) { + pthread_mutex_lock(&s_out_lock); + int take = s_out_len < 900 ? s_out_len : 900; /* newest tail */ + tn = json_escape_into(tail, sizeof(tail), + s_out + (s_out_len - take), take); + pthread_mutex_unlock(&s_out_lock); + } #endif (void)tn; uint64_t retry_ms = 0; From c10bad2ebbdc669b8e315cf41b38dbc5bb56fcad Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 17:44:36 +0300 Subject: [PATCH 44/56] cli: stage the overlay toolchain beside every built product No setup package ever staged one, so overlay_cache = true alone left every streamed overlay on the interpreter. rebuild now stages the pinned Python, tcc, this pin's recompiler, headers and the build's codegen hash beside the normal and diagnostic products, and writes compiler.txt naming the compiler the wizard already has; the runtime treats that as a gcc-tier compiler, so a player with nothing on PATH gets -O2 shards. Verified on the NCII kit: gcc tier reported and four shards built with no compiler on PATH. (cherry picked from commit 03865e10f6ee198d58c47c74526df5db46bb35a9) --- psxrecomp_cli.py | 53 ++++++++++++++++++++++++++++++++++++++++++ runtime/src/main.cpp | 23 +++++++++++++++--- tools/release_stage.py | 14 ++++++++--- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/psxrecomp_cli.py b/psxrecomp_cli.py index cc593f83..7f3d269b 100755 --- a/psxrecomp_cli.py +++ b/psxrecomp_cli.py @@ -1778,6 +1778,8 @@ def cmd_rebuild(args: argparse.Namespace, progress: ProgressReporter) -> int: if exe is None: progress.error(exe_err, code=EXIT_ERROR) return EXIT_ERROR + progress.phase("overlays", pct=0.93, message="Staging overlay toolchain beside the product...") + stage_overlay_toolchain_for_product(project_root, exe.parent, progress) prune_raw = (getattr(args, "prune_after", None) or "").strip() if prune_raw: @@ -1790,6 +1792,57 @@ def cmd_rebuild(args: argparse.Namespace, progress: ProgressReporter) -> int: return EXIT_OK +def stage_overlay_toolchain_for_product(project_root: Path, exe_dir: Path, progress) -> Optional[Path]: + """Stage overlay_toolchain/ beside a built product so the runtime's autocompile gate is + true for every player build (wave-5 F1), and name the compiler the wizard already has so + shards get an optimising compiler instead of tcc (F2). Best effort: a failure leaves the + product playable with overlays interpreted, and says so.""" + try: + fw = framework_root(project_root) + tools_dir = fw / "tools" + inc_dir = fw / "runtime" / "include" + game_emitter = find_psxrecomp_game(project_root) + if str(tools_dir) not in sys.path: + sys.path.insert(0, str(tools_dir)) + import release_stage # noqa: E402 + + dl_cache = project_root / ".cache" / "overlay-toolchain" + # Runtime DLLs only from the emitter's own directory, never from PATH: Git's + # mingw64 on PATH carries the MSVCRT runtime, which is the 0xC0000139 trap. + mingw_bin = game_emitter.parent if (game_emitter.parent / "libgcc_s_seh-1.dll").is_file() else None + tk = Path(release_stage.stage_toolchain( + str(exe_dir), str(game_emitter.parent), str(tools_dir), str(inc_dir), str(dl_cache), + mingw_bin=str(mingw_bin) if mingw_bin else None, log=progress.log)) + # The stale-recompiler check compares the emitter's codegen hash with the runtime's tag. + for cand in (exe_dir / "psxrecomp_codegen_include" / "overlay_codegen_hash.h", + inc_dir / "overlay_codegen_hash.h"): + if cand.is_file(): + shutil.copy2(cand, tk / "include" / "overlay_codegen_hash.h") + break + compiler = None + if sys.platform == "win32": + bin_dir = resolve_toolchain_bin(project_root) + if bin_dir and (bin_dir / "clang.exe").is_file(): + compiler = bin_dir / "clang.exe" + else: + for name in ("gcc", "cc", "clang"): + w = shutil.which(name) + if w: + compiler = Path(w) + break + if compiler: + (tk / "compiler.txt").write_text(str(compiler) + "\n", encoding="utf-8") + progress.log(f"overlay toolchain staged at {tk}; shard compiler: {compiler}") + else: + progress.log(f"overlay toolchain staged at {tk}; no optimising compiler found (tcc tier)") + return tk + except Exception as exc: # noqa: BLE001 + progress.log(f"WARNING: overlay toolchain staging failed; overlays will run interpreted: {exc}") + return None + + + + def cmd_pgo_train(args: argparse.Namespace, progress: ProgressReporter) -> int: args.force_pgo = True args.no_pgo = False diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index ca569218..b819287c 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -13530,6 +13530,20 @@ int main(int argc, char** argv) { const char *tk_tcc = "tcc"; #endif const bool tk_present = std::filesystem::exists(tk_py); + /* Wave-5 F2: the CLI names the compiler it already has (the wizard's pack on + * Windows, the native cc on POSIX) in overlay_toolchain/compiler.txt, so a player + * with nothing on PATH still gets optimised shards instead of tcc's. */ + std::string tk_compiler; + { + std::ifstream cf(tk_dir / "compiler.txt"); + std::string line; + if (cf.is_open() && std::getline(cf, line)) { + while (!line.empty() && (line.back() == '\r' || line.back() == '\n' || line.back() == ' ')) + line.pop_back(); + std::error_code cec; + if (!line.empty() && std::filesystem::exists(line, cec)) tk_compiler = line; + } + } auto build_toolchain_cmd = [&](const char *compiler) { auto cmd_quote = [](const std::string& s) { return std::string("\"") + s + "\""; @@ -13546,12 +13560,14 @@ int main(int argc, char** argv) { " --out-dir " + cmd_quote((tk_xd / "cache").string()) + (g_psx_cps_mode ? " --cps" : "") + " --compiler " + compiler; + if (std::string(compiler) == "gcc" && !tk_compiler.empty()) + c += " --gcc " + cmd_quote(tk_compiler); if (std::string(compiler) == "tcc") c += " --tcc " + cmd_quote((tk_dir / "tcc" / tk_tcc).string()); return c; }; int gcc_avail = (deferred_has_overlay_ac || tk_present) - && autocompile_toolchain_available(); + && (autocompile_toolchain_available() || !tk_compiler.empty()); OverlayBackend eff = overlay_backend_resolve(cfg_backend, gcc_avail); std::string built_tcc_cmd; std::string built_gcc_cmd; @@ -13579,8 +13595,9 @@ int main(int argc, char** argv) { built_gcc_cmd = build_toolchain_cmd("gcc"); ac_cmd = &built_gcc_cmd; std::fprintf(stdout, - "psxrecomp: gcc tier using bundled toolchain (%s) with gcc from PATH\n", - tk_dir.string().c_str()); + "psxrecomp: gcc tier using bundled toolchain (%s) with %s\n", + tk_dir.string().c_str(), + tk_compiler.empty() ? "gcc from PATH" : tk_compiler.c_str()); } } if (const char *e = std::getenv("PSX_OVERLAY_AUTOCOMPILE_CMD")) { diff --git a/tools/release_stage.py b/tools/release_stage.py index cb87a8f2..daac84e6 100644 --- a/tools/release_stage.py +++ b/tools/release_stage.py @@ -636,10 +636,18 @@ def stage_toolchain(stage, recomp_dir, recomp_tools, recomp_include, dl_cache, # Windows needs the mingw runtime beside the recompiler; a Linux build links # against the system libstdc++ that is already present. if platform_tag == 'win': - if not mingw_bin: - _die('--mingw-bin is required when staging a Windows toolchain') + # A recompiler built against a static CRT (the cmake-clang-v1 emitters) needs no + # runtime DLLs; a gcc-built one does. Copy whatever the named bin dir (or the + # emitter's own directory) holds, and say which case this is. + dll_src = mingw_bin or recomp_dir + copied = 0 for d in ('libgcc_s_seh-1.dll', 'libstdc++-6.dll', 'libwinpthread-1.dll'): - shutil.copy2(os.path.join(mingw_bin, d), os.path.join(toolchain, d)) + p = os.path.join(dll_src, d) + if os.path.isfile(p): + shutil.copy2(p, os.path.join(toolchain, d)) + copied += 1 + log('mingw runtime DLLs staged beside the recompiler: %d (from %s)%s' + % (copied, dll_src, '' if copied else ' -- assuming a static recompiler')) shutil.copy2(os.path.join(recomp_tools, 'compile_overlays.py'), toolchain) tool_inc = _mkdirs(os.path.join(toolchain, 'include')) From 68ff52c14a23e19ea9e6bc398554eb0afa457e4d Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 17:44:36 +0300 Subject: [PATCH 45/56] packaging: gate overlay_cache = true and report it from --setup-selfcheck 78 of 105 published releases lack the key, and the runtime never initialises the overlay loader without it. The packager now refuses a staged game.toml without it unless --ship-without-overlay-cache-because states why, and the setup host's self-check reports overlay_cache_configured so CI can assert it. (cherry picked from commit a030f4843d8e55eafed4cb42eafc6d74bb89c086) --- host/psxrecomp_codegen_host.c | 28 ++++++++++++++++++++++++++++ tools/package_setup_host.sh | 16 ++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c index 33f3af39..eb178c7e 100644 --- a/host/psxrecomp_codegen_host.c +++ b/host/psxrecomp_codegen_host.c @@ -4353,6 +4353,34 @@ static void host_selfcheck_or_return(const PsxrecompCodegenHostConfig* cfg, printf(",\n \"game_dispatch_present\": %s", game_ok ? "true" : "false"); printf(",\n \"bios_backends_present\": %s", bios_ok ? "true" : "false"); printf(",\n \"sources_missing\": %s", missing ? "true" : "false"); + /* Wave-5 F4: overlay_cache is the switch that makes the runtime initialise the + * overlay loader at all; without it every streamed overlay is interpreted. CI + * asserts this field so a title cannot ship silently without it again. */ + { + char toml_abs[1200]; + int overlay_cache = 0; + if (join_path(toml_abs, sizeof(toml_abs), g_project_root, "game.toml")) { + FILE* tf = fopen(toml_abs, "rb"); + if (tf) { + char line[512]; + while (fgets(line, sizeof(line), tf)) { + const char* p = line; + while (*p == ' ' || *p == '\t') ++p; + if (strncmp(p, "overlay_cache", 13) == 0) { + p += 13; + while (*p == ' ' || *p == '\t') ++p; + if (*p == '=') { + ++p; + while (*p == ' ' || *p == '\t') ++p; + overlay_cache = strncmp(p, "true", 4) == 0; + } + } + } + fclose(tf); + } + } + printf(",\n \"overlay_cache_configured\": %s", overlay_cache ? "true" : "false"); + } printf("\n}\n"); fflush(stdout); exit(missing ? 2 : 0); diff --git a/tools/package_setup_host.sh b/tools/package_setup_host.sh index 7482e9fa..019b9357 100755 --- a/tools/package_setup_host.sh +++ b/tools/package_setup_host.sh @@ -64,6 +64,10 @@ DISPLAY_NAME="" RECOMPILER_BUILD="build-recompiler" VERSION_ENV="RELEASE_VERSION" DISC_HINT="your legally owned game disc" +# Wave-5 F4: a packaged game.toml without `overlay_cache = true` ships a runtime that never +# initialises the overlay loader, so every streamed overlay runs on the interpreter (78 of +# 105 releases measured 2026-09-16). Refuse unless the caller states why. +SHIP_WITHOUT_OVERLAY_CACHE_BECAUSE="" PROJECT_FILES=() PROJECT_DIRS=() RUNTIME_DIRS=() @@ -115,6 +119,7 @@ while [[ $# -gt 0 ]]; do --root) ROOT="${2:?}"; shift 2 ;; --embed-toolchain) EMBED_TOOLCHAIN=1; shift ;; --no-embed-toolchain) EMBED_TOOLCHAIN=0; shift ;; + --ship-without-overlay-cache-because) SHIP_WITHOUT_OVERLAY_CACHE_BECAUSE="${2:?}"; shift 2 ;; *) echo "error: unknown arg: $1" >&2 usage 2 @@ -484,6 +489,17 @@ fi bash "${STAGE_SDK}" "${stage_args[@]}" +if ! grep -qE '^[[:space:]]*overlay_cache[[:space:]]*=[[:space:]]*true' "${STAGE}/game.toml"; then + if [[ -z "${SHIP_WITHOUT_OVERLAY_CACHE_BECAUSE}" ]]; then + echo "error: REFUSING TO PACKAGE: ${STAGE}/game.toml has no '[runtime] overlay_cache = true'." >&2 + echo " The runtime never initialises the overlay loader without it, so every streamed" >&2 + echo " overlay runs on the dirty-RAM interpreter for every player. Add the key, or pass" >&2 + echo " --ship-without-overlay-cache-because '<reason>' to record why this title ships without it." >&2 + exit 1 + fi + echo "warning: packaging without overlay_cache = true (reason: ${SHIP_WITHOUT_OVERLAY_CACHE_BECAUSE})" >&2 +fi + cat >"${STAGE}/README-SETUP.txt" <<EOF ${DISPLAY_NAME} ${VERSION} — setup package Platform: ${ARTIFACT} From 1da867ea99f05599cafd778a195894edf2fc0b72 Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 20:00:33 +0300 Subject: [PATCH 46/56] runtime: recompute the text-modified bitmap after a state restore dirty_ram_text_guard_resync_after_restore() cleared text_modified_bitmap, and the page-clean fast path in dirty_ram_text_native_ok_ranges_from() then trusted a clear bit as "bytes still equal the reference" and skipped the compare. Restored RAM may carry self-modified text (the CMR2 class) that no live write will ever re-flag: the bitmap is not part of the state image, and the writes happened in the process that produced it. After a save-state load into a fresh process, a stale native body could be admitted for those pages. Recompute the bitmap from the bytes: one pass over the reference range (at most 2 MB, once per restore), setting the bit for every page that differs, so those pages stay under the exact compare. (cherry picked from commit ab186edf6, memory.c part) --- runtime/src/memory.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/runtime/src/memory.c b/runtime/src/memory.c index cb1c70a8..dbb05188 100644 --- a/runtime/src/memory.c +++ b/runtime/src/memory.c @@ -857,10 +857,33 @@ void dirty_ram_text_guard_resync_after_restore(void) { * forking MotK selfcheck warm #2vs#3 at matched clocks (win#118 class: * cold≡0, warm FAIL; post-span irq_resume also drifts). Drop both * host-only text-guard bitmaps; live writes re-arm modified, and the - * next native_ok compare re-decides diverge against restored bytes. */ + * next native_ok compare re-decides diverge against restored bytes. + * + * "Modified" is recomputed, not just forgotten. The page-clean fast + * path in dirty_ram_text_native_ok_ranges_from trusts a clear bit as + * "bytes still equal the reference" and skips the compare, but the + * restored RAM may carry self-modified text (CMR2 class) that no live + * write will ever re-flag: the bitmap is not part of the state image, + * and the writes happened in the process that produced it. One pass + * over the reference range (at most 2 MB, once per restore) puts every + * mismatching page back under the exact compare. */ memset(text_modified_bitmap, 0, sizeof(text_modified_bitmap)); memset(text_diverged_bitmap, 0, sizeof(text_diverged_bitmap)); g_text_diverged_pages = 0; + if (text_ref_image && text_ref_hi > text_ref_lo) { + uint32_t p0 = text_ref_lo >> DIRTY_RAM_PAGE_SHIFT; + uint32_t p1 = (text_ref_hi - 1u) >> DIRTY_RAM_PAGE_SHIFT; + for (uint32_t p = p0; p <= p1; p++) { + uint32_t lo = p << DIRTY_RAM_PAGE_SHIFT; + uint32_t hi = lo + (1u << DIRTY_RAM_PAGE_SHIFT); + if (lo < text_ref_lo) lo = text_ref_lo; + if (hi > text_ref_hi) hi = text_ref_hi; + if (hi > RAM_SIZE) hi = RAM_SIZE; + if (hi <= lo) continue; + if (memcmp(ram + lo, text_ref_image + (lo - text_ref_lo), hi - lo) != 0) + text_modified_bitmap[p >> 5] |= (1u << (p & 31u)); + } + } } void overlay_watch_invalidate_after_ram_restore(void) { From 59f17edf6ac957ad93463e6097f6ce907dd71c49 Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 17:58:01 +0300 Subject: [PATCH 47/56] runtime: do not classify MDEC-active stillness as a spin freeze During a boot FMV the guest parks in BIOS MDEC code with the dirty-RAM counters legitimately still while frames advance, which is exactly the spin signature. Every boot FMV wrote a 47-76 MB dump, 1.8 GB per kit in one session, inside timed windows. MDEC activity within the window now rules the classification out. Verified on NCII: heartbeat live through the FMV, zero dumps. (cherry picked from commit bf60c128218949e3e3af7ea867d50cc7a1e8a774) --- runtime/src/freeze_heartbeat.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/runtime/src/freeze_heartbeat.c b/runtime/src/freeze_heartbeat.c index 46d82b9d..56e78e22 100644 --- a/runtime/src/freeze_heartbeat.c +++ b/runtime/src/freeze_heartbeat.c @@ -1,5 +1,6 @@ /* freeze_heartbeat.c — see header for rationale. */ +#include "mdec.h" /* mdec_recently_active: FMV parks the guest in BIOS MDEC code */ #include "freeze_heartbeat.h" #include "freeze_dump_policy.h" #include "debug_server.h" @@ -814,8 +815,12 @@ static void heartbeat_write(void) { wedge_kind = 2; else if (frame_delta < WEDGE_SLOW_FRAMES_MAX_DELTA) wedge_kind = 3; - else if (logic_pinned) + else if (logic_pinned && !mdec_recently_active((uint32_t)frame_delta + 1u)) wedge_kind = 5; /* spin freeze: game wedged while frames advance */ + /* Wave-5 F9: during an FMV the guest parks in BIOS MDEC code with the dirty-RAM + * counters legitimately still while frames advance -- exactly the spin signature. + * Every boot FMV wrote a 47-76 MB dump (1.8 GB per kit in one session). MDEC + * activity inside the window rules the spin classification out. */ } if (!s_wedge_classification_paused && From b53ab4bd41d3b00ae3229781d965e04e26ade13e Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 20:35:27 +0300 Subject: [PATCH 48/56] release_stage: pin the overlay toolchain interpreter for macOS TOOLCHAIN_PINS had entries for win and linux only, and stage_toolchain picked 'linux' for every non-Windows host, so a Mac either died with "no toolchain pins" or staged a Linux interpreter it cannot run. No Mac player ever got a bundled toolchain. The stock /usr/bin/python3 on macOS 13/14 is 3.9, which cannot run compile_overlays.py (tomllib, PEP 604 unions), so the system interpreter is no fallback either. Add macos-x64 and macos-arm64 pins: the same python-build-standalone release and CPython 3.13.1 as the Linux pin, install_only_stripped assets, SHA256 measured by downloading both and matched against the publisher's .sha256 sidecars. host_platform_tag() picks the key from sys.platform and the machine architecture. No prebuilt tcc on macOS: the bundled interpreter drives the system clang. Verified on an Intel Mac (macOS 13, 8 GB): toolchain staged beside the Parasite Eve diagnostic product (45 MB), and with the POSIX spawner the runtime compiled 9 shards in the first 24 s of a headless boot. (cherry picked from commit 5c49e4d3cf835ed026964cd24998907c41a8be4c) --- tools/release_stage.py | 47 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/tools/release_stage.py b/tools/release_stage.py index daac84e6..1a6bc2d0 100644 --- a/tools/release_stage.py +++ b/tools/release_stage.py @@ -468,9 +468,47 @@ TOOLCHAIN_PINS = { 'tcc_url': None, 'tcc_sha256': None, }, + # macOS had no pin at all, so stage_toolchain died with "no toolchain pins" + # (or, before that check, staged the Linux interpreter) and no Mac player + # ever got a bundled toolchain; the stock /usr/bin/python3 on macOS 13/14 + # is 3.9, which cannot run compile_overlays.py (tomllib, PEP 604 unions). + # Same python-build-standalone release and version as the Linux pin, one + # asset per architecture. SHA256 measured 2026-09-16 by downloading both and + # matched against the publisher's .sha256 sidecar assets. No prebuilt tcc: + # the bundled interpreter drives the system clang (Xcode command-line tools). + 'macos-x64': { + 'python_version': '3.13.1', + 'python_url': ('https://github.com/astral-sh/python-build-standalone/releases/' + 'download/20250115/cpython-3.13.1%2B20250115-x86_64-apple-' + 'darwin-install_only_stripped.tar.gz'), + 'python_sha256': '26e0d5320bff7d141531e09849f0735c634bba31003ed6b089b9bf434312a773', + 'tcc_version': None, + 'tcc_url': None, + 'tcc_sha256': None, + }, + 'macos-arm64': { + 'python_version': '3.13.1', + 'python_url': ('https://github.com/astral-sh/python-build-standalone/releases/' + 'download/20250115/cpython-3.13.1%2B20250115-aarch64-apple-' + 'darwin-install_only_stripped.tar.gz'), + 'python_sha256': '650f1d3242667c64959391105525469e0fe1502a6aab9f5db3b0bfefe7dcbabd', + 'tcc_version': None, + 'tcc_url': None, + 'tcc_sha256': None, + }, } +def host_platform_tag(): + """The toolchain pin key for the machine this runs on.""" + if co.is_windows(): + return 'win' + if sys.platform == 'darwin': + import platform + return 'macos-arm64' if platform.machine() in ('arm64', 'aarch64') else 'macos-x64' + return 'linux' + + def get_pinned_archive(url, sha256, destination, retries=4, log=print): """Fetch `url` to `destination`, verifying SHA256 on EVERY use. @@ -576,8 +614,11 @@ def _extract_tar_top_level(archive, parent, expect_top): # this exact path (runtime/src/main.cpp, `tk_py`). The name differs per # platform; the layout does not. TOOLCHAIN_PY_REL = {'win': os.path.join('python', 'python.exe'), - 'linux': os.path.join('python', 'bin', 'python3')} -TOOLCHAIN_RECOMPILER = {'win': 'psxrecomp-game.exe', 'linux': 'psxrecomp-game'} + 'linux': os.path.join('python', 'bin', 'python3'), + 'macos-x64': os.path.join('python', 'bin', 'python3'), + 'macos-arm64': os.path.join('python', 'bin', 'python3')} +TOOLCHAIN_RECOMPILER = {'win': 'psxrecomp-game.exe', 'linux': 'psxrecomp-game', + 'macos-x64': 'psxrecomp-game', 'macos-arm64': 'psxrecomp-game'} def stage_toolchain(stage, recomp_dir, recomp_tools, recomp_include, dl_cache, @@ -593,7 +634,7 @@ def stage_toolchain(stage, recomp_dir, recomp_tools, recomp_include, dl_cache, so on Linux neither the AOT cache nor the capture-and-compile fail-safe could extend itself. """ - platform_tag = platform_tag or ('win' if co.is_windows() else 'linux') + platform_tag = platform_tag or host_platform_tag() if platform_tag not in TOOLCHAIN_PINS: _die('no toolchain pins for platform %r' % platform_tag) pins = TOOLCHAIN_PINS[platform_tag] From d2e2c928874fee5e5052c115a2456e018d501a52 Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis <Alexbeav@live.com> Date: Wed, 16 Sep 2026 21:20:05 +0300 Subject: [PATCH 49/56] setup: keep the CLI's reason when disc verification fails cli_fail_msg replaced whatever the CLI said on exit 3 with a flat "Disc verification failed (wrong dump)." With the CHD reader, the CLI's own line can be "... is a CHD and the CHD reader (libchdr) is not built for this kit ... this does not mean your dump is bad", and the wizard then contradicted it. cli_tail_note already captures that line; show it, and fall back to the old text only when the CLI said nothing. Carried over from #369, which was closed in favour of this change. --- host/psxrecomp_codegen_host.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c index 33f3af39..c0c58e65 100644 --- a/host/psxrecomp_codegen_host.c +++ b/host/psxrecomp_codegen_host.c @@ -1816,7 +1816,14 @@ static void cli_fail_msg(char* err_msg, size_t err_cap, const char* fail_label, long code, const CliTail* t) { const char* why = t->last_err[0] ? t->last_err : t->last; if (code == 3) { - snprintf(err_msg, err_cap, "Disc verification failed (wrong dump)."); + /* The CLI names the failing check (a digest mismatch, or a .chd it + * cannot read) and cli_tail_note has captured that line. A flat + * "wrong dump" contradicts it and sends players hunting a bad rip + * when the dump is fine. Keep the reason when there is one. */ + if (why[0]) + snprintf(err_msg, err_cap, "Disc verification failed: %s", why); + else + snprintf(err_msg, err_cap, "Disc verification failed (wrong dump)."); return; } if (why[0]) From 8b50cd09e4f9a7d1e50640f51912829c2c65d6b4 Mon Sep 17 00:00:00 2001 From: Kevin <9866117+kerokline@users.noreply.github.com> Date: Thu, 17 Sep 2026 06:12:58 -0400 Subject: [PATCH 50/56] emit_dispatch: accept the OpenBIOS call-vector stub shape natively The generated psx_bios_try_native_call_stub reads the four live words at the A0/B0/C0 call vectors and, when they match the expected stub, decodes the handler address and tail-transfers to it without the dirty-RAM interpreter. The guard knew one shape, the retail kernel's lui $t0,hi; addiu $t0,$t0,lo; jr $t0; nop OpenBIOS installs addiu $t0,$zero,lo; jr $t0; nop; nop so on OpenBIOS the guard failed closed and every kernel call paid an interpreter entry for two instructions. Breath of Fire III yields through B0:10h ChangeTh ~230 times per frame in the field: ~460 interpreted instructions per frame, 87% of what OpenBIOS kernel RAM still interpreted after the install-slot ranges (#346), and the same A/B reads 0 on retail. Accept both shapes. The target is still decoded from live RAM on every call and any other word pattern still fails closed (Rule 18). Cycle steps charge the instructions the guest executes on each shape: four for the retail stub, three for OpenBIOS (its jr's delay slot is word 2). Dispatch totals are unchanged (OpenBIOS 3990, SCPH1001 13012). Adds recompiler/tests/test_bios_vector_stub_shapes.py (registered with CTest next to bios_backend_constant_codegen). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --- recompiler/CMakeLists.txt | 6 ++ recompiler/src/full_function_emitter.cpp | 57 +++++++++---- .../tests/test_bios_vector_stub_shapes.py | 80 +++++++++++++++++++ 3 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 recompiler/tests/test_bios_vector_stub_shapes.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index a4ae57b9..84888210 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -637,6 +637,12 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_bios_backend_constant_codegen.py --recompiler $<TARGET_FILE:psxrecomp-bios> WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + add_test( + NAME bios_vector_stub_shapes + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_bios_vector_stub_shapes.py + --recompiler $<TARGET_FILE:psxrecomp-bios> + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) add_test( NAME stall_report COMMAND ${Python3_EXECUTABLE} diff --git a/recompiler/src/full_function_emitter.cpp b/recompiler/src/full_function_emitter.cpp index 157f1b30..b6bf9d26 100644 --- a/recompiler/src/full_function_emitter.cpp +++ b/recompiler/src/full_function_emitter.cpp @@ -2063,11 +2063,17 @@ void FullFunctionEmitter::emit_dispatch( rom.size(), crc, wsum, bios_sha256, id, bundled); } // --- Runtime-installed BIOS call-vector stubs --- - // Every retail PSX BIOS installs the A0/B0/C0 ABI gates as the same - // four-instruction shape: lui/addiu $t0,target; jr $t0; nop. The target is - // BIOS-specific, so decode it from live RAM and require the complete shape - // before taking the native tail transfer. A game/BIOS patch that changes - // even one word fails closed to dirty_ram_interp below. + // Every PSX BIOS installs the A0/B0/C0 ABI gates as a 16-byte stub that + // loads the handler address into $t0 and jumps through it. Two shapes are + // in the wild: + // A (retail SCPH-xxxx): lui $t0,hi; addiu $t0,$t0,lo; jr $t0; nop + // B (OpenBIOS): addiu $t0,$zero,lo; jr $t0; nop; nop + // The target is BIOS-specific, so decode it from live RAM and require one + // complete shape before taking the native tail transfer. A game/BIOS patch + // that changes even one word fails closed to dirty_ram_interp below. The + // cycle steps charge exactly the instructions the guest executes on that + // shape (shape B's fourth word is never reached: the jr's delay slot is + // word 2). out += fmt::format("static const PsxNativeStub {}psx_bios_native_stubs[3] = {{\n", g_sym_prefix); out += " { 0x000000A0u, 0x000000A0u, 0x000000B0u },\n"; @@ -2084,23 +2090,42 @@ void FullFunctionEmitter::emit_dispatch( out += " uint32_t w1 = cpu->read_word(phys + 4u);\n"; out += " uint32_t w2 = cpu->read_word(phys + 8u);\n"; out += " uint32_t w3 = cpu->read_word(phys + 12u);\n"; - out += " if ((w0 & 0xFFFF0000u) != 0x3C080000u ||\n"; - out += " (w1 & 0xFFFF0000u) != 0x25080000u ||\n"; - out += " w2 != 0x01000008u || w3 != 0u) return 0;\n"; + out += " uint32_t target;\n"; + out += " if ((w0 & 0xFFFF0000u) == 0x3C080000u &&\n"; + out += " (w1 & 0xFFFF0000u) == 0x25080000u &&\n"; + out += " w2 == 0x01000008u && w3 == 0u) {\n"; + out += " /* shape A: lui $t0,hi; addiu $t0,$t0,lo; jr $t0; nop */\n"; + out += " target = ((w0 & 0xFFFFu) << 16) +\n"; + out += " (uint32_t)(int32_t)(int16_t)(w1 & 0xFFFFu);\n"; out += "#ifdef PSX_ENABLE_BLOCK_CYCLES\n"; - out += " psx_icache_fetch(cpu, addr);\n"; - out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + out += " psx_icache_fetch(cpu, addr);\n"; + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", psx_cyc_dep_res_mask(0x3C080000u)); - out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", psx_cyc_dep_res_mask(0x25080000u)); - out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", psx_cyc_dep_res_mask(0x01000008u)); - out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", psx_cyc_dep_res_mask(0x00000000u)); out += "#endif\n"; - out += " cpu->gpr[8] = ((w0 & 0xFFFFu) << 16) +\n"; - out += " (uint32_t)(int32_t)(int16_t)(w1 & 0xFFFFu);\n"; - out += " cpu->pc = cpu->gpr[8];\n"; + out += " } else if ((w0 & 0xFFFF0000u) == 0x24080000u &&\n"; + out += " w1 == 0x01000008u && w2 == 0u && w3 == 0u) {\n"; + out += " /* shape B: addiu $t0,$zero,lo; jr $t0; nop; nop */\n"; + out += " target = (uint32_t)(int32_t)(int16_t)(w0 & 0xFFFFu);\n"; + out += "#ifdef PSX_ENABLE_BLOCK_CYCLES\n"; + out += " psx_icache_fetch(cpu, addr);\n"; + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x24080000u)); + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x01000008u)); + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x00000000u)); + out += "#endif\n"; + out += " } else {\n"; + out += " return 0;\n"; + out += " }\n"; + out += " cpu->gpr[8] = target;\n"; + out += " cpu->pc = target;\n"; out += " return 1;\n"; out += "}\n\n"; diff --git a/recompiler/tests/test_bios_vector_stub_shapes.py b/recompiler/tests/test_bios_vector_stub_shapes.py new file mode 100644 index 00000000..e3e2b5ba --- /dev/null +++ b/recompiler/tests/test_bios_vector_stub_shapes.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""The emitted A0/B0/C0 native call-stub guard must accept both stub shapes. + +Retail kernels install `lui $t0,hi; addiu $t0,$t0,lo; jr $t0; nop` at the +call vectors; OpenBIOS installs `addiu $t0,$zero,lo; jr $t0; nop; nop`. A +guard that knows only the first shape fails closed on OpenBIOS and every +kernel call then pays a dirty-RAM interpreter entry (Breath of Fire III: +~230 per frame in the field, ~460 interpreted instructions per frame). +""" + +import argparse +import os +import re +import subprocess +import sys +import tempfile + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + root = os.path.normpath(os.path.join(here, "..", "..")) + default = os.path.join(root, "recompiler", "build", "psxrecomp-bios.exe") + + parser = argparse.ArgumentParser() + parser.add_argument("--recompiler", default=default) + args = parser.parse_args() + + if not os.path.isfile(args.recompiler): + print(f"FAIL: recompiler not found: {args.recompiler}", file=sys.stderr) + return 2 + + with tempfile.TemporaryDirectory() as out_dir: + result = subprocess.run( + [args.recompiler, "--config", + os.path.join(root, "bios", "OpenBIOS.toml"), + "--out-dir", out_dir], + cwd=root, capture_output=True, text=True, + ) + if result.returncode: + print(result.stderr or result.stdout, file=sys.stderr) + return 1 + with open(os.path.join(out_dir, "OpenBIOS_dispatch.c"), + encoding="utf-8") as stream: + generated = stream.read() + + m = re.search(r"static int psx_bios_try_native_call_stub\(.*?\n\}\n", + generated, re.S) + if not m: + print("FAIL: no psx_bios_try_native_call_stub in the dispatch", + file=sys.stderr) + return 1 + body = m.group(0) + + checks = { + "shape A guard (lui $t0)": + "(w0 & 0xFFFF0000u) == 0x3C080000u" in body, + "shape A guard (addiu $t0,$t0)": + "(w1 & 0xFFFF0000u) == 0x25080000u" in body, + "shape B guard (addiu $t0,$zero)": + "(w0 & 0xFFFF0000u) == 0x24080000u" in body, + "shape B guard (jr $t0 in word 1)": + "w1 == 0x01000008u && w2 == 0u && w3 == 0u" in body, + "shape B target is sign-extended word-0 immediate": + "target = (uint32_t)(int32_t)(int16_t)(w0 & 0xFFFFu);" in body, + "unrecognised words fail closed": + "return 0;" in body, + "native tail transfer publishes $t0 and pc": + "cpu->gpr[8] = target;" in body and "cpu->pc = target;" in body, + } + failed = [k for k, ok in checks.items() if not ok] + for k in failed: + print(f"FAIL: {k}", file=sys.stderr) + if failed: + return 1 + print("PASS: native call-stub guard accepts the retail and OpenBIOS shapes.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 798b66bbdc72953bccc3d5e7e4f9fcbf82f20612 Mon Sep 17 00:00:00 2001 From: Kevin Kline <9866117+kerokline@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:36:35 -0400 Subject: [PATCH 51/56] compile_overlays: name the no-walk-root skip message so its test cannot drift test_compile_overlays_static_split.test_data_only_capture_reports_skip_with_captured_log has been red on master since 4683e923 (2026-09-12). That commit reworded both emit sites deliberately -- "no walk-root seeds (data-only region)" became "no shared walk-root seeds; checking fragments separately", because the region is no longer declared data-only: its executed dispatch demands are still served by the isolated-fragment pass. The assertion still names the old text, so the test fails on a clean checkout: AssertionError: 'SKIP: no walk-root seeds' not found in '... SKIP: no shared walk-root seeds; checking fragments separately ...' The literal was duplicated at the two emit sites and copied a third time into the test, so rewording two of the three left the third stale with nothing to catch it. Give it a name -- NO_SHARED_WALK_ROOT_SEEDS_SKIP -- use it at both emit sites, and have the test assert the constant rather than a copy of its text. A future reword then updates all three at once or fails loudly. The printed output is unchanged, verified byte for byte against the pre-change text including both leading spaces and the trailing blank line. tools/test_compile_overlays_static_split.py: 10 tests, was 9 passed 1 failed, now 10 passed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- tools/compile_overlays.py | 13 +++++++++++-- tools/test_compile_overlays_static_split.py | 6 +++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 09ba2016..8ce786e8 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -3255,6 +3255,15 @@ def current_variant_func_id_coverage(func_ids: list, data: bytes, # contract/header change (which bumps the hash -> fresh dir -> fresh memo) # re-attempts everything (e.g. the psx_rfe_mark_escape contract fix). # So a memoized skip only ever elides a build that is deterministically doomed. +# Printed when a capture contributes no walk-root seed. The wording is +# deliberate: the region is NOT declared data-only -- its executed dispatch +# demands are still served by the isolated-fragment pass, which runs +# separately. Named so the two emit sites and the test cannot drift apart. +# They did: 4683e923 reworded both prints and left +# test_compile_overlays_static_split asserting the old string. +NO_SHARED_WALK_ROOT_SEEDS_SKIP = ( + 'SKIP: no shared walk-root seeds; checking fragments separately') + INTERIOR_FAIL_MEMO = 'interior_fail_memo.txt' @@ -5818,7 +5827,7 @@ def _static_capture_job(cap, args, toml, forced_interiors, static_out, result): seed.split()[0].startswith('0x')) ] if not root_seeds: - print(' SKIP: no shared walk-root seeds; checking fragments separately\n') + print(f' {NO_SHARED_WALK_ROOT_SEEDS_SKIP}\n') result['outcome'] = 'skip' return @@ -6366,7 +6375,7 @@ def main(): seed.split()[0].startswith('0x')) ] if not root_seeds: - print(' SKIP: no shared walk-root seeds; checking fragments separately\n') + print(f' {NO_SHARED_WALK_ROOT_SEEDS_SKIP}\n') stats.add_skip() return diff --git a/tools/test_compile_overlays_static_split.py b/tools/test_compile_overlays_static_split.py index 0d9f8c3f..1c67d16c 100644 --- a/tools/test_compile_overlays_static_split.py +++ b/tools/test_compile_overlays_static_split.py @@ -227,7 +227,11 @@ class StaticWorkerContractTests(unittest.TestCase): 'overlays_static.c') self.assertEqual(res['outcome'], 'skip') self.assertIsNone(res['part']) - self.assertIn('SKIP: no walk-root seeds', res['log']) + # Assert the constant, not a copy of its text: this line asserted a + # stale literal from 4683e923 until 2026-09-18 and the test was red + # on master the whole time. + self.assertIn(compile_overlays.NO_SHARED_WALK_ROOT_SEEDS_SKIP, + res['log']) self.assertEqual(res['requested_entries'], set()) From 32831ff758fb95aebbfb6dc6eb4fb2af0971cbe5 Mon Sep 17 00:00:00 2001 From: Alex Vanderveen <alex@technicallycomputers.ca> Date: Fri, 18 Sep 2026 19:38:53 -0400 Subject: [PATCH 52/56] Update package_setup_host.sh --- tools/package_setup_host.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/package_setup_host.sh b/tools/package_setup_host.sh index 019b9357..c82d42bf 100755 --- a/tools/package_setup_host.sh +++ b/tools/package_setup_host.sh @@ -534,8 +534,12 @@ EOF # a cmake variable cannot be resolved here. if [[ -f "${STAGE}/CMakeLists.txt" ]]; then cml="${STAGE}/CMakeLists.txt" - guarded="$(grep -oE 'if\(EXISTS[[:space:]]+"\$\{CMAKE_CURRENT_SOURCE_DIR\}/[^"]+"' "${cml}" \ - | sed -E 's|.*\$\{CMAKE_CURRENT_SOURCE_DIR\}/||; s|"$||' | sort -u)" + # A CMakeLists.txt with no if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/...") guard at + # all is normal, and grep exits 1 on no match. Under `set -e -o pipefail` that + # status propagates out of the command substitution and kills the packager + # here — silently, before any gate can report anything. Guard the assignment. + guarded="$( { grep -oE 'if\(EXISTS[[:space:]]+"\$\{CMAKE_CURRENT_SOURCE_DIR\}/[^"]+"' "${cml}" \ + || true; } | sed -E 's|.*\$\{CMAKE_CURRENT_SOURCE_DIR\}/||; s|"$||' | sort -u)" missing_refs=() while IFS= read -r rel; do [[ -z "${rel}" ]] && continue From 6c98726a66e626e548b9630f8d445531ba4f51ce Mon Sep 17 00:00:00 2001 From: Kevin <9866117+kerokline@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:59:49 -0400 Subject: [PATCH 53/56] code_generator: resolve computed-stride jumps into unrolled runs (Duff's device) A `jr R` with `R = base + index * stride` and no table in memory is how unrolled copy loops enter their body in the middle (Duff's device). The bounded-jump-table resolver only knows the `sltiu/beq; sll; addu; lw; jr` chain, so the emitter fell back to a bare CPS tail-transfer; the trampoline then found no dispatch key at the interior PC (straight-line code, no block leader) and the dirty-RAM interpreter ran the rest of the copy on every call. On Breath of Fire III that is the file decompressor's literal-copy loop (`0x80164D1C`, 32 groups of `lbu/nop/sb`), which fires on every load and was the whole non-kernel interpreted residual of a play session once the kernel patch ranges (#346) and the call vectors (#381) ran native. resolve_computed_stride_jump walks the chain back from the jr through nearest definitions in straight-line code: `addu R, B, S` with S from `sll S, I, k` or the two-sll sum `sll P,I,a; sll Q,I,b; addu S,P,Q`, and then requires the code after the delay slot to be N >= 2 shape-identical groups of `stride` bytes (same opcodes and register fields, immediates free, no control flow inside a group). Targets are run_start + k * stride for k = 0..N. The base register's value is never assumed: the emitter switches on the actual runtime target with the CPS tail-transfer as the default, so the recovered cases can only add native entries, never redirect a jump that lands elsewhere. Wired into both code-generator sites (interior labels + the switch) and the exact-entry discovery walker. On the title's boot EXE it matches exactly one site, the decompressor loop (33 entries); every other jr is untouched. Adds recompiler/tests/test_computed_stride_jump_codegen.py (synthetic PS-EXE with the idiom; asserts the switch, every case and the interior labels), registered with CTest. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --- recompiler/CMakeLists.txt | 1 + recompiler/include/function_analysis.h | 33 ++++ recompiler/src/code_generator.cpp | 24 ++- recompiler/src/function_analysis.cpp | 160 +++++++++++++++++- .../test_computed_stride_jump_codegen.py | 132 +++++++++++++++ 5 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 recompiler/tests/test_computed_stride_jump_codegen.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index a82aa26b..1c41506b 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -688,6 +688,7 @@ if(BUILD_TESTING) foreach(_t IN ITEMS aot_base_recovery bios_resident_shard + computed_stride_jump_codegen msvc_cps_codegen overlay_guard_codegen overlay_store_barrier_codegen diff --git a/recompiler/include/function_analysis.h b/recompiler/include/function_analysis.h index ea887462..4eb5d6b8 100644 --- a/recompiler/include/function_analysis.h +++ b/recompiler/include/function_analysis.h @@ -72,6 +72,10 @@ struct ExactJumpTable { uint32_t table_base = 0; uint32_t table_count = 0; std::vector<std::pair<uint32_t, uint32_t>> targets; + // Non-zero when the targets came from resolve_computed_stride_jump + // rather than a table in memory: `table_base` is then the first word of + // the unrolled run and the targets are table_base + k * stride. + uint32_t stride = 0; }; using ExactAddressMapper = uint32_t (*)(uint32_t, const PS1Executable&); @@ -97,6 +101,35 @@ bool resolve_exact_bounded_jump_table( uint32_t producer_lo = 0, uint32_t producer_hi = 0); +// Recognize a computed-stride entry into an unrolled run (Duff's device): +// +// sll S, I, k (stride = 1<<k) +// or +// sll P, I, a ; sll Q, I, b ; addu S, P, Q (stride = (1<<a)+(1<<b)) +// addu R, B, S (either operand order) +// jr R +// <delay slot> +// run_start: N >= 2 shape-identical groups of `stride` bytes — the same +// opcodes and register fields in every group, immediates free, +// no control flow inside a group. +// +// The dependency chain is walked backwards from the jr through nearest +// definitions with no control flow in between. The base register's value is +// deliberately NOT assumed: the caller emits a switch on the actual runtime +// target with the CPS tail-transfer as the default, so the recovered targets +// only ever add native cases and can never redirect a jump that lands +// elsewhere. Targets are run_start + k * stride for k = 0..N (k = N is the +// word after the run, the loop tail); all lie inside [entry, hard_cap). +// `table.stride` is set to the stride; `table.table_base` to run_start. +bool resolve_computed_stride_jump( + const PS1Executable& exe, + uint32_t entry, + uint32_t hard_cap, + uint32_t jr_pc, + uint32_t jr_rs, + ExactJumpTable& table, + ExactAddressMapper runtime_to_image = nullptr); + class FunctionAnalyzer { public: explicit FunctionAnalyzer(const PS1Executable& exe); diff --git a/recompiler/src/code_generator.cpp b/recompiler/src/code_generator.cpp index ef96f8d8..1a0318e2 100644 --- a/recompiler/src/code_generator.cpp +++ b/recompiler/src/code_generator.cpp @@ -2322,6 +2322,15 @@ std::string CodeGenerator::translate_basic_block( exe_, cfg.function_start, cfg.function_end, block.exit_instr.address, jr_rs, exact_table, ram_to_rom, cfg.producer_lo, cfg.producer_hi); + // Not a table: a computed-stride entry into an unrolled + // run (Duff's device — the decompressor copy loops). Same + // switch shape; the CPS default keeps every other target. + if (!have_exact_table) { + have_exact_table = resolve_computed_stride_jump( + exe_, cfg.function_start, cfg.function_end, + block.exit_instr.address, jr_rs, exact_table, + ram_to_rom); + } uint32_t table_base = have_exact_table ? exact_table.table_base : 0u; uint32_t table_count = have_exact_table @@ -2344,8 +2353,14 @@ std::string CodeGenerator::translate_basic_block( } } if (!targets.empty()) { - ss << config_.indent << fmt::format("/* jump table 0x{:08X} (rom 0x{:08X}), {} entries */\n", - table_base, rom_table_base, table_count); + if (exact_table.stride != 0u) { + ss << config_.indent << fmt::format( + "/* computed-stride jump into unrolled run 0x{:08X} (rom 0x{:08X}), stride {}, {} entries */\n", + table_base, rom_table_base, exact_table.stride, table_count); + } else { + ss << config_.indent << fmt::format("/* jump table 0x{:08X} (rom 0x{:08X}), {} entries */\n", + table_base, rom_table_base, table_count); + } ss << config_.indent << fmt::format("switch ({}) {{\n", delay_saved_target); for (auto& [rt, rom] : targets) { if (partial_block_cycle_count(rom, cfg) != 0) { @@ -2898,7 +2913,10 @@ void CodeGenerator::scan_jr_tables( if (!resolve_exact_bounded_jump_table( exe_, cfg.function_start, cfg.function_end, blk.exit_instr.address, jr_r, exact_table, ram_to_rom, - cfg.producer_lo, cfg.producer_hi)) { + cfg.producer_lo, cfg.producer_hi) && + !resolve_computed_stride_jump( + exe_, cfg.function_start, cfg.function_end, + blk.exit_instr.address, jr_r, exact_table, ram_to_rom)) { continue; } uint32_t tb = exact_table.table_base; diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index ef721a62..658868b0 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -1133,6 +1133,162 @@ bool resolve_exact_bounded_jump_table( return true; } +bool resolve_computed_stride_jump( + const PS1Executable& exe, + uint32_t entry, + uint32_t hard_cap, + uint32_t jr_pc, + uint32_t jr_rs, + ExactJumpTable& table, + ExactAddressMapper runtime_to_image) { + table = {}; + if (entry >= hard_cap || jr_pc < entry || jr_pc + 8u >= hard_cap || + (entry & 3u) != 0u || (hard_cap & 3u) != 0u || + (jr_pc & 3u) != 0u || jr_rs == 0u || jr_rs == 31u) { + return false; + } + auto read = [&](uint32_t pc) { return exe.read_word(pc); }; + auto jr_word = read(jr_pc); + if (!jr_word.has_value() || + *jr_word != ((jr_rs & 0x1Fu) << 21u | 0x08u)) { + return false; + } + + // Nearest definition of `reg` strictly before `use_pc`, walking back at + // most `window` instructions through straight-line code only. A control + // transfer (or its delay slot) in the way ends the search: the chain + // must be one basic block so every path to the jr computes the same + // shape. + auto nearest_def = [&](uint32_t reg, uint32_t use_pc, + uint32_t window) -> std::optional<uint32_t> { + if (reg == 0u) return std::nullopt; + uint32_t pc = use_pc; + for (uint32_t n = 0; n < window && pc >= entry + 4u; ++n) { + pc -= 4u; + auto w = read(pc); + if (!w.has_value()) return std::nullopt; + if (exact_classify_cf(pc, *w).kind != ExactCfKind::Normal) + return std::nullopt; + if (exact_instruction_writes_gpr(*w, reg)) return pc; + } + return std::nullopt; + }; + auto fields = [&](uint32_t w, uint32_t& op, uint32_t& rs, uint32_t& rt, + uint32_t& rd, uint32_t& sa, uint32_t& fn) { + op = (w >> 26) & 0x3Fu; rs = (w >> 21) & 0x1Fu; rt = (w >> 16) & 0x1Fu; + rd = (w >> 11) & 0x1Fu; sa = (w >> 6) & 0x1Fu; fn = w & 0x3Fu; + }; + auto is_addu = [&](uint32_t w, uint32_t& rd, uint32_t& rs, uint32_t& rt) { + uint32_t op, sa, fn; + fields(w, op, rs, rt, rd, sa, fn); + return op == 0u && fn == 0x21u && sa == 0u && rd != 0u; + }; + // sll rd, rt, sa -> (rd, index rt, shift) + auto is_sll = [&](uint32_t w, uint32_t& rd, uint32_t& idx, uint32_t& sh) { + uint32_t op, rs, fn; + fields(w, op, rs, idx, rd, sh, fn); + return op == 0u && fn == 0u && rs == 0u && rd != 0u && idx != 0u && w != 0u; + }; + + // 1. jr_rs = addu B, S (either order), nearest definition before the jr. + auto addu_pc = nearest_def(jr_rs, jr_pc, 8u); + if (!addu_pc.has_value()) return false; + uint32_t a_rd, a_rs, a_rt; + if (!is_addu(*read(*addu_pc), a_rd, a_rs, a_rt) || a_rs == a_rt || + a_rs == 0u || a_rt == 0u) { + return false; + } + + // 2. One operand is the scaled index: sll S,I,k or the two-sll sum. + // Try each operand as the scaled register; the other is the base. + auto scaled_stride = [&](uint32_t s_reg, uint32_t use_pc) -> uint32_t { + auto def = nearest_def(s_reg, use_pc, 8u); + if (!def.has_value()) return 0u; + uint32_t w = *read(*def); + uint32_t rd, idx, sh; + if (is_sll(w, rd, idx, sh)) { + return (sh >= 2u && sh <= 8u) ? (1u << sh) : 0u; + } + uint32_t s_rd, p, q; + if (!is_addu(w, s_rd, p, q) || p == q || p == 0u || q == 0u) return 0u; + auto p_def = nearest_def(p, *def, 8u); + auto q_def = nearest_def(q, *def, 8u); + if (!p_def.has_value() || !q_def.has_value()) return 0u; + uint32_t p_rd, p_idx, p_sh, q_rd, q_idx, q_sh; + if (!is_sll(*read(*p_def), p_rd, p_idx, p_sh) || + !is_sll(*read(*q_def), q_rd, q_idx, q_sh) || + p_idx != q_idx || p_sh == q_sh || p_sh > 8u || q_sh > 8u) { + return 0u; + } + // The index must reach both shifts unchanged: no write to it between + // the earlier sll and the later one. + uint32_t lo = std::min(*p_def, *q_def), hi = std::max(*p_def, *q_def); + for (uint32_t pc = lo + 4u; pc < hi; pc += 4u) { + if (exact_instruction_writes_gpr(*read(pc), p_idx)) return 0u; + } + return (1u << p_sh) + (1u << q_sh); + }; + uint32_t stride = scaled_stride(a_rt, *addu_pc); + if (stride == 0u) stride = scaled_stride(a_rs, *addu_pc); + if (stride == 0u || (stride & 3u) != 0u || stride > 256u) return false; + + // 3. The run: N >= 2 shape-identical groups right after the delay slot. + // Same opcode/register fields per position; immediates free; no + // control flow inside group 0 (so every group is straight-line). + const uint32_t run_start = jr_pc + 8u; + const uint32_t words_per_group = stride / 4u; + if (run_start + stride > hard_cap) return false; + auto shape = [](uint32_t w) -> uint32_t { + uint32_t op = (w >> 26) & 0x3Fu; + // I-type: keep opcode + rs + rt, drop the immediate. J-type and + // SPECIAL/REGIMM/COP forms compare whole. + if (op == 0u || op == 1u || op == 2u || op == 3u || op == 0x10u || + op == 0x12u) { + return w; + } + return w & 0xFFFF0000u; + }; + std::vector<uint32_t> group0; + bool all_nop = true; + for (uint32_t i = 0; i < words_per_group; ++i) { + auto w = read(run_start + i * 4u); + if (!w.has_value()) return false; + if (exact_classify_cf(run_start + i * 4u, *w).kind != ExactCfKind::Normal) + return false; + if (*w != 0u) all_nop = false; + group0.push_back(*w); + } + if (all_nop) return false; + uint32_t groups = 1; + for (;;) { + uint32_t g = run_start + groups * stride; + if (g + stride > hard_cap) break; + bool same = true; + for (uint32_t i = 0; i < words_per_group && same; ++i) { + auto w = read(g + i * 4u); + same = w.has_value() && shape(*w) == shape(group0[i]); + } + if (!same) break; + ++groups; + } + if (groups < 2u) return false; + + auto mapped = [&](uint32_t addr) { + return runtime_to_image ? runtime_to_image(addr, exe) : addr; + }; + ExactJumpTable resolved; + resolved.table_base = run_start; + resolved.stride = stride; + for (uint32_t k = 0; k <= groups; ++k) { + uint32_t t = run_start + k * stride; + if (t >= hard_cap) break; + resolved.targets.emplace_back(t, mapped(t)); + } + resolved.table_count = static_cast<uint32_t>(resolved.targets.size()); + table = std::move(resolved); + return true; +} + FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( const std::vector<uint32_t>& entries, const std::vector<std::pair<uint32_t, uint32_t>>& producer_ranges, @@ -1304,7 +1460,9 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( } if (resolve_exact_bounded_jump_table( exe_, entry, hard_cap, pc, jr_rs, table, - nullptr, producer_lo, producer_hi)) { + nullptr, producer_lo, producer_hi) || + resolve_computed_stride_jump( + exe_, entry, hard_cap, pc, jr_rs, table)) { bool all_owned = std::all_of( table.targets.begin(), table.targets.end(), [&](const auto& target_pair) { diff --git a/recompiler/tests/test_computed_stride_jump_codegen.py b/recompiler/tests/test_computed_stride_jump_codegen.py new file mode 100644 index 00000000..2bfc9a4f --- /dev/null +++ b/recompiler/tests/test_computed_stride_jump_codegen.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Codegen test: a computed-stride jump into an unrolled run gets a switch. + +Duff's-device copy loops enter their unrolled body in the middle through +`jr R` with `R = base + index * stride`. There is no table in memory, so the +bounded-jump-table resolver cannot see it and the emitter used to fall back +to a bare CPS tail-transfer; the trampoline then found no dispatch key at the +interior PC and the dirty-RAM interpreter ran the rest of the copy on every +call (Breath of Fire III's decompressor: the whole non-kernel residual of a +play session). resolve_computed_stride_jump recognises the shape and the +emitter registers the interior targets and switches on them. + +Synthesized function (load 0x80010000): + + sll t7,a0,3 + sll t5,a0,2 + addu t7,t7,t5 ; t7 = a0 * 12 + addu t7,a1,t7 ; t7 = base + a0 * 12 + jr t7 + nop + lbu t5,0(a2); nop; sb t5,0(a3) ; group 0 @ +0x18 + lbu t5,1(a2); nop; sb t5,1(a3) ; group 1 @ +0x24 + lbu t5,2(a2); nop; sb t5,2(a3) ; group 2 @ +0x30 + lbu t5,3(a2); nop; sb t5,3(a3) ; group 3 @ +0x3C + jr ra ; nop ; tail @ +0x48 + +Expected: a `computed-stride jump` switch with cases for every group and the +tail, each a `goto block_...`, and the CPS tail-transfer kept as default. + +Usage: python test_computed_stride_jump_codegen.py [--recompiler <psxrecomp-game.exe>] +""" +import argparse +import os +import re +import struct +import subprocess +import sys +import tempfile + +LOAD = 0x80010000 + + +def w(words): + return b"".join(struct.pack("<I", x) for x in words) + + +def make_psxexe(entry, data): + h = bytearray(2048) + h[0:8] = b"PS-X EXE" + struct.pack_into("<I", h, 0x10, entry) + struct.pack_into("<I", h, 0x18, LOAD) + struct.pack_into("<I", h, 0x1C, len(data)) + return bytes(h) + data + + +def build_exe(): + body = [ + 0x000478C0, # sll t7,a0,3 + 0x00046880, # sll t5,a0,2 + 0x01ED7821, # addu t7,t7,t5 + 0x00AF7821, # addu t7,a1,t7 + 0x01E00008, # jr t7 + 0x00000000, # nop + ] + for k in range(4): + body += [0x90CD0000 | k, 0x00000000, 0xA0ED0000 | k] # lbu t5,k(a2); nop; sb t5,k(a3) + body += [0x03E00008, 0x00000000] # jr ra; nop + return make_psxexe(LOAD, w(body)) + + +def gen_c(recompiler, tmp): + psx = os.path.join(tmp, "t.psx") + seeds = os.path.join(tmp, "seeds.txt") + out = os.path.join(tmp, "out") + os.makedirs(out, exist_ok=True) + with open(psx, "wb") as f: + f.write(build_exe()) + with open(seeds, "w") as f: + f.write("0x80010000\n") + r = subprocess.run([recompiler, psx, "--seeds", seeds, "--out-dir", out], + capture_output=True, text=True) + if r.returncode != 0: + raise SystemExit(f"recompiler failed:\n{r.stderr or r.stdout}") + chunks = [] + for name in sorted(os.listdir(out)): + if "_full" in name and name.endswith(".c") and "_dispatch" not in name: + with open(os.path.join(out, name), encoding="utf-8") as f: + chunks.append(f.read()) + if not chunks: + raise SystemExit(f"no _full*.c emitted in {out}") + return "\n".join(chunks) + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + default_recomp = os.path.normpath(os.path.join(here, "..", "build", "psxrecomp-game.exe")) + ap = argparse.ArgumentParser() + ap.add_argument("--recompiler", default=default_recomp) + args = ap.parse_args() + if not os.path.isfile(args.recompiler): + raise SystemExit(f"recompiler not found: {args.recompiler} (build it first)") + + with tempfile.TemporaryDirectory() as tmp: + c = gen_c(args.recompiler, tmp) + + fails = [] + m = re.search(r"/\* computed-stride jump into unrolled run 0x80010018 \(rom 0x80010018\), " + r"stride 12, 5 entries \*/", c) + if not m: + fails.append("no computed-stride switch comment for run 0x80010018 / stride 12 / 5 entries") + for k in range(5): + t = 0x80010018 + 12 * k + if not re.search(r"case 0x%08Xu:\s*(?:\n.*?)*?goto block_%08X;" % (t, t), c): + fails.append("missing case 0x%08X -> goto block_%08X" % (t, t)) + if not re.search(r"default:\s*\n(?:.*\n)*?\s*cpu->pc = .*; return; /\* CPS: jr table miss", c): + fails.append("the CPS tail-transfer default is missing from the switch") + # Interior labels must exist mid-block for the gotos to land on. + for k in range(1, 4): + t = 0x80010018 + 12 * k + if ("block_%08X:" % t) not in c: + fails.append("no interior label block_%08X" % t) + + for f in fails: + print("FAIL:", f) + if fails: + return 1 + print("PASS: computed-stride jump emits a switch over every unrolled-run entry.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From b0e923dcd0c5fe90320647dc06b880a7998f4dff Mon Sep 17 00:00:00 2001 From: Kevin <9866117+kerokline@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:27:43 -0400 Subject: [PATCH 54/56] code_generator: resolve self-limited pointer tables with an unchecked index The bounded-jump-table resolver requires the guest's own bounds check (`sltiu/beq`) and index scaling (`sll`), because those are what prove the table's extent. A decoder that keeps its state as a pre-scaled byte offset and dispatches through `lui/ori base; addu base,offset; lw; jr` has neither, so the jr fell back to a bare CPS tail-transfer and every entry it reached that was not already a dispatch key ran on the dirty-RAM interpreter. On Breath of Fire III that is the file decoder's state dispatch at `0x80164AB0` (five entries; the first entry is the word right after the table), the last boot-EXE row a play session still interpreted after the computed-stride resolver. resolve_self_limited_jump_table recovers the extent from the table's own layout instead: words are taken from the base while each is a 4-aligned in-function code address outside every delay slot, and the table ends where its lowest target begins, since a pointer table cannot overlap the code it points at. As with the stride resolver the emitter switches on the actual runtime target with the CPS tail-transfer as default, so the recovered entries only add native cases; an index past the recovered extent still takes the fallback. Wired as the third fallback at both code-generator sites and in the exact-entry discovery walker. On the title's boot EXE it matches exactly one site (five entries); every other shard is byte-identical. The synthetic test gains a second function with this shape. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --- recompiler/include/function_analysis.h | 31 +++++ recompiler/src/code_generator.cpp | 16 +++ recompiler/src/function_analysis.cpp | 126 ++++++++++++++++++ .../test_computed_stride_jump_codegen.py | 32 ++++- 4 files changed, 203 insertions(+), 2 deletions(-) diff --git a/recompiler/include/function_analysis.h b/recompiler/include/function_analysis.h index 4eb5d6b8..b006c712 100644 --- a/recompiler/include/function_analysis.h +++ b/recompiler/include/function_analysis.h @@ -76,6 +76,10 @@ struct ExactJumpTable { // rather than a table in memory: `table_base` is then the first word of // the unrolled run and the targets are table_base + k * stride. uint32_t stride = 0; + // True when the targets came from resolve_self_limited_jump_table: the + // guest checks no bound, so the extent was taken from the table's own + // layout (it ends where its lowest target begins). + bool self_limited = false; }; using ExactAddressMapper = uint32_t (*)(uint32_t, const PS1Executable&); @@ -130,6 +134,33 @@ bool resolve_computed_stride_jump( ExactJumpTable& table, ExactAddressMapper runtime_to_image = nullptr); +// Recognize an in-function pointer table indexed by a value the guest never +// bounds-checks (a stored, pre-scaled state offset rather than a checked +// case number): +// +// lui T, hi ; ori|addiu T, T, lo (table base, an in-function constant) +// addu T, T, I (either operand order; I is the byte offset) +// lw R, off(T) +// [nop] +// jr R +// +// With no guard there is no count to read, so the extent comes from the +// table itself: words are taken from the base while each is a 4-aligned +// in-function code address outside every delay slot, and the table ends +// where its lowest target begins (a pointer table cannot overlap the code +// it points at). As with resolve_computed_stride_jump the caller switches +// on the actual runtime target with the CPS tail-transfer as default, so the +// recovered entries only add native cases; a value past the recovered +// extent still takes the fallback. Requires at least two entries. +bool resolve_self_limited_jump_table( + const PS1Executable& exe, + uint32_t entry, + uint32_t hard_cap, + uint32_t jr_pc, + uint32_t jr_rs, + ExactJumpTable& table, + ExactAddressMapper runtime_to_image = nullptr); + class FunctionAnalyzer { public: explicit FunctionAnalyzer(const PS1Executable& exe); diff --git a/recompiler/src/code_generator.cpp b/recompiler/src/code_generator.cpp index 1a0318e2..24e0e692 100644 --- a/recompiler/src/code_generator.cpp +++ b/recompiler/src/code_generator.cpp @@ -2331,6 +2331,15 @@ std::string CodeGenerator::translate_basic_block( block.exit_instr.address, jr_rs, exact_table, ram_to_rom); } + // Nor a bounded table: an in-function pointer table + // indexed by a stored, unchecked offset (a decoder's + // state dispatch). Extent from the table's own layout. + if (!have_exact_table) { + have_exact_table = resolve_self_limited_jump_table( + exe_, cfg.function_start, cfg.function_end, + block.exit_instr.address, jr_rs, exact_table, + ram_to_rom); + } uint32_t table_base = have_exact_table ? exact_table.table_base : 0u; uint32_t table_count = have_exact_table @@ -2357,6 +2366,10 @@ std::string CodeGenerator::translate_basic_block( ss << config_.indent << fmt::format( "/* computed-stride jump into unrolled run 0x{:08X} (rom 0x{:08X}), stride {}, {} entries */\n", table_base, rom_table_base, exact_table.stride, table_count); + } else if (exact_table.self_limited) { + ss << config_.indent << fmt::format( + "/* self-limited jump table 0x{:08X} (rom 0x{:08X}), {} entries, unchecked index */\n", + table_base, rom_table_base, table_count); } else { ss << config_.indent << fmt::format("/* jump table 0x{:08X} (rom 0x{:08X}), {} entries */\n", table_base, rom_table_base, table_count); @@ -2915,6 +2928,9 @@ void CodeGenerator::scan_jr_tables( blk.exit_instr.address, jr_r, exact_table, ram_to_rom, cfg.producer_lo, cfg.producer_hi) && !resolve_computed_stride_jump( + exe_, cfg.function_start, cfg.function_end, + blk.exit_instr.address, jr_r, exact_table, ram_to_rom) && + !resolve_self_limited_jump_table( exe_, cfg.function_start, cfg.function_end, blk.exit_instr.address, jr_r, exact_table, ram_to_rom)) { continue; diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 658868b0..5b1564db 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -1289,6 +1289,130 @@ bool resolve_computed_stride_jump( return true; } +bool resolve_self_limited_jump_table( + const PS1Executable& exe, + uint32_t entry, + uint32_t hard_cap, + uint32_t jr_pc, + uint32_t jr_rs, + ExactJumpTable& table, + ExactAddressMapper runtime_to_image) { + table = {}; + if (entry >= hard_cap || jr_pc < entry || jr_pc + 8u > hard_cap || + (entry & 3u) != 0u || (hard_cap & 3u) != 0u || + (jr_pc & 3u) != 0u || jr_rs == 0u || jr_rs == 31u) { + return false; + } + auto read = [&](uint32_t pc) { return exe.read_word(pc); }; + auto jr_word = read(jr_pc); + if (!jr_word.has_value() || + *jr_word != ((jr_rs & 0x1Fu) << 21u | 0x08u)) { + return false; + } + auto nearest_def = [&](uint32_t reg, uint32_t use_pc, + uint32_t window) -> std::optional<uint32_t> { + if (reg == 0u) return std::nullopt; + uint32_t pc = use_pc; + for (uint32_t n = 0; n < window && pc >= entry + 4u; ++n) { + pc -= 4u; + auto w = read(pc); + if (!w.has_value()) return std::nullopt; + if (exact_classify_cf(pc, *w).kind != ExactCfKind::Normal) + return std::nullopt; + if (exact_instruction_writes_gpr(*w, reg)) return pc; + } + return std::nullopt; + }; + + // lw R, off(T) — the nearest definition of the jr register; allow the + // canonical single nop between the load and the jr. + auto lw_pc = nearest_def(jr_rs, jr_pc, 2u); + if (!lw_pc.has_value()) return false; + uint32_t lw = *read(*lw_pc); + if (((lw >> 26) & 0x3Fu) != 0x23u || ((lw >> 16) & 0x1Fu) != jr_rs) + return false; + uint32_t t_reg = (lw >> 21) & 0x1Fu; + int32_t lw_off = static_cast<int32_t>(static_cast<int16_t>(lw & 0xFFFFu)); + if (t_reg == 0u) return false; + + // addu T, A, B + auto addu_pc = nearest_def(t_reg, *lw_pc, 6u); + if (!addu_pc.has_value()) return false; + uint32_t au = *read(*addu_pc); + if (((au >> 26) & 0x3Fu) != 0u || (au & 0x3Fu) != 0x21u || + ((au >> 6) & 0x1Fu) != 0u || ((au >> 11) & 0x1Fu) != t_reg) { + return false; + } + uint32_t a = (au >> 21) & 0x1Fu, b = (au >> 16) & 0x1Fu; + if (a == 0u || b == 0u || a == b) return false; + + // One operand is `lui X,hi; ori|addiu X,X,lo` — an in-function constant. + auto constant_of = [&](uint32_t reg, uint32_t use_pc) -> std::optional<uint32_t> { + auto lo_pc = nearest_def(reg, use_pc, 6u); + if (!lo_pc.has_value()) return std::nullopt; + uint32_t lo = *read(*lo_pc); + uint32_t op = (lo >> 26) & 0x3Fu; + uint32_t rs = (lo >> 21) & 0x1Fu, rt = (lo >> 16) & 0x1Fu; + if ((op != 0x0Du && op != 0x09u) || rt != reg || rs != reg) + return std::nullopt; + auto hi_pc = nearest_def(reg, *lo_pc, 6u); + if (!hi_pc.has_value()) return std::nullopt; + uint32_t hi = *read(*hi_pc); + if (((hi >> 26) & 0x3Fu) != 0x0Fu || ((hi >> 16) & 0x1Fu) != reg) + return std::nullopt; + uint32_t v = (hi & 0xFFFFu) << 16; + if (op == 0x0Du) v |= (lo & 0xFFFFu); + else v += static_cast<uint32_t>(static_cast<int32_t>(static_cast<int16_t>(lo & 0xFFFFu))); + return v; + }; + std::optional<uint32_t> base = constant_of(a, *addu_pc); + if (!base.has_value()) base = constant_of(b, *addu_pc); + if (!base.has_value()) return false; + const uint32_t runtime_base = *base + static_cast<uint32_t>(lw_off); + auto mapped = [&](uint32_t addr) { + return runtime_to_image ? runtime_to_image(addr, exe) : addr; + }; + const uint32_t image_base = mapped(runtime_base); + if ((image_base & 3u) != 0u || image_base < entry || image_base >= hard_cap) + return false; + + auto in_delay_slot = [&](uint32_t pc) { + if (pc < entry + 4u) return false; + auto prev = read(pc - 4u); + return prev.has_value() && + exact_classify_cf(pc - 4u, *prev).kind != ExactCfKind::Normal; + }; + + // Walk the table: stop at the first word that is not an in-function code + // address, or when the walk reaches the lowest target seen so far. + ExactJumpTable resolved; + resolved.table_base = runtime_base; + resolved.self_limited = true; + uint32_t lowest_target = hard_cap; + for (uint32_t i = 0; i < 256u; ++i) { + uint32_t slot = image_base + i * 4u; + if (slot >= lowest_target || slot + 4u > hard_cap) break; + auto word = read(slot); + if (!word.has_value()) break; + uint32_t rt_target = *word; + uint32_t it_target = mapped(rt_target); + if ((it_target & 3u) != 0u || it_target < entry || it_target >= hard_cap || + it_target == jr_pc + 4u || in_delay_slot(it_target)) { + break; + } + auto first = read(it_target); + if (!first.has_value() || !exact_is_valid_mips_word(*first)) break; + // A target inside the table itself would be data run as code. + if (it_target >= image_base && it_target <= slot) break; + resolved.targets.emplace_back(rt_target, it_target); + lowest_target = std::min(lowest_target, it_target); + } + if (resolved.targets.size() < 2u) return false; + resolved.table_count = static_cast<uint32_t>(resolved.targets.size()); + table = std::move(resolved); + return true; +} + FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( const std::vector<uint32_t>& entries, const std::vector<std::pair<uint32_t, uint32_t>>& producer_ranges, @@ -1462,6 +1586,8 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( exe_, entry, hard_cap, pc, jr_rs, table, nullptr, producer_lo, producer_hi) || resolve_computed_stride_jump( + exe_, entry, hard_cap, pc, jr_rs, table) || + resolve_self_limited_jump_table( exe_, entry, hard_cap, pc, jr_rs, table)) { bool all_owned = std::all_of( table.targets.begin(), table.targets.end(), diff --git a/recompiler/tests/test_computed_stride_jump_codegen.py b/recompiler/tests/test_computed_stride_jump_codegen.py index 2bfc9a4f..c4ba5a65 100644 --- a/recompiler/tests/test_computed_stride_jump_codegen.py +++ b/recompiler/tests/test_computed_stride_jump_codegen.py @@ -27,6 +27,11 @@ Synthesized function (load 0x80010000): Expected: a `computed-stride jump` switch with cases for every group and the tail, each a `goto block_...`, and the CPS tail-transfer kept as default. +A second function (see build_exe) covers the sibling shape from the same +decoder: an in-function pointer table indexed by a stored, unchecked byte +offset (`lui/ori base; addu; lw; jr`, no sltiu/beq guard, no sll), whose +extent resolve_self_limited_jump_table takes from the table's own layout. + Usage: python test_computed_stride_jump_codegen.py [--recompiler <psxrecomp-game.exe>] """ import argparse @@ -65,6 +70,22 @@ def build_exe(): for k in range(4): body += [0x90CD0000 | k, 0x00000000, 0xA0ED0000 | k] # lbu t5,k(a2); nop; sb t5,k(a3) body += [0x03E00008, 0x00000000] # jr ra; nop + body += [0x00000000] * 4 # pad to 0x80010060 + + # Second function @ 0x80010060: a self-limited pointer table indexed by a + # stored, unchecked byte offset (no sltiu/beq guard, no sll): + # lui t5,0x8001 ; ori t5,t5,0x007C ; addu t5,a0,t5 ; lw t4,0(t5) ; nop ; jr t4 ; nop (0x60..0x78) + # table @ 0x8001007C: 0x80010084, 0x80010090 (ends where its lowest target begins) + # 0x80010084: addiu v0,zero,1 ; j exit ; nop + # 0x80010090: addiu v0,zero,2 ; j exit ; nop + # 0x8001009C exit: jr ra ; nop + exit_pc = 0x8001009C + j_exit = 0x08000000 | ((exit_pc >> 2) & 0x03FFFFFF) + body += [0x3C0D8001, 0x35AD007C, 0x008D6821, 0x8DAC0000, 0x00000000, 0x01800008, 0x00000000] + body += [0x80010084, 0x80010090] + body += [0x24020001, j_exit, 0x00000000] + body += [0x24020002, j_exit, 0x00000000] + body += [0x03E00008, 0x00000000] return make_psxexe(LOAD, w(body)) @@ -76,7 +97,7 @@ def gen_c(recompiler, tmp): with open(psx, "wb") as f: f.write(build_exe()) with open(seeds, "w") as f: - f.write("0x80010000\n") + f.write("0x80010000\n0x80010060\n") r = subprocess.run([recompiler, psx, "--seeds", seeds, "--out-dir", out], capture_output=True, text=True) if r.returncode != 0: @@ -120,11 +141,18 @@ def main(): if ("block_%08X:" % t) not in c: fails.append("no interior label block_%08X" % t) + # Second function: the self-limited table. + if not re.search(r"/\* self-limited jump table 0x8001007C \(rom 0x8001007C\), 2 entries, unchecked index \*/", c): + fails.append("no self-limited table switch comment for table 0x8001007C / 2 entries") + for t in (0x80010084, 0x80010090): + if not re.search(r"case 0x%08Xu:\s*(?:\n.*?)*?goto block_%08X;" % (t, t), c): + fails.append("missing self-limited case 0x%08X -> goto block_%08X" % (t, t)) + for f in fails: print("FAIL:", f) if fails: return 1 - print("PASS: computed-stride jump emits a switch over every unrolled-run entry.") + print("PASS: computed-stride jump and self-limited table each emit a switch over every entry.") return 0 From 5803def02a240956adb9805108321f5b6d93e77f Mon Sep 17 00:00:00 2001 From: Kevin Kline <9866117+kerokline@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:39:53 -0400 Subject: [PATCH 55/56] compile_overlays: require CFG proof for a frameless dispatch walk root Overlay bands are shared address windows, so every observed interpreted PC is demanded for every occupant of the band. classify_overlay_seeds promotes such a demand to a linear walk root whenever _callable_legacy_seed accepts it, and that predicate accepts three unequal grades of evidence: the image entry, an `addiu sp, sp, -imm` prologue, or a `jr $ra` two words back. The third grade fires exactly where it is least trustworthy. The first word after a function's return is also the first word of whatever the linker laid down next, and in an overlay image that is routinely a pointer table, a packed record array or zero fill. The walk then starts at the table head, never reaches a return, and runs to the image end; the generated-C audit reports a column of UNSUPPORTED_INSTRUCTION and the shard fails. Require the bounded CFG probe the discovery roots already use (plausible_callable_target: reachable return, no fallthrough past the probe) before a prologue-less dispatch entry may be a walk root. A failure demotes it to DISPATCH_INTERIOR, which keeps the dispatch evidence and the isolated fragment demand and only declines to start a walk there; the image entry and the prologue keep their own evidence and are not probed. Measured on Breath of Fire III (SLPS-00990), 406 overlay captures: dispatch entries already accepted as walk roots 5,339 with a prologue 2,066 unchanged on the preceding `jr $ra` alone 3,273 rejected by the probe 41 DISPATCH_ENTRY roots 1,141 -> 1,100 (-41) DISPATCH_INTERIOR 81,474 -> 80,851 (-623, interiors of the fabricated walks) STATIC_DISCOVERY_ROOT 10,741 -> 10,741 STATIC_DISPATCH_ENTRY 194 -> 194 FUNCTION_POINTER_TARGET 20 -> 20 isolated fragment demands 105,891 -> 105,891 (nothing lost) 41 of 406 captures change at all. All 41 rejected addresses are data by inspection: in-image code-pointer tables, packed 16-bit record arrays, signed delta pairs, zero fill. 27 of them are precisely the 27 shards whose all-bands compile failed the audit -- one bad root per failing shard. The other 14 are the same defect in shards whose data happened to decode all the way to the image end without meeting an unsupported opcode. New CTest: frameless_dispatch_root_cfg_proof. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- recompiler/CMakeLists.txt | 8 ++ tools/compile_overlays.py | 39 +++++- .../test_frameless_dispatch_root_cfg_proof.py | 124 ++++++++++++++++++ 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 tools/tests/test_frameless_dispatch_root_cfg_proof.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 1c41506b..5c0b8a14 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -630,6 +630,14 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_compile_overlays_static_tail.py --recompiler $<TARGET_FILE:psxrecomp-game> WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + # The word after a `jr $ra` is as often a pointer table as a frameless + # function; a dispatch entry there must prove a CFG before it may be a + # walk root, or the walk runs through the table to the image end. + add_test( + NAME frameless_dispatch_root_cfg_proof + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_frameless_dispatch_root_cfg_proof.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) add_test( NAME fmv_cycle_batch_codegen COMMAND ${Python3_EXECUTABLE} diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 8ce786e8..2230de2d 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -1269,6 +1269,35 @@ def plausible_callable_target(data: bytes, load_addr: int, size: int, return saw_return and not bad and not work and len(visited) < 2048 +def _frameless_dispatch_root_proven(data: bytes, load_addr: int, size: int, + addr: int, producer_hi: int) -> bool: + """CFG proof for a dispatch entry admitted only by the word before it. + + `_callable_legacy_seed` accepts an address on three different grades of + evidence: it is the image entry, it opens a stack frame, or the word two + slots back is `jr $ra`. The third is the weak one, and it is weak exactly + where it fires most: the first word *after* a function's return is also + the first word of whatever the linker laid down next, which in these + images is routinely a pointer table, a packed record array or zero fill. + A dispatch PC there is real -- a sibling occupant of the shared band has + code at that address -- but for *this* image it is data, and promoting it + to a walk root sends the linear walk through the table to the image end. + + So require the same bounded CFG probe the discovery roots already use + when the classic prologue is absent. Failing it does not discard the + entry: the caller demotes it to DISPATCH_INTERIOR, which keeps the + dispatch evidence and the isolated-fragment demand and only declines to + start a walk there. + """ + word = _word_at(data, load_addr, addr) + prev = _word_at(data, load_addr, addr - 4) + if addr == load_addr: + return True + if _is_addiu_sp_neg(word) and not _is_control_flow(prev): + return True + return plausible_callable_target(data, load_addr, size, addr, producer_hi) + + def _walk_overlay_function(data: bytes, load_addr: int, size: int, entry: int, hard_cap: int, producer_ranges=(), @@ -1642,14 +1671,22 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, # Invalid words stay excluded. Call-edge-proven reasons # (DIRECT_JAL_TARGET, FUNCTION_POINTER_TARGET, TOML_DECLARED_ENTRY) are # exempt — they carry their own proof. + # A dispatch entry that has no prologue and is callable only because a + # `jr $ra` precedes it is the head of whatever follows the last + # function, data included, so it needs the bounded CFG probe before it + # may be a walk root (_frameless_dispatch_root_proven). if reason in ('DISPATCH_ENTRY', 'STATIC_DISPATCH_ENTRY'): if impossible_entry_start(addr): excluded[addr] = 'UNKNOWN' return if addr + 4 <= fragment_hi: dispatch_fragment_demands.add(addr) + producer = capture_producer_for(addr) if (addr in jump_table_targets or - not _callable_legacy_seed(data, load_addr, addr)): + not _callable_legacy_seed(data, load_addr, addr) or + not _frameless_dispatch_root_proven( + data, load_addr, size, addr, + producer[1] if producer else hi)): included.setdefault(addr, 'DISPATCH_INTERIOR') return elif (reason not in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', diff --git a/tools/tests/test_frameless_dispatch_root_cfg_proof.py b/tools/tests/test_frameless_dispatch_root_cfg_proof.py new file mode 100644 index 00000000..d82434a7 --- /dev/null +++ b/tools/tests/test_frameless_dispatch_root_cfg_proof.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""A prologue-less dispatch entry needs CFG proof before it becomes a walk root. + +Overlay bands are shared address windows, so every observed interpreted PC is +demanded for every occupant of the band. `_callable_legacy_seed` then admits a +demanded address as a callable walk root on any of three grades of evidence: +it is the image entry, it opens a stack frame, or the word two slots back is +`jr $ra`. That third grade is the weak one, and it fires precisely where it is +least trustworthy -- the first word after a function's return is also the first +word of whatever the linker placed next, which in these images is routinely a +pointer table, a packed record array or zero fill. The walk then runs from the +table head to the image end and the generated-C audit reports a column of +UNSUPPORTED_INSTRUCTION. + +Measured on Breath of Fire III (SLPS-00990), 406 overlay captures, 5,339 +dispatch entries already accepted as walk roots: 2,066 have a prologue, 3,273 +ride on the preceding `jr $ra` alone, and the bounded CFG probe rejects 41 of +those 3,273. All 41 are data by inspection (in-image pointer tables, packed +16-bit record arrays, signed delta pairs, zero fill), and 27 of them are +exactly the 27 shards whose all-bands compile failed the audit -- one bad root +per failing shard. The other 14 are the same bug in shards whose data happened +to decode all the way to the image end. + +Demotion is not a discard: DISPATCH_INTERIOR keeps the dispatch evidence and +the isolated-fragment demand, and only declines to start a linear walk there. +""" + +import struct +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +import compile_overlays # noqa: E402 + +LOAD = 0x801EEC00 + +NOP = 0x00000000 +JR_RA = 0x03E00008 +ADDIU_SP_NEG = 0x27BDFFE0 # addiu sp, sp, -0x20 +ADDIU_SP_POS = 0x27BD0020 # addiu sp, sp, 0x20 +ORI_V0_ZERO = 0x34020001 # ori v0, zero, 1 + + +def image(words): + return b"".join(struct.pack("<I", w) for w in words) + + +class FramelessDispatchRootTests(unittest.TestCase): + + def proven(self, data, addr): + return compile_overlays._frameless_dispatch_root_proven( + data, LOAD, len(data), addr, LOAD + len(data)) + + def test_pointer_table_after_a_return_is_not_a_walk_root(self): + """BATL_OVR 0x801EF5F8: five in-image code pointers, then a word table. + + Every word decodes as some R3000 instruction, so opcode validity alone + says nothing; the walk simply never reaches a return. + """ + data = image([ + ADDIU_SP_NEG, NOP, JR_RA, ADDIU_SP_POS, # a real leaf function + 0x801EEC40, 0x801EEC7C, 0x801EEEB8, # the pointer table + 0x801EEF10, 0x801EF0A8, + 0x000000A0, 0x00000084, 0x00000070, # the word table + ]) + table = LOAD + 4 * 4 + self.assertTrue(compile_overlays._callable_legacy_seed(data, LOAD, table)) + self.assertFalse(self.proven(data, table)) + + def test_packed_record_table_after_a_return_is_not_a_walk_root(self): + """AREA050 0x801F2DD4: 12-byte script records, no pointers at all.""" + data = image([ + ADDIU_SP_NEG, NOP, JR_RA, NOP, + 0x00322F08, 0x93800500, 0x00070201, + 0x00329206, 0x2F800900, 0x00030401, + ]) + table = LOAD + 4 * 4 + self.assertTrue(compile_overlays._callable_legacy_seed(data, LOAD, table)) + self.assertFalse(self.proven(data, table)) + + def test_zero_fill_after_a_return_is_not_a_walk_root(self): + data = image([ADDIU_SP_NEG, NOP, JR_RA, NOP] + [0] * 8) + self.assertFalse(self.proven(data, LOAD + 4 * 4)) + + def test_a_frameless_leaf_after_a_return_stays_a_walk_root(self): + """The probe must not cost real frameless exports their root. + + 3,273 of BoF3's accepted dispatch roots are admitted by the preceding + `jr $ra` alone; only 41 fail this probe. + """ + data = image([ + ADDIU_SP_NEG, NOP, JR_RA, NOP, + ORI_V0_ZERO, JR_RA, NOP, NOP, + ]) + frameless = LOAD + 4 * 4 + self.assertTrue(compile_overlays._callable_legacy_seed(data, LOAD, frameless)) + self.assertTrue(self.proven(data, frameless)) + + def test_a_prologue_needs_no_probe(self): + """A stack frame is its own evidence, even without a reachable return. + + The walk from a prologue is the recompiler's ordinary business; this + gate exists only for the grade of evidence that has none. + """ + data = image([ + ADDIU_SP_NEG, NOP, JR_RA, NOP, + ADDIU_SP_NEG, 0xAFBF0018, 0x0C000000, NOP, + ]) + entry = LOAD + 4 * 4 + self.assertFalse( + compile_overlays.plausible_callable_target( + data, LOAD, len(data), entry, LOAD + len(data))) + self.assertTrue(self.proven(data, entry)) + + def test_the_image_entry_needs_no_probe(self): + data = image([0x00000070, 0x0000007A, 0x0000009C, 0x00000094]) + self.assertTrue(self.proven(data, LOAD)) + + +if __name__ == "__main__": + unittest.main() From fa435a4a0c1c1a668a4b508ee2841696d6fdc4a6 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 19 Sep 2026 16:19:23 -0700 Subject: [PATCH 56/56] render: add uncapped adaptive viewport and stable world classification --- lib/recomp-net | 2 +- runtime/codegen_hash_sources.cmake | 2 + runtime/include/gpu.h | 3 + runtime/include/mod_plugins.h | 14 ++ runtime/include/overlay_api.h | 6 +- runtime/include/ws_backdrop_detect.h | 198 ++++++++++++++++++++++----- runtime/include/ws_backdrop_margin.h | 28 ++++ runtime/include/ws_primitive_roles.h | 47 +++++++ runtime/include/ws_scene_latch.h | 28 ++++ runtime/src/debug_server.c | 1 + runtime/src/gpu.c | 90 ++++++++++-- runtime/src/gpu_gl_renderer.c | 20 ++- runtime/src/main.cpp | 17 ++- runtime/src/mod_runtime.cpp | 15 ++ 14 files changed, 414 insertions(+), 57 deletions(-) create mode 100644 runtime/include/ws_backdrop_margin.h create mode 100644 runtime/include/ws_primitive_roles.h create mode 100644 runtime/include/ws_scene_latch.h diff --git a/lib/recomp-net b/lib/recomp-net index 268e74fe..02c29e6a 160000 --- a/lib/recomp-net +++ b/lib/recomp-net @@ -1 +1 @@ -Subproject commit 268e74fe718b38fe38643c358588bbc1e0f0af70 +Subproject commit 02c29e6ac7a5d2eb604cf497f8613f6a36e47cc1 diff --git a/runtime/codegen_hash_sources.cmake b/runtime/codegen_hash_sources.cmake index 96124d2b..cc213948 100644 --- a/runtime/codegen_hash_sources.cmake +++ b/runtime/codegen_hash_sources.cmake @@ -33,6 +33,8 @@ set(PSXRECOMP_CODEGEN_HASH_SRCS ${PSXRECOMP_CODEGEN_HASH_ROOT}/recompiler/src/function_discovery.cpp ${PSXRECOMP_CODEGEN_HASH_ROOT}/recompiler/src/function_discovery.h ${PSXRECOMP_CODEGEN_HASH_ROOT}/recompiler/include/gte_register_classification.h + # Shared instruction detector changes alter emitted overlay substitutions. + ${PSXRECOMP_CODEGEN_HASH_ROOT}/runtime/include/ws_backdrop_detect.h # --- Shard COMPILE surface ------------------------------------------------- # The cg tag must invalidate on anything a shard compiles/links against, not diff --git a/runtime/include/gpu.h b/runtime/include/gpu.h index f5d8c449..74225443 100644 --- a/runtime/include/gpu.h +++ b/runtime/include/gpu.h @@ -336,6 +336,9 @@ void gpu_ws_set_gameplay_state_gate(uint32_t addr, * outer-third screen-space HUD primitives out to the true wide-frame corners * (they otherwise sit inset by the reveal). Runtime-only. Off by default. */ void gpu_ws_set_nw_hud_corners(int on); +void gpu_ws_tag_hud_primitive(uint32_t primitive, int edge); +void gpu_ws_tag_world_primitive(uint32_t primitive, int is_world); +void gpu_ws_set_adaptive_backdrop_preload(int enabled); /* Targeted alternative for sprite-heavy 2D games: corner-anchor only primitives * whose ordering-table packet lives in the configured half-open RAM range. */ void gpu_ws_set_nw_left_hud_packet_range(uint32_t lo, uint32_t hi); diff --git a/runtime/include/mod_plugins.h b/runtime/include/mod_plugins.h index d2299b30..227948e8 100644 --- a/runtime/include/mod_plugins.h +++ b/runtime/include/mod_plugins.h @@ -58,6 +58,18 @@ uint32_t psx_mod_alloc_gpu_dma_memory(uint32_t size, uint32_t alignment); /* Current per-side widescreen reveal in native game pixels (zero at 4:3). */ int32_t psx_mod_widescreen_x_margin(void); +/* Mark a guest GPU packet (P_TAG address) as persistent screen-space HUD. + * edge = -1 left, +1 right, 0 clears a reused packet's tag. The native-wide + * compositor translates it by the live reveal, excluding culling guards. + * Guest coordinates, world sprites, and native 4:3 remain unchanged. */ +void psx_mod_tag_hud_primitive(uint32_t primitive, int edge); +/* Exclude a known world packet from screen-space backdrop stretching, even + * if it sorts before the first shaded polygon. Zero clears a recycled tag. */ +void psx_mod_tag_world_primitive(uint32_t primitive, int is_world); +/* Enable aspect-derived column selection for a title that opted into the + * auto_backdrop detector. No effect on titles that did not opt in. */ +void psx_mod_set_adaptive_backdrop_preload(int enabled); + /* * Read the committed value of one of this package's declared options, as the * player left it in the launcher (or the manifest default when untouched). @@ -98,6 +110,8 @@ int psx_mod_set_fixed_display_aspect(uint32_t numerator, uint32_t denominator); /* * Request resize-driven widescreen, capped at the supplied maximum aspect. + * Pass (0, 0) for Fit to window with no upper aspect limit. Both modes retain + * the native 4:3 minimum; a single zero is invalid. * The current fixed aspect continues to shape the initial game window, so a * plugin may select that first with psx_mod_set_fixed_display_aspect(). */ diff --git a/runtime/include/overlay_api.h b/runtime/include/overlay_api.h index 2ceda151..7959c990 100644 --- a/runtime/include/overlay_api.h +++ b/runtime/include/overlay_api.h @@ -156,7 +156,11 @@ * 10: + PGXP dataflow-shadowing PGXP_*() macro emission at loads/stores/ * COP2 transfers/precision-carrying ALU ops (pgxp_hooks.h; no-ops in * base-flavour objects, real hooks under -DPSX_PGXP=1). */ -#define PSX_OVERLAY_CODEGEN_VER 10 +/* v11: backdrop windows require independent, intact quotient consumers; + * reject layer offsets previously mistaken for window widths. */ +/* v12: camera provenance through copied/biased registers and two independent + * biased bounds, for multiple terrain strips in one finite column table. */ +#define PSX_OVERLAY_CODEGEN_VER 12 /* Auto codegen hash (generated by hash_codegen.cmake from the recompiler codegen * sources). Folded into the cache PATH next to cg<N> (gcc/<arch-abi>/cg<N>_<hash>/) diff --git a/runtime/include/ws_backdrop_detect.h b/runtime/include/ws_backdrop_detect.h index 09ecd4f1..98cf0c2e 100644 --- a/runtime/include/ws_backdrop_detect.h +++ b/runtime/include/ws_backdrop_detect.h @@ -12,14 +12,14 @@ * WHAT IT FINDS * ------------- * PSX scrollers generate a far-background tile row one CAMERA-WINDOWED column - * range per frame. The generator divides the camera world-X by ~96 to a column + * range per frame. The generator divides the camera world-X by 80 to a column * index and emits a small window [START, END] of columns around it: * - * lui M,0x6666 ; ori M,M,0x6667 ; M = 0x66666667 (/96 reciprocal) + * lui M,0x6666 ; ori M,M,0x6667 ; M = 0x66666667 (/80 reciprocal) * lh/lhu D,0x176(base) ; D = camera world-X (offset 0x176) * [addiu/addu D, D, Koff] ; optional per-layer parallax bias * mult D, M - * mfhi H ; sra Q,H,5 ; subu/addu Q,Q,sign ; Q = camX / 96 (the quotient) + * mfhi H ; sra Q,H,5 ; subu/addu Q,Q,sign ; Q = camX / 80 (the quotient) * move rStart, Q ; START finalize (offset 0) * addiu rEnd, Q, N ; END delta (offset N) * <low clamp on START to a floor> <high clamp on END to the finite extent> @@ -36,14 +36,15 @@ * * THE TWO REWRITE SITES (per window) * ---------------------------------- - * The quotient is consumed by exactly two instructions: a `move rX, Q` (offset - * 0) and an `addiu rY, Q, N` (offset N). Which is START and which is END is NOT + * Two independent affine consumers finalize bounds: `move rX,Q` (offset 0) + * or `addiu rX,Q,bias`. Both may be biased when a table packs multiple strips. + * Which is START and which is END is NOT * fixed by move-vs-addiu — it is decided by the generator's clamp tail and is * captured generically by OFFSET ORDERING: the smaller signed offset is the * START (left) bound, the larger is the END (right) bound. Verified on Tomba's * village (FUN_80116a28: a2=START/a3=END, +8) and flower-field (s2=START/s1=END; * one window has the addiu at offset -18 acting as START with the move as END). - * The window width in columns is |N| (the addiu offset magnitude) — recorded so + * The window width in columns is the DIFFERENCE between offsets — recorded so * the runtime sizes the widen margin to the reveal. * * The actual value substitution is done at runtime by psx_ws_backdrop_value() @@ -71,6 +72,97 @@ typedef struct { int window_cols; /* window width in columns (|addiu offset|), drives the widen margin */ } WsBackdropSite; +/* Backward def/use proof for the dividend. Some strips reuse a camera value + * loaded for a preceding strip, or keep &scratchpad.camera_x in a saved reg. + * Follow copies/biases, not just a matching load somewhere in the lookback. */ +static inline int psx_ws_bd_written_reg(uint32_t w) { + unsigned op = w >> 26, fn = w & 63u; + if (!op) { + if (fn <= 7u || fn == 9u || fn == 0x10u || fn == 0x12u || + (fn >= 0x20u && fn <= 0x2Bu)) return (int)((w >> 11) & 31u); + } else if ((op >= 8u && op <= 15u) || (op >= 0x20u && op <= 0x26u) || + ((op == 0x10u || op == 0x12u) && ((w >> 21) & 31u) < 3u)) { + return (int)((w >> 16) & 31u); + } else if (op == 3u) return 31; + return -1; +} + +/* An unconditional jump before a block means the lexically preceding arm + * does not reach it. Follow a unique local conditional predecessor (including + * its delay slot), never borrow definitions from the skipped arm. */ +static inline int psx_ws_bd_predecessor(const uint32_t* words, int boundary, int lo) { + int found = -1; + for (int b = boundary - 3; b >= lo; --b) { + unsigned op = words[b] >> 26; + if ((op == 1u || (op >= 4u && op <= 7u)) && + b + 1 + (int)(int16_t)words[b] == boundary) { + if (found >= 0) return -1; + found = b; + } + } + return found; +} + +static inline int psx_ws_bd_constant(const uint32_t* words, int before, int lo, + unsigned reg, unsigned depth, uint32_t* value) { + if (!reg) { *value = 0; return 1; } + if (!depth) return 0; + for (int b = before - 1; b >= lo; --b) { + if (b > lo && (words[b - 1] >> 26) == 2u) { + int pred = psx_ws_bd_predecessor(words, b + 1, lo); + return pred >= 0 && psx_ws_bd_constant(words, pred + 2, lo, + reg, depth - 1, value); + } + uint32_t w = words[b], op = w >> 26, fn = w & 63u; + unsigned rs = (w >> 21) & 31u, rt = (w >> 16) & 31u; + /* A call may clobber every caller-saved register. */ + if ((op == 3u || (!op && fn == 9u)) && (reg < 16u || reg > 23u)) return 0; + if (psx_ws_bd_written_reg(w) != (int)reg) continue; + if (op == 15u) { *value = w << 16; return 1; } + uint32_t source; + if ((op == 9u || op == 13u) && + psx_ws_bd_constant(words, b, lo, rs, depth - 1, &source)) { + *value = op == 13u ? source | (w & 0xFFFFu) + : source + (uint32_t)(int32_t)(int16_t)w; + return 1; + } + if (!op && (fn == 0x21u || fn == 0x25u) && (!rs || !rt)) + return psx_ws_bd_constant(words, b, lo, rs ? rs : rt, depth - 1, value); + return 0; + } + return 0; +} + +static inline int psx_ws_bd_camera(const uint32_t* words, int before, int lo, + unsigned reg, unsigned depth) { + if (!reg || !depth) return 0; + for (int b = before - 1; b >= lo; --b) { + if (b > lo && (words[b - 1] >> 26) == 2u) { + int pred = psx_ws_bd_predecessor(words, b + 1, lo); + return pred >= 0 && psx_ws_bd_camera(words, pred + 2, lo, reg, depth - 1); + } + uint32_t w = words[b], op = w >> 26, fn = w & 63u; + unsigned rs = (w >> 21) & 31u, rt = (w >> 16) & 31u; + if ((op == 3u || (!op && fn == 9u)) && (reg < 16u || reg > 23u)) return 0; + if (psx_ws_bd_written_reg(w) != (int)reg) continue; + if (op == 0x21u || op == 0x25u) { + uint32_t base; + return psx_ws_bd_constant(words, b, lo, rs, 4, &base) && + base + (uint32_t)(int32_t)(int16_t)w == 0x1F800176u; + } + if (op == 9u || op == 8u) + return psx_ws_bd_camera(words, b, lo, rs, depth - 1); + if (!op && (fn == 0x21u || fn == 0x25u || fn == 0x23u)) { + /* OR is only a copy. SUBU permits camera - bias, never bias - camera. */ + if (fn == 0x25u && rs && rt) return 0; + return psx_ws_bd_camera(words, b, lo, rs, depth - 1) || + (fn != 0x23u && psx_ws_bd_camera(words, b, lo, rt, depth - 1)); + } + return 0; /* nearest definition is not a proven camera-derived value */ + } + return 0; +} + /* Scan `n` instruction words starting at guest address `base_pc` for backdrop * column-window generators and record the START/END rewrite sites. Returns the * number of sites written to `out` (capped at `max_sites`). Pure: the result @@ -97,19 +189,10 @@ static inline int psx_ws_find_backdrop_windows(const uint32_t *words, int n, uint32_t D = swap ? op_rs : op_rt; /* dividend candidate */ if (M == D) continue; - int have_lui = 0, have_ori = 0, have_camx = 0; - int blo = i - 32; if (blo < 0) blo = 0; - for (int b = i - 1; b >= blo; b--) { - uint32_t bw = words[b]; - uint32_t bop = bw >> 26; - uint32_t brs = (bw >> 21) & 31u; - uint32_t brt = (bw >> 16) & 31u; - uint32_t bimm = bw & 0xFFFFu; - if (bop == 0x0Fu && brt == M && bimm == 0x6666u) have_lui = 1; /* lui M,0x6666 */ - else if (bop == 0x0Du && brt == M && brs == M && bimm == 0x6667u) have_ori = 1; /* ori M,M,0x6667 */ - else if ((bop == 0x21u || bop == 0x25u) && brt == D && bimm == 0x176u) have_camx = 1; /* lh/lhu D,0x176 */ - } - if (!have_lui || !have_ori || !have_camx) continue; + int blo = i - 48; if (blo < 0) blo = 0; + uint32_t magic; + if (!psx_ws_bd_constant(words, i, blo, M, 4, &magic) || + magic != 0x66666667u || !psx_ws_bd_camera(words, i, blo, D, 8)) continue; /* Forward divide tail: mfhi H -> sra Q,H,5. Bail on a control * transfer before the tail (the divide is straight-line). */ @@ -132,12 +215,16 @@ static inline int psx_ws_find_backdrop_windows(const uint32_t *words, int n, } if (quot < 0) continue; - /* Forward: optional sign correction (addu/subu chaining the - * quotient) + the two window consumers (a move and an addiu reading - * the quotient, or the addiu reading the move's START register). */ + /* Both bounds must independently read the intact quotient. A + * dependent addiu can instead be a layer-table bias (Tomba's +28 + * after a clamp), NOT a window end. Independently shifting both + * results is also wrong if one consumes the already-shifted other + * bound. Leave those ambiguous shapes unchanged. */ int cur = quot, move_dest = -1; uint32_t move_pc = 0, addiu_pc = 0; - int have_move = 0, have_addiu = 0, addiu_n = 0; + /* Historical names: move is the first affine bound, addiu the + * second. Either may now have a nonzero table-segment bias. */ + int have_move = 0, have_addiu = 0, move_n = 0, addiu_n = 0; int fend = jdiv + 1 + 8; if (fend > n) fend = n; for (int f = jdiv + 1; f < fend && !(have_move && have_addiu); f++) { uint32_t fw = words[f]; @@ -149,37 +236,80 @@ static inline int psx_ws_find_backdrop_windows(const uint32_t *words, int n, int fsimm = (int)(int16_t)(uint16_t)(fw & 0xFFFFu); uint32_t pc = base_pc + (uint32_t)f * 4u; + /* A jump's delay slot still executes in this block. Never + * scan beyond it into an unrelated branch leg. Conditional + * clamps are not part of the independent-bound pattern. */ + if (fop == 0x02u) { + if (fend > f + 2) fend = f + 2; + continue; + } + if (fop == 0x03u || (fop >= 0x04u && fop <= 0x07u) || + fop == 0x01u || (fop == 0u && (ffn == 0x08u || ffn == 0x09u))) break; + if (fop == 0u && (ffn == 0x21u || ffn == 0x25u)) { /* addu/or */ int src = -1; if (frt == 0u && frs != 0u) src = (int)frs; /* move rD, rS (rT == $0) */ else if (frs == 0u && frt != 0u) src = (int)frt; /* move rD, rT (rS == $0) */ if (src == cur) { - if (!have_move) { move_pc = pc; move_dest = (int)frd; have_move = 1; } + if ((int)frd == cur) break; + /* Some rows first clamp the signed quotient to zero + * and add a table-segment bias. Recognize the COMPLETE + * diamond, then match the real bounds after its join: + * move T,Q; sll Q,Q,16; bgez Q,+3; addiu Q,T,bias; + * move T,zero; addiu Q,T,bias; <bounds from Q>. + * Neither the temporary move nor bias is a bound. */ + if (!have_move && !have_addiu && f + 6 < n && + words[f + 1] == ((uint32_t)cur << 16 | (uint32_t)cur << 11 | 16u << 6) && + words[f + 2] == (0x04010003u | (uint32_t)cur << 21) && + (words[f + 3] & 0xFFFF0000u) == + (0x24000000u | frd << 21 | (uint32_t)cur << 16) && + words[f + 4] == (frd << 11 | 0x21u) && + words[f + 5] == words[f + 3]) { + f += 5; + fend = f + 1 + 8; + if (fend > n) fend = n; + continue; + } + if (!have_move) { + move_pc = pc; move_dest = (int)frd; move_n = 0; have_move = 1; + } else if ((int)frd != move_dest) { + addiu_pc = pc; addiu_n = 0; have_addiu = 1; + } continue; } if (ffn == 0x21u && frs != 0u && frt != 0u && ((int)frs == cur || (int)frt == cur)) { /* addu sign correction */ + if (have_move || have_addiu) break; cur = (int)frd; continue; } } if (fop == 0u && ffn == 0x23u && ((int)frs == cur || (int)frt == cur)) { /* subu sign correction */ + if (have_move || have_addiu) break; cur = (int)frd; continue; } - if ((fop == 0x09u || fop == 0x08u) && /* addiu/addi rD, (Q|START), N */ - ((int)frs == cur || (have_move && (int)frs == move_dest))) { - if (!have_addiu) { addiu_pc = pc; addiu_n = fsimm; have_addiu = 1; } + if ((fop == 0x09u || fop == 0x08u) && /* addiu/addi rD, Q, N */ + (int)frs == cur && (!have_move || (int)frt != move_dest)) { + if (!have_move) { + if ((int)frt == cur) break; /* quotient bias, not a bound */ + move_pc = pc; move_dest = (int)frt; move_n = fsimm; have_move = 1; + } else if (!have_addiu) { + addiu_pc = pc; addiu_n = fsimm; have_addiu = 1; + } continue; } + /* An unrecognized overwrite invalidates the tracked value. */ + if ((fop == 0u && ((int)frd == cur || (int)frd == move_dest)) || + ((fop >= 0x08u && fop <= 0x0Fu) && + ((int)frt == cur || (int)frt == move_dest))) break; } - if (!have_move || !have_addiu || addiu_n == 0) continue; + if (!have_move || !have_addiu || addiu_n == move_n) continue; - /* Role by offset: smaller signed offset = START (left bound), larger - * = END (right bound). The move's offset is 0; the addiu's is - * addiu_n. Window width = |addiu_n|. */ - int move_kind = (addiu_n > 0) ? WS_BD_START : WS_BD_END; - int addiu_kind = (addiu_n > 0) ? WS_BD_END : WS_BD_START; - int wcols = (addiu_n < 0) ? -addiu_n : addiu_n; + /* Role by relative offset, not the absolute table-segment bias. */ + int delta = addiu_n - move_n; + int move_kind = (delta > 0) ? WS_BD_START : WS_BD_END; + int addiu_kind = (delta > 0) ? WS_BD_END : WS_BD_START; + int wcols = delta < 0 ? -delta : delta; for (int s = 0; s < 2; s++) { uint32_t spc = s ? addiu_pc : move_pc; diff --git a/runtime/include/ws_backdrop_margin.h b/runtime/include/ws_backdrop_margin.h new file mode 100644 index 00000000..fa52eca0 --- /dev/null +++ b/runtime/include/ws_backdrop_margin.h @@ -0,0 +1,28 @@ +#ifndef PSX_WS_BACKDROP_MARGIN_H +#define PSX_WS_BACKDROP_MARGIN_H +#include <stdint.h> + +/* Round outward: even a partial newly visible column must be submitted. The + * caller includes its screen-space guard in reveal_pixels. The guest's own + * row clamps still enforce finite level bounds. */ +static inline int psx_ws_backdrop_columns(int window_columns, int reveal_pixels, + int native_width) { + if (window_columns <= 0 || reveal_pixels <= 0 || native_width <= 0) return 0; + int64_t columns = ((int64_t)window_columns * reveal_pixels + native_width - 1) + / native_width; + /* The detector's consumers narrow to signed 16-bit indices. More than + * 0x3fff extra columns cannot reveal anything further in byte-count rows. */ + return columns > 0x3fff ? 0x3fff : (int)columns; +} + +/* Some segmented rows clamp START before their final table bias. Widening + * that final bound must not bypass index zero or wrap the signed-16-bit END + * before the guest's finite-table upper clamp executes. */ +static inline uint32_t psx_ws_backdrop_bound(uint32_t orig, int is_end, int margin) { + int64_t value = (int32_t)orig; + value += is_end ? margin : -margin; + if (value < 0) value = 0; + if (value > 0x7fff) value = 0x7fff; + return (uint32_t)value; +} +#endif diff --git a/runtime/include/ws_primitive_roles.h b/runtime/include/ws_primitive_roles.h new file mode 100644 index 00000000..7fcac2df --- /dev/null +++ b/runtime/include/ws_primitive_roles.h @@ -0,0 +1,47 @@ +#ifndef PSX_WS_PRIMITIVE_ROLES_H +#define PSX_WS_PRIMITIVE_ROLES_H +#include <stdint.h> +#include <stddef.h> + +/* Title-supplied packet identity, independent of draw order and sprite tags. + * Keys name PsyQ P_TAG headers; GPU lookup names the following command word. */ +#define WS_ROLE_BUCKETS 4096u +typedef struct { + uint32_t key, stamp; + int8_t hud_edge; + uint8_t world; +} WsPrimitiveRole; + +static inline WsPrimitiveRole* ws_role_write(WsPrimitiveRole* tags, + uint32_t primitive, uint32_t now) { + uint32_t key = primitive & 0x1FFFFCu; + if (!key) return NULL; + uint32_t slot = (key >> 2) & (WS_ROLE_BUCKETS - 1u), victim = slot; + for (unsigned i = 0; i < 4; ++i) { + uint32_t j = (slot + i) & (WS_ROLE_BUCKETS - 1u); + if (tags[j].key == key || !tags[j].key) { victim = j; break; } + if (now - tags[j].stamp > 2u) victim = j; + } + WsPrimitiveRole* tag = &tags[victim]; + if (tag->key != key || tag->stamp != now) { + tag->hud_edge = 0; + tag->world = 0; + } + tag->key = key; + tag->stamp = now; + return tag; +} + +static inline const WsPrimitiveRole* ws_role_read(const WsPrimitiveRole* tags, + uint32_t source, uint32_t now) { + if (source == 0xFFFFFFFFu) return NULL; + uint32_t key = (source - 4u) & 0x1FFFFCu; + if (!key) return NULL; + uint32_t slot = (key >> 2) & (WS_ROLE_BUCKETS - 1u); + for (unsigned i = 0; i < 4; ++i) { + const WsPrimitiveRole* tag = &tags[(slot + i) & (WS_ROLE_BUCKETS - 1u)]; + if (tag->key == key && now - tag->stamp <= 2u) return tag; + } + return NULL; +} +#endif diff --git a/runtime/include/ws_scene_latch.h b/runtime/include/ws_scene_latch.h new file mode 100644 index 00000000..67f0d41e --- /dev/null +++ b/runtime/include/ws_scene_latch.h @@ -0,0 +1,28 @@ +#ifndef PSX_WS_SCENE_LATCH_H +#define PSX_WS_SCENE_LATCH_H +#include <stdint.h> + +/* Overhang proves there is revealable world content. Projection activity can + * maintain that proof, but MUST NOT establish it: 2D room jingles also use GTE. + * The CPU/GTE can build a frame across multiple vblanks before submitting any + * terrain packets, so gaps in GP0 overhang are not evidence of a scene change. */ +typedef struct { + uint32_t last_frame, last_activity; + int initialized, confirmed; +} WsSceneLatch; + +static inline int ws_scene_is_2d(WsSceneLatch* state, uint32_t frame, + int confirmed_overhang, int projecting_world, + uint32_t grace) { + if (state->initialized && (int32_t)(frame - state->last_frame) < 0) + state->confirmed = 0; + state->initialized = 1; + state->last_frame = frame; + if (confirmed_overhang) state->confirmed = 1; + if (confirmed_overhang || (state->confirmed && projecting_world)) + state->last_activity = frame; + if (state->confirmed && frame - state->last_activity > grace) + state->confirmed = 0; + return !state->confirmed; +} +#endif diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index 3ad80b69..445c5c39 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -7857,6 +7857,7 @@ static void handle_ws_backdrop_margin(int id, const char *json) if (m != -123456789) g_ws_bd_margin = m; send_fmt("{\"id\":%d,\"ok\":true,\"margin\":%d,\"mode\":\"%s\"}", id, g_ws_bd_margin, + g_ws_bd_margin == -2 ? "adaptive" : g_ws_bd_margin < 0 ? "whole-row" : (g_ws_bd_margin == 0 ? "off" : "widen-cols")); } diff --git a/runtime/src/gpu.c b/runtime/src/gpu.c index c4b80f1a..5c3bdc35 100644 --- a/runtime/src/gpu.c +++ b/runtime/src/gpu.c @@ -26,8 +26,11 @@ #include "mod_runtime.h" #include "sio.h" #include "ws_cull_detect.h" +#include "ws_backdrop_margin.h" #include "ws_aspect_cone_math.h" #include "ws_ui_group.h" +#include "ws_primitive_roles.h" +#include "ws_scene_latch.h" #include <stdint.h> #include <stdio.h> #include <stdlib.h> @@ -301,12 +304,29 @@ static int ws_game_mode(void) { * see the overhang block for the full lineage. Scoped to the tag-classified * (2.5D) mechanism: full-2D titles (MMX6) are 2D-only by definition and * genuinely reveal more, and GTE-detector titles (Ape) already classify 2D - * screens by projection count. The hysteresis rides out 1-2 frame gaps; a - * real scene change crosses it in ~0.1 s. */ + * screens by projection count. Once overhang confirms a world, active world + * projection keeps it alive between terrain-submission bursts. A real 2D + * scene with neither signal crosses the grace period in ~0.1 s. */ #define WS_2D_SCENE_HYSTERESIS 6u +static WsSceneLatch ws_scene_latch; static int ws_2d_only_scene(void) { if (ws_full_2d_mode() || ws_gte_game_mode_cfg) return 0; - return (uint32_t)s_frame_count - ws_sust_ovh_stamp > WS_2D_SCENE_HYSTERESIS; + uint32_t f = (uint32_t)s_frame_count; + return ws_scene_is_2d(&ws_scene_latch, f, + f - ws_sust_ovh_stamp <= WS_2D_SCENE_HYSTERESIS, + f - ws_last_world3d_stamp <= 2u, WS_2D_SCENE_HYSTERESIS); +} + +/* Host-derived presentation evidence is not serialized guest state. Relearn it + * after reset/load instead of carrying the old scene's classification over. */ +static void ws_reset_scene_history(void) { + uint32_t expired = (uint32_t)s_frame_count - 1000u; + memset(&ws_scene_latch, 0, sizeof ws_scene_latch); + ws_last_tag_stamp = ws_last_3d_stamp = ws_last_gte_stamp = expired; + ws_last_world3d_stamp = ws_sust_world3d_stamp = expired; + ws_last_ovh_stamp = ws_sust_ovh_stamp = expired; + ws_gte_frame = ws_ovh_frame = (uint32_t)-1; + ws_gte_count = ws_gte_prev_verts = ws_ovh_count = ws_ovh_prev = 0; } static uint32_t s_ws_fmv_frame_cache = 0xFFFFFFFFu; @@ -314,7 +334,10 @@ static int s_ws_fmv_cached = 0; int gpu_ws_present_native_43(void) { if (!ws_engaged()) return 0; - if (!ws_game_mode()) return 1; /* full-2D screen */ + if (!ws_game_mode()) { + ws_scene_latch.confirmed = 0; + return 1; /* full-2D screen */ + } if (ws_2d_only_scene()) return 1; /* 2D-only gameplay scene */ uint32_t f = (uint32_t)s_frame_count; if (f != s_ws_fmv_frame_cache) { @@ -392,9 +415,10 @@ static int ws_nw_configured_offset(void) { int local_offset = 0; if (ws_local_viewport_layout(NULL, NULL, NULL, &local_offset)) return local_offset; - int numr = 3 * ws_cfg_num - 4 * ws_cfg_den; + int64_t numr = (int64_t)3 * ws_cfg_num - (int64_t)4 * ws_cfg_den; int w = (int)ws_disp_w(); - return (w * numr + 4 * ws_cfg_den) / (8 * ws_cfg_den); + return (int)((w * numr + (int64_t)4 * ws_cfg_den) / + ((int64_t)8 * ws_cfg_den)); } static int ws_nw_offset(void) { if (!ws_native_wide_active()) return 0; @@ -1487,7 +1511,7 @@ int psx_ws_backdrop_x(int x) { * auto_backdrop). The recompiler/interp detect each scrolling-backdrop * column-window generator (see ws_backdrop_detect.h) and route its window START * and END bounds through psx_ws_backdrop_value(). When native-wide is engaged we - * WIDEN the camera-tracked window by the 16:9 reveal: the START (left) bound + * WIDEN the camera-tracked window by the live viewport reveal: the START (left) bound * moves left by `margin`, the END (right) bound moves right by `margin`, where * margin is the per-side reveal in COLUMNS. The window of window_cols columns * spans (at least) the full display width, so reveal_cols = window_cols * @@ -1501,12 +1525,16 @@ int psx_ws_backdrop_preload(void) { } /* Live-tunable widen amount (set via the `ws_backdrop_margin` debug command): - * <0 => WHOLE-ROW preload: START->0, END->extent-1 (max generous) + * -2 => adaptive: live viewport reveal plus the screen-space guard + * -1 => WHOLE-ROW preload: START->0, END->extent-1 (diagnostic) * 0 => identity: no widening, native-wide still on (A/B the effect) * >0 => widen the camera-tracked window by N columns each side (bounded) - * Default is a safe-but-generous bounded value; dialed live while debugging so - * the widen strategy can be tuned without a rebuild. */ -int g_ws_bd_margin = 0; /* column preload proven irrelevant to the void; default off */ + * Keep the default inert: a title must opt in after validating its producers. + * The debug override permits A/B testing without a rebuild. */ +int g_ws_bd_margin = 0; +void gpu_ws_set_adaptive_backdrop_preload(int enabled) { + g_ws_bd_margin = enabled ? -2 : 0; +} /* Set to 1 by the interpreter around its psx_ws_backdrop_value call so the value * function does NOT also ring-note: the interp records a richer entry (live * extent / camera / DL count). Native cache-DLL calls leave it 0, so the value @@ -1517,10 +1545,16 @@ uint32_t psx_ws_backdrop_value(uint32_t orig, int is_end, int window_cols) { if (!psx_ws_backdrop_preload()) return orig; /* 4:3 -> byte-identical */ if (window_cols < 0) window_cols = -window_cols; int m = g_ws_bd_margin; + int adaptive = m == -2; + if (adaptive) + m = psx_ws_backdrop_columns(window_cols, psx_ws_x_margin(), + (int)ws_disp_w()); int from_interp = g_ws_bd_from_interp; g_ws_bd_from_interp = 0; /* consume one-shot flag */ uint32_t finalv; - if (m < 0) { + if (adaptive) { + finalv = psx_ws_backdrop_bound(orig, is_end, m); + } else if (m < 0) { /* Whole-row: the generator's shared clamps (START<0 -> 0; END>=extent -> * extent-1) pin the loop to [0, extent-1]. Bounded because `extent` is a * byte and the DL count is a byte (row <= 255 cols). */ @@ -1838,6 +1872,7 @@ static int ws_bg_phase_over(void) { return s_bg_phase_over; } +static int ws_tagged_world_primitive(void); int psx_ws_prim_in_backdrop(void) { if (gp0_cmd_source_addr != 0xFFFFFFFFu) { uint32_t f = (uint32_t)s_frame_count; @@ -1846,6 +1881,9 @@ int psx_ws_prim_in_backdrop(void) { if (gp0_cmd_source_addr > g_bdg_src_hi) g_bdg_src_hi = gp0_cmd_source_addr; } if (g_dbg_mode != 0) return dbg_gate_match(); /* correlation override */ + /* Explicit title provenance outranks the legacy draw-order heuristic. + * Early-sorted world sprites must not be stretched into the reveal. */ + if (ws_tagged_world_primitive()) return 0; if (ws_nw_textured_edges) { uint32_t op = (gp0_cmd_buf[0] >> 24) & 0xFFu; if (op >= 0x20u && op <= 0x3Fu && (op & 0x04u)) @@ -2075,6 +2113,30 @@ static int ws_nw_hud_corners = 0; static int ws_nw_hud_tag_rects = 0; /* rects shift even when tagged (A/B) */ static uint32_t ws_nw_left_hud_packet_lo = 0; static uint32_t ws_nw_left_hud_packet_hi = 0; +/* Explicit title-owned roles. Separate from sprite tags: world sprites and + * HUD can share the same builder, and either entry hook may run first. Clear + * tags for recycled non-HUD packets; frame expiry protects unused addresses. */ +static WsPrimitiveRole ws_primitive_roles[WS_ROLE_BUCKETS]; +void gpu_ws_tag_hud_primitive(uint32_t primitive, int edge) { + WsPrimitiveRole* tag = ws_role_write(ws_primitive_roles, primitive, + (uint32_t)s_frame_count); + if (tag) tag->hud_edge = edge < 0 ? -1 : edge > 0 ? 1 : 0; +} +void gpu_ws_tag_world_primitive(uint32_t primitive, int is_world) { + WsPrimitiveRole* tag = ws_role_write(ws_primitive_roles, primitive, + (uint32_t)s_frame_count); + if (tag) tag->world = is_world != 0; +} +static int ws_tagged_world_primitive(void) { + const WsPrimitiveRole* tag = ws_role_read(ws_primitive_roles, + gp0_cmd_source_addr, (uint32_t)s_frame_count); + return tag && tag->world; +} +static int ws_tagged_hud_edge(void) { + const WsPrimitiveRole* tag = ws_role_read(ws_primitive_roles, + gp0_cmd_source_addr, (uint32_t)s_frame_count); + return tag ? tag->hud_edge : 0; +} void gpu_ws_set_nw_hud_corners(int on) { ws_nw_hud_corners = on ? 1 : 0; } void gpu_ws_set_nw_hud_tag_rects(int on) { ws_nw_hud_tag_rects = on ? 1 : 0; } void gpu_ws_set_nw_left_hud_packet_range(uint32_t lo, uint32_t hi) { @@ -2093,6 +2155,8 @@ static int32_t ws_nw_hud_shift(int32_t x, int32_t w) { if (!ws_native_wide_active()) return 0; int32_t off = ws_nw_offset(); if (off <= 0) return 0; + int hud_edge = ws_tagged_hud_edge(); + if (hud_edge) return hud_edge * off; if (!ws_nw_left_hud_packet() && !ws_nw_hud_corners) return 0; /* Sprite-tag titles (anchor configured): HUD ≡ UNTAGGED rect-family prims * — the same discriminator the squash path's hud_sprt_squash used. Tagged @@ -2517,6 +2581,7 @@ uint64_t g_pollhack_vblank_count = 0; /* instrumentation: poll-fallback VBlank /* ---- Initialization ---- */ static void gpu_reset_state(int clear_vram) { + ws_reset_scene_history(); if (clear_vram) { memset(vram, 0, sizeof(vram)); gpu_vram_dirty_mark_all(); @@ -5615,6 +5680,7 @@ int gpu_snapshot_read(const uint8_t *p, uint32_t len) { if (len != gpu_snapshot_bytes()) return 0; pst_r_init(&r, p, len); if (!gpu_snap_parse(&r)) return 0; + ws_reset_scene_history(); /* Sync renderer clip/scissor to restored GP0(E3/E4); vars alone leave GL * on a stale draw area after savestate load. */ gr_set_draw_area((int)draw_area_left, (int)draw_area_top, diff --git a/runtime/src/gpu_gl_renderer.c b/runtime/src/gpu_gl_renderer.c index b9081d69..3db48a0d 100644 --- a/runtime/src/gpu_gl_renderer.c +++ b/runtime/src/gpu_gl_renderer.c @@ -364,7 +364,8 @@ static GLuint s_raw_tex = 0, s_raw_fbo = 0; /* CPU->VRAM upload staging (native RGBA8). */ static GLuint s_up_tex = 0; /* copy_rect staging (hr-sized RGBA8). */ -static GLuint s_scratch_tex = 0, s_scratch_fbo = 0; +static GLuint s_scratch_tex = 0, s_scratch_fbo = 0; +static int s_scratch_w = 0, s_scratch_h = 0; /* Programs. */ static GLuint s_geo_prog = 0, s_geo_vao = 0, s_geo_vbo = 0; @@ -1385,8 +1386,17 @@ static void flush_cpu_upload(void) { /* Recreate one target's stencil mask from its authoritative alpha channel. * Sampling an attached render target is undefined, so copy color to the shared - * scratch texture first. Wide targets never exceed the 1024-pixel VRAM width. */ -static void rebuild_target_stencil(GLuint target_fbo, int target_w, int target_h) { + * scratch texture first. Uncapped Fit can exceed the 1024-pixel VRAM width, + * so grow this temporary storage to cover the actual target before copying. */ +static void rebuild_target_stencil(GLuint target_fbo, int target_w, int target_h) { + if (target_w > s_scratch_w || target_h > s_scratch_h) { + if (target_w > s_scratch_w) s_scratch_w = target_w; + if (target_h > s_scratch_h) s_scratch_h = target_h; + p_glActiveTexture(PSXGL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, s_scratch_tex); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, s_scratch_w, s_scratch_h, + 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL); + } p_glBindFramebuffer(PSXGL_READ_FRAMEBUFFER, target_fbo); p_glBindFramebuffer(PSXGL_DRAW_FRAMEBUFFER, s_scratch_fbo); p_glBlitFramebuffer(0, 0, target_w, target_h, 0, 0, target_w, target_h, @@ -2664,7 +2674,9 @@ static int init_gpu_raster(void) { int hw = VRAM_W * s_scale, hh = VRAM_H * s_scale; s_hr_tex = make_tex(GL_RGBA8, hw, hh, GL_RGBA, GL_UNSIGNED_BYTE); - s_scratch_tex = make_tex(GL_RGBA8, hw, hh, GL_RGBA, GL_UNSIGNED_BYTE); + s_scratch_tex = make_tex(GL_RGBA8, hw, hh, GL_RGBA, GL_UNSIGNED_BYTE); + s_scratch_w = hw; + s_scratch_h = hh; s_up_tex = make_tex(GL_RGBA8, VRAM_W, VRAM_H, GL_RGBA, GL_UNSIGNED_BYTE); s_raw_tex = make_tex(PSXGL_R16UI, VRAM_W, VRAM_H, PSXGL_RED_INTEGER, GL_UNSIGNED_SHORT); /* Force the driver's first texture-upload allocation while the renderer is diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 9406c8e7..a2294f35 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -1292,10 +1292,11 @@ extern "C" int psx_mod_set_fixed_display_aspect( extern "C" int psx_mod_set_adaptive_display_aspect( uint32_t max_numerator, uint32_t max_denominator) { - if (max_numerator == 0 || max_denominator == 0 || + const bool uncapped = max_numerator == 0 && max_denominator == 0; + if (!uncapped && (max_numerator == 0 || max_denominator == 0 || max_numerator > 99 || max_denominator > 99 || 3u * max_numerator < 4u * max_denominator || - 9u * max_numerator > 32u * max_denominator) { + 9u * max_numerator > 32u * max_denominator)) { std::fprintf(stderr, "psxrecomp: mod rejected invalid adaptive display aspect %u:%u\n", (unsigned)max_numerator, (unsigned)max_denominator); @@ -1304,7 +1305,12 @@ extern "C" int psx_mod_set_adaptive_display_aspect( g_ws_adaptive_view = true; g_ws_adaptive_max_num = (int)max_numerator; g_ws_adaptive_max_den = (int)max_denominator; - std::fprintf(stdout, + if (uncapped) { + std::fprintf(stdout, + "psxrecomp: mod selected adaptive display aspect " + "(initial %d:%d, fit to window, no upper aspect limit)\n", + g_video_aspect_num, g_video_aspect_den); + } else std::fprintf(stdout, "psxrecomp: mod selected adaptive display aspect " "(initial %d:%d, range 4:3 through %u:%u)\n", g_video_aspect_num, g_video_aspect_den, @@ -1647,7 +1653,8 @@ static void update_adaptive_widescreen() { int num = width, den = height; if ((int64_t)width * 3 <= (int64_t)height * 4) { num = 4; den = 3; - } else if ((int64_t)width * g_ws_adaptive_max_den >= + } else if (g_ws_adaptive_max_num > 0 && g_ws_adaptive_max_den > 0 && + (int64_t)width * g_ws_adaptive_max_den >= (int64_t)height * g_ws_adaptive_max_num) { num = g_ws_adaptive_max_num; den = g_ws_adaptive_max_den; @@ -1662,7 +1669,7 @@ static void update_adaptive_widescreen() { g_video_aspect_den = den; gl_renderer_set_display_aspect(num, den); if (sdl_renderer) { - g_logical_w = 480 * num * g_video_scale / den; + g_logical_w = (int)((int64_t)480 * num * g_video_scale / den); SDL_RenderSetLogicalSize(sdl_renderer, g_logical_w, 480 * g_video_scale); } diff --git a/runtime/src/mod_runtime.cpp b/runtime/src/mod_runtime.cpp index b4d72770..5fed322a 100644 --- a/runtime/src/mod_runtime.cpp +++ b/runtime/src/mod_runtime.cpp @@ -43,6 +43,9 @@ extern "C" uint32_t psx_mod_memory_alloc(uint32_t size, uint32_t alignment); extern "C" uint32_t psx_mod_gpu_dma_memory_alloc(uint32_t size, uint32_t alignment); extern "C" int psx_ws_x_margin(void); +extern "C" void gpu_ws_tag_hud_primitive(uint32_t primitive, int edge); +extern "C" void gpu_ws_tag_world_primitive(uint32_t primitive, int is_world); +extern "C" void gpu_ws_set_adaptive_backdrop_preload(int enabled); extern "C" void dirty_ram_mark_executable_range(uint32_t phys, uint32_t len); extern "C" int fntrace_is_game_started(void); @@ -1382,6 +1385,18 @@ extern "C" int32_t psx_mod_widescreen_x_margin(void) { return (int32_t)psx_ws_x_margin(); } +extern "C" void psx_mod_tag_hud_primitive(uint32_t primitive, int edge) { + gpu_ws_tag_hud_primitive(primitive, edge); +} + +extern "C" void psx_mod_tag_world_primitive(uint32_t primitive, int is_world) { + gpu_ws_tag_world_primitive(primitive, is_world); +} + +extern "C" void psx_mod_set_adaptive_backdrop_preload(int enabled) { + gpu_ws_set_adaptive_backdrop_preload(enabled); +} + extern "C" int psx_mod_register_function_entry_plugin( const char* id, uint32_t address, PSXModFunctionEntryCallback callback) { using namespace PSXRecompV4;