tooling: fatal halt-and-serve + always-on GP1 display ring

- psx_fatal_halt: deliberate fatal sites (unhandled MMIO access,
  trap_crash) now write the structured crash report AND a full
  freeze-style ring dump (wedge kind "fatal"), then keep pumping the
  TCP debug server forever in debug-tools builds so every ring stays
  queryable against the exact crash state (ring-buffer-first: crashes
  are interrogated, not snapshotted). Release builds exit(1) as before.
  Re-entry guarded; heartbeat JSON gains a "fatal" field.
- gp1_dump: dedicated always-on GP1 (0x1F801814 display control) ring,
  512K entries (~15 min of gameplay) with frame-window server-side
  filtering. The general MMIO ring rolls over in under a minute of SPU/
  DMA traffic, which kept evicting boot-window display history before a
  probe could ask "who toggled the display during the logo" (bug 1).
- mmio_dump/mmio_clear documented in TCP_COMMANDS.md.

The halt-and-serve path is what made the pig-crash root cause a
one-repro diagnosis.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

TCP_COMMANDS.md

@@ -72,6 +72,9 @@ Columns: **N** = native, **D** = DuckStation oracle.
 | `wtrace_range` | ✓ |   | `lo`, `hi` | Set RAM-write trace range (ring of 1024 writes with RA) |
 | `wtrace_dump` | ✓ | beetle | optional `addr_lo`, `addr_hi`, `count`, `newest` | Dump RAM-write trace entries as JSON. The address filter is applied server-side over the FULL ring before the emit cap — always pass it when hunting a specific buffer, otherwise you only see the oldest `count` entries of the whole ring |
 | `wtrace_clear` | ✓ |   | — | Reset the trace ring |
+| `mmio_dump` | ✓ |   | optional `addr`, `count`, `newest` | Dump the always-on MMIO write ring (256K entries, ALL 0x1F801xxx writes — SPU/DMA traffic rolls it in well under a minute of gameplay; for display history use `gp1_dump`) |
+| `mmio_clear` | ✓ |   | — | Reset the MMIO write ring |
+| `gp1_dump` | ✓ |   | optional `frame_lo`, `frame_hi`, `count`, `newest` | Dump the dedicated ALWAYS-ON GP1 (0x1F801814 display control) ring — 512K entries ≈ 15 min of gameplay (Tomba writes ~10 GP1/frame), survives the general MMIO ring's eviction. Frame filter is server-side over the full ring. Each entry: val + func/pc/cpu_pc/ra/sp/a0/a1/sr/epc/frame |
 | `pc_break` |   | ✓² | `addr` | DS execute breakpoint, state captured on hit (via `pc_hit_last`) |
 | `pc_unbreak` |   | ✓² | `addr` | Remove an execute breakpoint |
 | `pc_break_list` |   | ✓² | — | List active execute breakpoints |

runtime/include/crash_trace.h

@@ -21,6 +21,21 @@ void psx_crash_trace_dump(const char *reason, void *seh_info);
  * exit_origin is still "unknown" means an untagged exit fired. */
 void psx_crash_trace_set_exit_origin(const char *origin);
 
+/* Terminal stop for deliberate fatal sites (unhandled MMIO access,
+ * trap_crash, fail-fast unknown dispatch). Writes the structured crash
+ * report AND a full freeze-style ring dump, then:
+ *   - debug-tools builds: halts emulation but keeps pumping the TCP
+ *     debug server forever so every ring buffer stays queryable
+ *     post-mortem (ring-buffer-first: the crash state is interrogated,
+ *     not just snapshotted).
+ *   - PSX_NO_DEBUG_TOOLS builds: exit(1) as before.
+ * Never returns. */
+void psx_fatal_halt(const char *reason);
+
+/* Set when psx_fatal_halt fires; the freeze heartbeat includes it so an
+ * external reader can tell a fatal halt from a wedge. NULL = healthy. */
+extern const char *g_psx_fatal_reason;
+
 #ifdef __cplusplus
 }
 #endif

runtime/src/crash_trace.c

@@ -259,6 +259,48 @@ void psx_crash_trace_dump(const char *reason, void *seh_info) {
     }
 }
 
+/* ── Fatal halt ──────────────────────────────────────────────────────── */
+
+const char *g_psx_fatal_reason = NULL;
+
+/* freeze_heartbeat.c — full ring dump with wedge_kind "fatal". */
+extern void freeze_heartbeat_fatal_dump(const char *reason);
+
+void psx_fatal_halt(const char *reason) {
+    /* Re-entry guard: a post-mortem TCP command served from the halt
+     * loop below can itself trip a fatal site. Don't re-dump (the first
+     * fatal is the real one) and don't recurse another serve loop. */
+    static int s_halted = 0;
+    if (!s_halted) {
+        s_halted = 1;
+        g_psx_fatal_reason = reason ? reason : "(fatal)";
+        psx_crash_trace_dump(g_psx_fatal_reason, NULL);
+        freeze_heartbeat_fatal_dump(g_psx_fatal_reason);
+    }
+#ifndef PSX_NO_DEBUG_TOOLS
+    /* Halt-and-serve: emulation is dead but the rings are not. Keep the
+     * TCP debug server pumping on this (main) thread so a post-mortem
+     * client can run wtrace_dump / read_ram / screenshot / etc. against
+     * the exact crash state. */
+    extern void debug_server_poll(void);
+    fprintf(stderr,
+            "FATAL: %s — emulation halted; TCP debug server stays live "
+            "for post-mortem ring queries.\n", g_psx_fatal_reason);
+    fflush(stderr);
+    for (;;) {
+        debug_server_poll();
+#ifdef _WIN32
+        Sleep(1);
+#else
+        struct timespec req = {0, 1000000};
+        nanosleep(&req, NULL);
+#endif
+    }
+#else
+    exit(1);
+#endif
+}
+
 /* ── Crash handlers ──────────────────────────────────────────────────── */
 
 #include <signal.h>

runtime/src/debug_server.c

@@ -3223,6 +3223,19 @@ static MmioTraceEntry *s_mmio_trace = NULL;
 static uint64_t s_mmio_trace_seq  = 0;
 static uint32_t s_mmio_trace_head = 0;
 
+/* ---- GP1 display-control trace (ALWAYS-ON, dedicated) ----
+ * The general MMIO ring rolls over in well under a minute of gameplay
+ * (SPU/DMA traffic), evicting the boot-window display history before a
+ * post-hoc probe can ask "who toggled the display during the logo?".
+ * GP1 writes are ~10/frame in-game (measured: Tomba attract ≈ 600/s),
+ * so 512K entries ≈ 15 minutes — enough to attribute a boot window from
+ * well after the title screen. ~40 MB heap.
+ * Same entry layout as the general ring; dumped via `gp1_dump`. */
+#define GP1_TRACE_CAP (1 << 19)
+static MmioTraceEntry *s_gp1_trace = NULL;
+static uint64_t s_gp1_trace_seq  = 0;
+static uint32_t s_gp1_trace_head = 0;
+
 /* ---- Platform helpers ---- */
 static void set_nonblocking(sock_t s)
 {
@@ -5828,6 +5841,15 @@ void debug_server_trace_mmio_write(uint32_t addr, uint32_t val, uint8_t width)
     e->frame     = (uint32_t)s_frame_count;
     e->width     = width;
     s_mmio_trace_head = (s_mmio_trace_head + 1) % MMIO_TRACE_CAP;
+
+    /* Mirror GP1 (0x1F801814) writes into the dedicated long-retention
+     * display-control ring. */
+    if (addr == 0x1F801814u && s_gp1_trace) {
+        MmioTraceEntry *g = &s_gp1_trace[s_gp1_trace_head];
+        *g = *e;
+        g->seq = s_gp1_trace_seq++;
+        s_gp1_trace_head = (s_gp1_trace_head + 1) % GP1_TRACE_CAP;
+    }
 }
 
 static void handle_wtrace_range(int id, const char *json)
@@ -6842,6 +6864,65 @@ static void handle_mmio_clear(int id, const char *json)
     send_ok(id);
 }
 
+/* gp1_dump — dump the dedicated GP1 display-control ring. Optional
+ * `frame_lo`/`frame_hi` filter (applied server-side over the FULL ring,
+ * like wtrace_dump's address filter), optional `count` (default 4096),
+ * optional `newest` (1 = newest-first). */
+static void handle_gp1_dump(int id, const char *json)
+{
+    if (!s_gp1_trace) { send_err(id, "gp1 trace not initialized"); return; }
+
+    int frame_lo = json_get_int(json, "frame_lo", 0);
+    int frame_hi = json_get_int(json, "frame_hi", 0x7FFFFFFF);
+
+    uint64_t total = s_gp1_trace_seq;
+    uint32_t avail = (total < GP1_TRACE_CAP) ? (uint32_t)total : GP1_TRACE_CAP;
+    uint32_t start = (total < GP1_TRACE_CAP) ? 0 : s_gp1_trace_head;
+
+    int max_out = json_get_int(json, "count", 4096);
+    if (max_out < 1) max_out = 1;
+    if (max_out > (int)GP1_TRACE_CAP) max_out = (int)GP1_TRACE_CAP;
+    int newest_first = json_get_int(json, "newest", 0) != 0;
+
+    const uint32_t MAX_OUT = (uint32_t)max_out;
+    size_t BUF_SZ = 256u + (size_t)MAX_OUT * 512u;
+    if (BUF_SZ > (size_t)128 * 1024 * 1024) BUF_SZ = (size_t)128 * 1024 * 1024;
+    char *buf = (char *)malloc(BUF_SZ);
+    if (!buf) { send_err(id, "oom"); return; }
+    size_t pos = 0;
+    uint32_t emitted = 0;
+    pos += snprintf(buf + pos, BUF_SZ - pos,
+                    "{\"id\":%d,\"ok\":true,\"total\":%llu,\"available\":%u,\"entries\":[",
+                    id, (unsigned long long)total, avail);
+    for (uint32_t i = 0; i < avail && emitted < MAX_OUT && pos < BUF_SZ - 256; i++) {
+        uint32_t idx;
+        if (newest_first) {
+            uint32_t newest = (s_gp1_trace_head + GP1_TRACE_CAP - 1u) % GP1_TRACE_CAP;
+            idx = (newest + GP1_TRACE_CAP - (i % GP1_TRACE_CAP)) % GP1_TRACE_CAP;
+        } else {
+            idx = (start + i) % GP1_TRACE_CAP;
+        }
+        MmioTraceEntry *e = &s_gp1_trace[idx];
+        if ((int)e->frame < frame_lo || (int)e->frame > frame_hi) continue;
+        pos += snprintf(buf + pos, BUF_SZ - pos,
+                        "%s{\"seq\":%llu,\"val\":\"0x%08X\","
+                        "\"func\":\"0x%08X\",\"pc\":\"0x%08X\",\"cpu_pc\":\"0x%08X\","
+                        "\"ra\":\"0x%08X\",\"sp\":\"0x%08X\","
+                        "\"a0\":\"0x%08X\",\"a1\":\"0x%08X\","
+                        "\"sr\":\"0x%08X\",\"epc\":\"0x%08X\","
+                        "\"frame\":%u}",
+                        (emitted == 0) ? "" : ",",
+                        (unsigned long long)e->seq,
+                        e->val, e->func_addr, e->pc, e->cpu_pc,
+                        e->ra, e->sp, e->a0, e->a1,
+                        e->sr, e->epc, e->frame);
+        emitted++;
+    }
+    pos += snprintf(buf + pos, BUF_SZ - pos, "],\"emitted\":%u}", emitted);
+    debug_server_send_line(buf);
+    free(buf);
+}
+
 static const char *mdec_event_kind_name(uint32_t kind)
 {
     switch (kind) {
@@ -8210,6 +8291,7 @@ static const CmdEntry s_commands[] = {
     { "freeze_check",      handle_freeze_check },
     { "mmio_dump",         handle_mmio_dump },
     { "mmio_clear",        handle_mmio_clear },
+    { "gp1_dump",          handle_gp1_dump },
     { "mdec_state",        handle_mdec_state },
     { "mdec_trace",        handle_mdec_trace },
     { "mdec_trace_clear",  handle_mdec_trace_clear },
@@ -8629,6 +8711,13 @@ void debug_server_init(int port)
     s_mmio_trace_seq = 0;
     s_mmio_trace_head = 0;
 
+    /* Dedicated GP1 display-control ring (long retention). */
+    if (!s_gp1_trace) {
+        s_gp1_trace = (MmioTraceEntry *)calloc(GP1_TRACE_CAP, sizeof(MmioTraceEntry));
+    }
+    s_gp1_trace_seq = 0;
+    s_gp1_trace_head = 0;
+
     memset(s_watchpoints, 0, sizeof(s_watchpoints));
     memset(s_snapshot_addrs, 0, sizeof(s_snapshot_addrs));
     memset(s_snapshot_active, 0, sizeof(s_snapshot_active));

runtime/src/freeze_heartbeat.c

@@ -2,6 +2,7 @@
 
 #include "freeze_heartbeat.h"
 #include "debug_server.h"
+#include "crash_trace.h"   /* g_psx_fatal_reason */
 
 #include <stdint.h>
 #include <stdio.h>
@@ -321,7 +322,8 @@ static void freeze_dump_write(long long wall, uint64_t frame, uint64_t cyc,
         s_last_wedge_kind,
         (s_last_wedge_kind == 1) ? "hard_freeze" :
         (s_last_wedge_kind == 2) ? "reentry_storm" :
-        (s_last_wedge_kind == 3) ? "slow_frames" : "unknown",
+        (s_last_wedge_kind == 3) ? "slow_frames" :
+        (s_last_wedge_kind == 4) ? "fatal" : "unknown",
         (unsigned)DUMP_CAP_WTRACE_ALL,
         (unsigned)DUMP_CAP_WTRACE,
         (unsigned)DUMP_CAP_FRAME_HISTORY,
@@ -413,6 +415,42 @@ static void freeze_dump_write(long long wall, uint64_t frame, uint64_t cyc,
     fclose(f);
 }
 
+/* Full ring dump for deliberate fatal sites (psx_fatal_halt). Runs ON the
+ * faulting (main) thread, unlike the watchdog path — wedge kind 4 keeps
+ * freeze_dump_write away from the SuspendThread-based stack walkers,
+ * which would self-suspend here. Gathers the same diag snapshot the
+ * heartbeat tick does so the dump is self-contained. */
+void freeze_heartbeat_fatal_dump(const char *reason) {
+    (void)reason;  /* reason travels via g_psx_fatal_reason in the heartbeat */
+
+    uint64_t total_checks = 0;
+    uint32_t dispatch_count = 0;
+    int in_exc = 0;
+    int post_cool = 0;
+    uint64_t exc_entries = 0;
+    uint64_t exc_reentry = 0;
+    psx_get_freeze_diag(&total_checks, &dispatch_count, &in_exc,
+                        &post_cool, &exc_entries, &exc_reentry);
+
+    int sio_irq_pending = 0;
+    int sio_irq_countdown = 0;
+    uint16_t sio_stat = 0;
+    uint16_t sio_ctrl = 0;
+    int card_active = 0;
+    sio_get_freeze_diag(&sio_irq_pending, &sio_irq_countdown,
+                        &sio_stat, &sio_ctrl, &card_active);
+
+    s_last_wedge_kind = 4;
+    s_dump_armed = 0;  /* the watchdog must not overwrite the fatal dump */
+    freeze_dump_write((long long)time(NULL), s_frame_count,
+                      psx_get_cycle_count(), exc_reentry,
+                      g_debug_current_func_addr, g_debug_last_store_pc,
+                      i_stat, i_mask, in_exc, total_checks,
+                      dispatch_count, exc_entries,
+                      sio_stat, sio_ctrl, card_active,
+                      sio_get_mc_max_state(), sio_get_tx_writes());
+}
+
 static void heartbeat_write(void) {
     uint64_t cyc = psx_get_cycle_count();
     uint64_t frame = s_frame_count;
@@ -535,7 +573,8 @@ static void heartbeat_write(void) {
         "  \"mc_max_state\":%d,\n"
         "  \"tx_writes\":%d,\n"
         "  \"dirty_ram_blocks\":%llu,\n"
-        "  \"dirty_ram_insns\":%llu\n"
+        "  \"dirty_ram_insns\":%llu,\n"
+        "  \"fatal\":%s%s%s\n"
         "}\n",
         s_backend,
         wall,
@@ -558,7 +597,10 @@ static void heartbeat_write(void) {
         mc_max,
         tx_writes,
         (unsigned long long)g_dirty_ram_blocks_run,
-        (unsigned long long)g_dirty_ram_insns_run);
+        (unsigned long long)g_dirty_ram_insns_run,
+        g_psx_fatal_reason ? "\"" : "",
+        g_psx_fatal_reason ? g_psx_fatal_reason : "null",
+        g_psx_fatal_reason ? "\"" : "");
 
     if (n <= 0 || n >= (int)sizeof(buf)) return;
 

runtime/src/memory.c

@@ -10,6 +10,7 @@
 
 #include "cpu_state.h"
 #include "cdrom.h"
+#include "crash_trace.h"
 #include "dma.h"
 #include "gpu.h"
 #include "mdec.h"
@@ -284,23 +285,24 @@ void memory_init(const char* bios_path) {
 }
 
 static void mmio_fatal(uint32_t vaddr, uint32_t phys, const char* op) {
-    fprintf(stderr, "MMIO %s @ 0x%08X (phys 0x%08X)\n", op, vaddr, phys);
+    static char reason[96];
+    snprintf(reason, sizeof(reason), "MMIO %s @ 0x%08X (phys 0x%08X)", op, vaddr, phys);
+    fprintf(stderr, "%s\n", reason);
     fflush(stderr);
     FILE* cf = fopen("psx_crash.txt", "w");
-    if (cf) { fprintf(cf, "MMIO %s @ 0x%08X (phys 0x%08X)\n", op, vaddr, phys); fclose(cf); }
-    exit(1);
+    if (cf) { fprintf(cf, "%s\n", reason); fclose(cf); }
+    psx_fatal_halt(reason);
 }
 
 static void mmio_unimplemented(uint32_t addr, const char* op) {
-    fprintf(stderr, "UNIMPLEMENTED MMIO %s @ 0x%08X\n", op, addr);
+    static char reason[96];
+    snprintf(reason, sizeof(reason), "UNIMPLEMENTED MMIO %s @ 0x%08X", op, addr);
+    fprintf(stderr, "%s\n", reason);
     fflush(stderr);
     /* Also write to a crash file for capture when stderr is lost. */
     FILE* cf = fopen("psx_crash.txt", "w");
-    if (cf) {
-        fprintf(cf, "UNIMPLEMENTED MMIO %s @ 0x%08X\n", op, addr);
-        fclose(cf);
-    }
-    exit(1);
+    if (cf) { fprintf(cf, "%s\n", reason); fclose(cf); }
+    psx_fatal_halt(reason);
 }
 
 static void unmapped_fatal(uint32_t vaddr, uint32_t phys, const char* op) {

runtime/src/traps.c

@@ -6,6 +6,7 @@
 
 #include "cpu_state.h"
 #include "debug_server.h"
+#include "crash_trace.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -19,9 +20,10 @@ void psx_exception_longjmp(void);
 static void trap_crash(const char* msg) {
     FILE* cf = fopen("psx_crash.txt", "w");
     if (cf) { fprintf(cf, "%s\n", msg); fclose(cf); }
-    /* Also dump the structured crash trace with rings + cpu state. */
-    extern void psx_crash_trace_dump(const char *reason, void *seh_info);
-    psx_crash_trace_dump("trap_crash", NULL);
+    /* Full ring dump, then halt-and-serve (debug builds) / exit(1)
+     * (release). Never returns — call sites' trailing exit(1) are
+     * unreachable belt-and-braces. */
+    psx_fatal_halt(msg);
 }
 
 static int psx_is_valid_tcb(CPUState* cpu, uint32_t tcb)