diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 56997453..00b9f975 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -357,6 +357,12 @@ if(BUILD_TESTING) target_include_directories(frame_pacing_test PRIVATE include) add_test(NAME frame_pacing_test COMMAND frame_pacing_test) + add_executable(freeze_dump_policy_test + tests/test_freeze_dump_policy.c + src/freeze_dump_policy.c) + target_include_directories(freeze_dump_policy_test PRIVATE src) + add_test(NAME freeze_dump_policy_test COMMAND freeze_dump_policy_test) + add_executable(frame_interpolation_schedule_test tests/test_frame_interpolation_schedule.c src/frame_interpolation.c) diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index ef60dc37..732c15e1 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -293,6 +293,7 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/cosim.c ${PSXRECOMP_ROOT}/runtime/src/traps.c ${PSXRECOMP_ROOT}/runtime/src/crash_trace.c + ${PSXRECOMP_ROOT}/runtime/src/freeze_dump_policy.c ${PSXRECOMP_ROOT}/runtime/src/freeze_heartbeat.c ${PSXRECOMP_ROOT}/runtime/src/gte.cpp ${PSXRECOMP_ROOT}/runtime/src/pgxp.cpp diff --git a/runtime/src/freeze_dump_policy.c b/runtime/src/freeze_dump_policy.c new file mode 100644 index 00000000..8197dae4 --- /dev/null +++ b/runtime/src/freeze_dump_policy.c @@ -0,0 +1,91 @@ +#include "freeze_dump_policy.h" + +#include + +static unsigned wedge_priority(uint32_t wedge_kind) { + switch (wedge_kind) { + case 1: return 4; /* hard freeze */ + case 2: return 3; /* exception re-entry storm */ + case 3: return 2; /* slow frames */ + case 5: return 1; /* logical spin */ + default: return 0; + } +} + +int freeze_dump_policy_observe(FreezeDumpPolicy *policy, uint32_t wedge_kind, + int fatal_active) { + if (!policy) return 0; + if (fatal_active) return 0; + + unsigned priority = wedge_priority(wedge_kind); + if (wedge_kind != 0 && priority == 0) return 0; + + if (wedge_kind != 0) { + policy->healthy_ticks = 0; + if (policy->in_wedge) { + /* Only a hard freeze can reserve the second slot during an + * existing event. Lesser severity changes stay in one incident. */ + if (wedge_kind != 1 || policy->wedge_kind == 1) return 0; + } + + policy->in_wedge = 1; + policy->wedge_kind = wedge_kind; + + if (wedge_kind == 1) { + if (!policy->hard_slot_consumed) { + policy->pending_kind = wedge_kind; + return 1; + } + } else if (!policy->ordinary_slot_consumed) { + policy->pending_kind = wedge_kind; + return 1; + } + + policy->suppressed_events++; + return 0; + } + + if (!policy->in_wedge) return 0; + + if (++policy->healthy_ticks >= FREEZE_DUMP_REARM_HEALTHY_TICKS) { + policy->healthy_ticks = 0; + policy->wedge_kind = 0; + policy->in_wedge = 0; + } + return 0; +} + +void freeze_dump_policy_record_result(FreezeDumpPolicy *policy, int written) { + if (!policy || policy->pending_kind == 0) return; + + int hard = policy->pending_kind == 1; + if (written) { + if (hard) + policy->hard_slot_consumed = 1; + else + policy->ordinary_slot_consumed = 1; + policy->automatic_dumps++; + } else { + uint32_t *attempts = hard ? &policy->hard_failed_attempts + : &policy->ordinary_failed_attempts; + int *consumed = hard ? &policy->hard_slot_consumed + : &policy->ordinary_slot_consumed; + policy->failed_dumps++; + if (++*attempts >= FREEZE_DUMP_MAX_FAILED_ATTEMPTS) *consumed = 1; + + /* A failed attempt ends this event. The next sample can retry only + * while the slot remains within its fixed attempt limit. */ + policy->healthy_ticks = 0; + policy->wedge_kind = 0; + policy->in_wedge = 0; + } + policy->pending_kind = 0; +} + +int freeze_dump_format_path(char *out, size_t cap, const char *backend, + long long wall, uint32_t sequence) { + if (!out || cap == 0 || !backend) return 0; + int n = snprintf(out, cap, "psx_freeze_dump_%s_%lld_%u.json", + backend, wall, sequence); + return n > 0 && (size_t)n < cap; +} diff --git a/runtime/src/freeze_dump_policy.h b/runtime/src/freeze_dump_policy.h new file mode 100644 index 00000000..22f2c8e6 --- /dev/null +++ b/runtime/src/freeze_dump_policy.h @@ -0,0 +1,46 @@ +#ifndef PSXRECOMP_FREEZE_DUMP_POLICY_H +#define PSXRECOMP_FREEZE_DUMP_POLICY_H + +#include +#include + +/* Full freeze dumps contain several large trace rings. Keep automatic + * watchdog output bounded while leaving deliberate fatal dumps independent. */ +#define FREEZE_DUMP_ORDINARY_SLOTS 1u +#define FREEZE_DUMP_HARD_SLOTS 1u +#define FREEZE_DUMP_AUTO_LIMIT \ + (FREEZE_DUMP_ORDINARY_SLOTS + FREEZE_DUMP_HARD_SLOTS) +#define FREEZE_DUMP_MAX_FAILED_ATTEMPTS 3u +/* Policy hysteresis, independent of the detector's sample window. The total + * time before rearm can therefore be longer than this tick count alone. */ +#define FREEZE_DUMP_REARM_HEALTHY_TICKS 20u + +typedef struct FreezeDumpPolicy { + uint32_t automatic_dumps; + uint32_t failed_dumps; + uint32_t suppressed_events; + uint32_t healthy_ticks; + uint32_t wedge_kind; + uint32_t pending_kind; + uint32_t ordinary_failed_attempts; + uint32_t hard_failed_attempts; + int ordinary_slot_consumed; + int hard_slot_consumed; + int in_wedge; +} FreezeDumpPolicy; + +/* Observe one watchdog sample. Returns nonzero when the caller must write an + * automatic full dump. A new event requires a sustained healthy interval. + * Fatal state bypasses this automatic policy and does not change its budget. */ +int freeze_dump_policy_observe(FreezeDumpPolicy *policy, uint32_t wedge_kind, + int fatal_active); + +/* Record whether the authorized full dump reached disk. A slot permits a + * bounded number of failed attempts before the policy consumes that slot. */ +void freeze_dump_policy_record_result(FreezeDumpPolicy *policy, int written); + +/* Format a unique dump path. The sequence prevents same-second truncation. */ +int freeze_dump_format_path(char *out, size_t cap, const char *backend, + long long wall, uint32_t sequence); + +#endif /* PSXRECOMP_FREEZE_DUMP_POLICY_H */ diff --git a/runtime/src/freeze_heartbeat.c b/runtime/src/freeze_heartbeat.c index d6029574..3df0fa5f 100644 --- a/runtime/src/freeze_heartbeat.c +++ b/runtime/src/freeze_heartbeat.c @@ -1,6 +1,7 @@ /* freeze_heartbeat.c — see header for rationale. */ #include "freeze_heartbeat.h" +#include "freeze_dump_policy.h" #include "debug_server.h" #include "crash_trace.h" /* g_psx_fatal_reason */ #include "cpu_state.h" /* g_psx_bail_* call-contract counters */ @@ -164,10 +165,9 @@ static HbRingEntry s_ring[RING_CAP]; static uint32_t s_ring_head = 0; static uint32_t s_ring_count = 0; -/* Wedge detection state. - * s_dump_armed - 1 if a wedge dump is allowed to fire; cleared after - * firing, re-armed when the wedge clears (healthy tick). */ -static int s_dump_armed = 1; +/* Automatic full dumps are bounded independently from deliberate fatal dumps. + * The heartbeat JSON reports both written and suppressed event counts. */ +static FreezeDumpPolicy s_dump_policy = {0}; static uint32_t s_last_wedge_kind = 0; /* informational, last detected kind */ #ifdef _WIN32 @@ -314,6 +314,7 @@ static void freeze_dump_main_stack_samples_json(FILE *f, int n) { * main thread mid-dump (2026-06-10 chest-freeze postmortem). First dump * in wins; the loser skips (same rings either way). */ static volatile long s_dump_in_progress = 0; +static uint32_t s_dump_sequence = 0; /* protected by s_dump_in_progress */ static void freeze_dump_unlock(void) { #ifdef _WIN32 @@ -458,14 +459,14 @@ static int hb_format_cpu_scratchpad(char *out, size_t cap) { return n; } -static void freeze_dump_write(long long wall, uint64_t frame, uint64_t cyc, - uint64_t exc_reentry, uint32_t cur_fn, - uint32_t last_store, uint32_t i_stat_v, - uint32_t i_mask_v, int in_exc, - uint64_t total_checks, uint32_t dispatch_count, - uint64_t exc_entries, - uint16_t sio_stat_v, uint16_t sio_ctrl_v, - int sio_card_active, int mc_max, int tx_writes) +static int freeze_dump_write(long long wall, uint64_t frame, uint64_t cyc, + uint64_t exc_reentry, uint32_t cur_fn, + uint32_t last_store, uint32_t i_stat_v, + uint32_t i_mask_v, int in_exc, + uint64_t total_checks, uint32_t dispatch_count, + uint64_t exc_entries, + uint16_t sio_stat_v, uint16_t sio_ctrl_v, + int sio_card_active, int mc_max, int tx_writes) { /* Snapshot the kind at entry — the global can be flipped mid-write by * the other thread, which is what sent the fatal (kind 4) path into @@ -474,18 +475,21 @@ static void freeze_dump_write(long long wall, uint64_t frame, uint64_t cyc, #ifdef _WIN32 if (InterlockedCompareExchange((volatile LONG *)&s_dump_in_progress, 1, 0) != 0) - return; + return 0; #else if (__sync_lock_test_and_set(&s_dump_in_progress, 1) != 0) - return; + return 0; #endif char path[128]; - snprintf(path, sizeof(path), - "psx_freeze_dump_%s_%lld.json", s_backend, wall); + uint32_t sequence = s_dump_sequence++; + if (!freeze_dump_format_path(path, sizeof(path), s_backend, wall, sequence)) { + freeze_dump_unlock(); + return 0; + } FILE *f = fopen(path, "wb"); - if (!f) { freeze_dump_unlock(); return; } + if (!f) { freeze_dump_unlock(); return 0; } /* Large stdio buffer so multi-MB JSON arrays write efficiently. */ static char io_buf[1 << 16]; @@ -657,8 +661,11 @@ static void freeze_dump_write(long long wall, uint64_t frame, uint64_t cyc, } fputs("}\n", f); - fclose(f); + int written = !ferror(f); + if (fclose(f) != 0) written = 0; + if (!written) (void)remove(path); freeze_dump_unlock(); + return written; } /* Full ring dump for deliberate fatal sites (psx_fatal_halt). Runs ON the @@ -694,14 +701,13 @@ void freeze_heartbeat_fatal_dump(const char *reason) { &sio_stat, &sio_ctrl, &card_active); s_last_wedge_kind = 4; - s_dump_armed = 0; /* the watchdog must not overwrite the fatal dump */ - freeze_dump_write((long long)time(NULL), s_frame_count, - psx_get_cycle_count(), exc_reentry, - g_debug_current_func_addr, g_debug_last_store_pc, - i_stat, i_mask, in_exc, total_checks, - dispatch_count, exc_entries, - sio_stat, sio_ctrl, card_active, - sio_get_mc_max_state(), sio_get_tx_writes()); + (void)freeze_dump_write((long long)time(NULL), s_frame_count, + psx_get_cycle_count(), exc_reentry, + g_debug_current_func_addr, g_debug_last_store_pc, + i_stat, i_mask, in_exc, total_checks, + dispatch_count, exc_entries, + sio_stat, sio_ctrl, card_active, + sio_get_mc_max_state(), sio_get_tx_writes()); } static void heartbeat_write(void) { @@ -757,15 +763,16 @@ static void heartbeat_write(void) { s_ring_head = (s_ring_head + 1) % RING_CAP; if (s_ring_count < RING_CAP) s_ring_count++; - /* ---- Wedge detection: arm-once auto-dump ---- + /* ---- Wedge detection: bounded automatic dump ---- * Walk back WEDGE_WINDOW_TICKS in the heartbeat ring (just pushed * above) and compute deltas. Trigger if any of: * A. frame_count delta == 0 (hard freeze) * B. exc_reentry delta PER FRAME > threshold (reentry storm) * C. frame_count delta < slow-frames floor (host-side stall) * - * Only fires once the ring has enough history. When the wedge clears - * (a healthy tick), re-arm. */ + * Only fires once the ring has enough history. The dump policy requires + * a sustained healthy interval before it accepts a new event, and bounds + * automatic full dumps for the process. Deliberate fatal dumps bypass it. */ uint32_t wedge_kind = 0; /* 0=healthy 1=hard 2=reentry storm 3=slow frames */ if (s_ring_count >= WEDGE_WINDOW_TICKS) { /* The just-pushed tick is at (s_ring_head - 1). The oldest in @@ -803,19 +810,15 @@ static void heartbeat_write(void) { wedge_kind = 5; /* spin freeze: game wedged while frames advance */ } - if (wedge_kind != 0) { - if (s_dump_armed) { - s_dump_armed = 0; - s_last_wedge_kind = wedge_kind; - freeze_dump_write(wall, frame, cyc, exc_reentry, cur_fn, last_store, - i_stat, i_mask, in_exc, total_checks, - dispatch_count, exc_entries, - sio_stat, sio_ctrl, card_active, - mc_max, tx_writes); - } - } else { - /* Healthy tick: re-arm for the next wedge. */ - s_dump_armed = 1; + if (freeze_dump_policy_observe(&s_dump_policy, wedge_kind, + g_psx_fatal_reason != NULL)) { + s_last_wedge_kind = wedge_kind; + int written = freeze_dump_write( + wall, frame, cyc, exc_reentry, cur_fn, last_store, + i_stat, i_mask, in_exc, total_checks, + dispatch_count, exc_entries, + sio_stat, sio_ctrl, card_active, mc_max, tx_writes); + freeze_dump_policy_record_result(&s_dump_policy, written); } /* Timer1/RootCounter1 decode (Tomba 2 RCnt-wait diagnosis): if the game @@ -915,6 +918,10 @@ static void heartbeat_write(void) { " \"bail_resolved\":%llu,\n" " \"bail_flattened\":%llu,\n" " \"bail_anomaly\":%llu,\n" + " \"automatic_freeze_dumps\":%u,\n" + " \"failed_freeze_dumps\":%u,\n" + " \"suppressed_freeze_events\":%u,\n" + " \"automatic_freeze_dump_limit\":%u,\n" " \"fatal\":%s%s%s\n" "}\n", s_backend, @@ -974,6 +981,10 @@ static void heartbeat_write(void) { (unsigned long long)g_psx_bail_resolved, (unsigned long long)g_psx_bail_flattened, (unsigned long long)g_psx_bail_anomaly, + s_dump_policy.automatic_dumps, + s_dump_policy.failed_dumps, + s_dump_policy.suppressed_events, + (unsigned)FREEZE_DUMP_AUTO_LIMIT, g_psx_fatal_reason ? "\"" : "", g_psx_fatal_reason ? fatal_esc : "null", g_psx_fatal_reason ? "\"" : ""); diff --git a/runtime/tests/test_freeze_dump_policy.c b/runtime/tests/test_freeze_dump_policy.c new file mode 100644 index 00000000..89b0013b --- /dev/null +++ b/runtime/tests/test_freeze_dump_policy.c @@ -0,0 +1,123 @@ +#include "freeze_dump_policy.h" + +#include +#include + +static int failures = 0; + +#define EXPECT_EQ(actual, expected) do { \ + unsigned got_ = (unsigned)(actual); \ + unsigned want_ = (unsigned)(expected); \ + if (got_ != want_) { \ + fprintf(stderr, "%s:%d: got %u, expected %u\n", \ + __FILE__, __LINE__, got_, want_); \ + failures++; \ + } \ +} while (0) + +static int observe(FreezeDumpPolicy *policy, uint32_t kind) { + return freeze_dump_policy_observe(policy, kind, 0); +} + +static void observe_healthy(FreezeDumpPolicy *policy, uint32_t ticks) { + for (uint32_t i = 0; i < ticks; i++) EXPECT_EQ(observe(policy, 0), 0); +} + +int main(void) { + FreezeDumpPolicy policy = {0}; + + /* Authorization does not consume a slot until the file reaches disk. */ + EXPECT_EQ(observe(&policy, 5), 1); + EXPECT_EQ(policy.automatic_dumps, 0); + freeze_dump_policy_record_result(&policy, 1); + EXPECT_EQ(policy.automatic_dumps, 1); + EXPECT_EQ(policy.ordinary_slot_consumed, 1); + EXPECT_EQ(observe(&policy, 5), 0); + + /* A short healthy gap does not turn one oscillating wedge into two. */ + observe_healthy(&policy, FREEZE_DUMP_REARM_HEALTHY_TICKS - 1u); + EXPECT_EQ(observe(&policy, 5), 0); + + /* Lesser severity changes share the ordinary slot. */ + observe_healthy(&policy, FREEZE_DUMP_REARM_HEALTHY_TICKS); + EXPECT_EQ(observe(&policy, 3), 0); + EXPECT_EQ(policy.suppressed_events, 1); + + /* A hard freeze owns the reserved second slot during the same event. */ + EXPECT_EQ(observe(&policy, 1), 1); + freeze_dump_policy_record_result(&policy, 1); + EXPECT_EQ(policy.automatic_dumps, FREEZE_DUMP_AUTO_LIMIT); + EXPECT_EQ(policy.hard_slot_consumed, 1); + EXPECT_EQ(observe(&policy, 1), 0); + + /* Re-entry storms are known events and are counted after the limit. */ + observe_healthy(&policy, FREEZE_DUMP_REARM_HEALTHY_TICKS); + EXPECT_EQ(observe(&policy, 2), 0); + EXPECT_EQ(policy.suppressed_events, 2); + + /* Fatal state bypasses the automatic policy without consuming state. */ + FreezeDumpPolicy fatal = {0}; + EXPECT_EQ(freeze_dump_policy_observe(&fatal, 5, 1), 0); + EXPECT_EQ(fatal.in_wedge, 0); + EXPECT_EQ(fatal.automatic_dumps, 0); + EXPECT_EQ(observe(&fatal, 5), 1); + + /* A failed write keeps its slot for a bounded retry. */ + FreezeDumpPolicy retry = {0}; + EXPECT_EQ(observe(&retry, 5), 1); + freeze_dump_policy_record_result(&retry, 0); + EXPECT_EQ(retry.automatic_dumps, 0); + EXPECT_EQ(retry.failed_dumps, 1); + EXPECT_EQ(observe(&retry, 5), 1); + freeze_dump_policy_record_result(&retry, 1); + EXPECT_EQ(retry.automatic_dumps, 1); + EXPECT_EQ(retry.failed_dumps, 1); + + /* Three failed attempts consume the ordinary slot and stop retries. */ + FreezeDumpPolicy exhausted = {0}; + for (uint32_t i = 0; i < FREEZE_DUMP_MAX_FAILED_ATTEMPTS; i++) { + EXPECT_EQ(observe(&exhausted, 5), 1); + freeze_dump_policy_record_result(&exhausted, 0); + } + EXPECT_EQ(exhausted.failed_dumps, FREEZE_DUMP_MAX_FAILED_ATTEMPTS); + EXPECT_EQ(exhausted.ordinary_slot_consumed, 1); + EXPECT_EQ(observe(&exhausted, 5), 0); + EXPECT_EQ(exhausted.suppressed_events, 1); + + /* An exhausted ordinary slot does not consume the hard-freeze slot. */ + EXPECT_EQ(observe(&exhausted, 1), 1); + freeze_dump_policy_record_result(&exhausted, 1); + EXPECT_EQ(exhausted.hard_slot_consumed, 1); + EXPECT_EQ(exhausted.automatic_dumps, 1); + + /* Unknown future kinds do not consume or suppress an existing slot. */ + FreezeDumpPolicy unknown = {0}; + EXPECT_EQ(observe(&unknown, 7), 0); + EXPECT_EQ(unknown.in_wedge, 0); + EXPECT_EQ(unknown.suppressed_events, 0); + + /* A hard-first event leaves the ordinary evidence slot available. */ + FreezeDumpPolicy hard_first = {0}; + EXPECT_EQ(observe(&hard_first, 1), 1); + freeze_dump_policy_record_result(&hard_first, 1); + observe_healthy(&hard_first, FREEZE_DUMP_REARM_HEALTHY_TICKS); + EXPECT_EQ(observe(&hard_first, 5), 1); + freeze_dump_policy_record_result(&hard_first, 1); + EXPECT_EQ(hard_first.automatic_dumps, FREEZE_DUMP_AUTO_LIMIT); + + /* Equal wall seconds still produce distinct file names. */ + char path0[128], path1[128]; + EXPECT_EQ(freeze_dump_format_path(path0, sizeof(path0), + "psx-runtime", 1234, 0), 1); + EXPECT_EQ(freeze_dump_format_path(path1, sizeof(path1), + "psx-runtime", 1234, 1), 1); + EXPECT_EQ(strcmp(path0, path1) != 0, 1); + EXPECT_EQ(freeze_dump_format_path(path0, 4, "psx-runtime", 1234, 0), 0); + + if (failures != 0) { + fprintf(stderr, "freeze dump policy: %d failure(s)\n", failures); + return 1; + } + puts("freeze dump policy: all checks passed"); + return 0; +}