From a353f9d7aa0368f5f38bf7626b7b2808c8f1f863 Mon Sep 17 00:00:00 2001 From: Dipshet <264011288+Dipshet@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:02:33 +0200 Subject: [PATCH] Write a crash report naming the guest instruction when the game dies Any fatal event now writes one self-contained crash-.txt - guest call stack, registers, a guest-versus-host verdict on the faulting address, modules and the log tail naming rex_sub_* frames with no symbols. --- .gitignore | 6 +- .../include/rex/diag/crash_handler.h | 94 +++ .../rexglue-sdk/include/rex/logging/sink.h | 35 + .../rexglue-sdk/src/native/ui/rex_app.cpp | 43 ++ .../rexglue-sdk/src/system/CMakeLists.txt | 1 + .../src/system/diag_crash_handler.cpp | 721 ++++++++++++++++++ thirdparty/rexglue-sdk/src/system/xthread.cpp | 2 + tools/symbolize_crash.py | 236 ++++++ 8 files changed, 1137 insertions(+), 1 deletion(-) create mode 100644 thirdparty/rexglue-sdk/include/rex/diag/crash_handler.h create mode 100644 thirdparty/rexglue-sdk/src/system/diag_crash_handler.cpp create mode 100644 tools/symbolize_crash.py diff --git a/.gitignore b/.gitignore index cd6741fa..e5c35477 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,8 @@ New Text Document.txt # Local executables and configs ac6recomp.exe ac6recomp.toml -extract-xiso.exe \ No newline at end of file +extract-xiso.exe + +# Python bytecode cache (tools/*.py) +__pycache__/ +*.pyc diff --git a/thirdparty/rexglue-sdk/include/rex/diag/crash_handler.h b/thirdparty/rexglue-sdk/include/rex/diag/crash_handler.h new file mode 100644 index 00000000..512c822e --- /dev/null +++ b/thirdparty/rexglue-sdk/include/rex/diag/crash_handler.h @@ -0,0 +1,94 @@ +/** + * @file rex/diag/crash_handler.h + * @brief Crash reporting to the exact guest instruction, symbol-free at + * runtime. + * + * On any fatal event (unhandled SEH exception, abort/assert_always, + * std::terminate, pure-virtual call, invalid CRT parameter) one + * crash-.txt is written that stands alone: the fault, the guest + * call stack and registers (via the thread-local PPCContext and the PPC + * backchain - no symbols needed), the host stack as raw exe RVAs (decoded + * offline against the reproducible-build PDB), a guest-vs-host fault + * classification, and the embedded session report + log tail. + * + * An atexit hook additionally writes a lighter report when the process exits + * without the orderly shutdown path having run. TerminateProcess and power + * loss are physically uncatchable and leave nothing behind by definition. + * + * This is self-contained: it needs no other diagnostics module. A host that + * has more context to offer (a session report, say) can push it into the + * crash file with SetContextProvider. + * + * The Windows implementation lives in diag_crash_handler.cpp; other + * platforms currently get no-op stubs (a POSIX signal implementation slots + * in behind this same header). + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#pragma once + +#include +#include +#include + +struct PPCFuncMapping; + +namespace rex { +class LogCaptureSink; +} + +namespace rex::diag::crash { + +struct InstallOptions { + /** Where crash-.txt files go (same directory as the report). */ + std::filesystem::path directory; + std::string app_name; + std::string build_title; + std::string build_commit; + std::string build_timestamp; + /** Ties the crash file to the run. Leave empty to have one generated. */ + std::string session_id; +}; + +/** + * Supplies an extra block copied verbatim into the crash file under + * [CONTEXT]. Called ON THE CRASH PATH, so it must not allocate, take locks, + * or touch the filesystem - hand back a pointer to memory that already + * exists. No provider = no section. + */ +using ContextProvider = void (*)(const char** out_data, size_t* out_size); +void SetContextProvider(ContextProvider provider); + +/** Install all fatal-event hooks. Idempotent. Preallocates every buffer the + * crash path needs so the handlers never touch the heap. */ +void Install(const InstallOptions& options); + +/** Guest memory range for fault classification and safe backchain reads. + * Pass the host base of the guest mapping and its extent in bytes. */ +void SetGuestMemoryBounds(const void* host_base, uint64_t extent_bytes); + +/** + * The generated guest->host function table (0-guest-terminated). A copy is + * sorted by host address at call time so a crash can name the guest function + * (rex_sub_XXXXXXXX) containing any host return address with no symbols. + */ +void SetGuestFunctionTable(const PPCFuncMapping* mappings); + +/** Ring-buffer sink whose tail gets embedded into crash reports (best + * effort: skipped without blocking if its lock is held). */ +void SetLogTailSink(rex::LogCaptureSink* sink); + +/** Reserve handler stack space on the calling thread so a stack-overflow + * report can still be written. Call once per guest thread. */ +void PrepareCurrentThread(); + +/** The orderly shutdown path ran; the atexit unexpected-exit report is + * suppressed. */ +void NotifyOrderlyShutdown(); + +} // namespace rex::diag::crash diff --git a/thirdparty/rexglue-sdk/include/rex/logging/sink.h b/thirdparty/rexglue-sdk/include/rex/logging/sink.h index 7032b77b..4b254218 100644 --- a/thirdparty/rexglue-sdk/include/rex/logging/sink.h +++ b/thirdparty/rexglue-sdk/include/rex/logging/sink.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -60,6 +61,40 @@ class LogCaptureSink : public spdlog::sinks::base_sink { return generation_; } + // Crash-path tail copy: appends up to `max_lines` of the newest entries + // (oldest first) into `out`, one line each, without blocking - if either + // mutex is held by the crashed thread, returns 0 rather than deadlocking. + // No allocation: writes into the caller's buffer only. + size_t CopyTailForCrash(char* out, size_t out_size, size_t max_lines) { + if (!mutable_mutex_.try_lock()) + return 0; + if (!base_sink::mutex_.try_lock()) { + mutable_mutex_.unlock(); + return 0; + } + size_t written = 0; + size_t available = count_; + size_t take = available < max_lines ? available : max_lines; + for (size_t i = available - take; i < available; ++i) { + const LogEntry& entry = + (count_ < kCapacity) ? buf_[i] : buf_[(write_pos_ + i) % kCapacity]; + size_t need = entry.category.size() + entry.text.size() + 4; + if (written + need + 1 >= out_size) + break; + out[written++] = '['; + std::memcpy(out + written, entry.category.data(), entry.category.size()); + written += entry.category.size(); + out[written++] = ']'; + out[written++] = ' '; + std::memcpy(out + written, entry.text.data(), entry.text.size()); + written += entry.text.size(); + out[written++] = '\n'; + } + base_sink::mutex_.unlock(); + mutable_mutex_.unlock(); + return written; + } + protected: void sink_it_(const spdlog::details::log_msg& msg) override { std::string cat(msg.logger_name.begin(), msg.logger_name.end()); diff --git a/thirdparty/rexglue-sdk/src/native/ui/rex_app.cpp b/thirdparty/rexglue-sdk/src/native/ui/rex_app.cpp index c7a4ba05..e54812ca 100644 --- a/thirdparty/rexglue-sdk/src/native/ui/rex_app.cpp +++ b/thirdparty/rexglue-sdk/src/native/ui/rex_app.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -389,6 +390,32 @@ bool ReXApp::OnInitialize() { if (REXCVAR_GET(log_verbose) && log_level_str == "info") { log_level_str = "trace"; } + // Crash reporting, installed before anything heavy runs so a fault during + // setup still produces a file. Crash files live beside the log, which is + // where a user already looks and what a bug report already asks for. + { + std::filesystem::path crash_dir; + if (log_file_cvar.empty()) { + crash_dir = exe_dir / "logs"; + } else { + std::error_code crash_dir_ec; + crash_dir = std::filesystem::absolute(std::filesystem::path(log_file_cvar), crash_dir_ec) + .parent_path(); + if (crash_dir.empty()) + crash_dir = std::filesystem::current_path(); + } + std::error_code crash_dir_ec; + std::filesystem::create_directories(crash_dir, crash_dir_ec); + + rex::diag::crash::InstallOptions crash_options; + crash_options.directory = crash_dir; + crash_options.app_name = std::string(GetName()); + crash_options.build_title = REXGLUE_BUILD_TITLE; + crash_options.build_commit = REXGLUE_GIT_HASH; + crash_options.build_timestamp = REXGLUE_BUILD_TIMESTAMP; + rex::diag::crash::Install(crash_options); + } + auto category_levels = rex::ParseCategoryLevelsFromConfig(config_path); auto log_config = rex::BuildLogConfig(log_file_cvar.empty() ? nullptr : log_file_cvar.c_str(), log_level_str, category_levels); @@ -402,6 +429,10 @@ bool ReXApp::OnInitialize() { // Attach log capture sink to all loggers for the console overlay log_sink_ = std::make_shared(); rex::AddSink(log_sink_); + // The crash file embeds this ring buffer's tail: the log itself is replaced + // on the next launch, so the lines around a fault would otherwise be gone + // by the time anyone looks. + rex::diag::crash::SetLogTailSink(log_sink_.get()); if (std::filesystem::exists(config_path)) { REXLOG_INFO("Loaded config: {}", config_path.filename().string()); } @@ -449,6 +480,14 @@ bool ReXApp::OnInitialize() { return false; } + // Guest-side crash reporting: the memory bounds classify a faulting address + // as guest vs host, and the generated function table lets a crash name + // rex_sub_* frames at runtime with no symbols shipped. + if (runtime_->memory()) { + rex::diag::crash::SetGuestMemoryBounds(runtime_->memory()->virtual_membase(), 0x11FFFFFFFull); + } + rex::diag::crash::SetGuestFunctionTable(ppc_info_.func_mappings); + std::string xex_image = "game:\\default.xex"; // Allow subclass to override xex image @@ -620,6 +659,10 @@ void ReXApp::OnClosing(ui::UIEvent& e) { } void ReXApp::OnDestroy() { + // The shutdown path is running, so the atexit hook must not report this + // exit as unexpected. + rex::diag::crash::NotifyOrderlyShutdown(); + // Notify subclass before cleanup OnShutdown(); diff --git a/thirdparty/rexglue-sdk/src/system/CMakeLists.txt b/thirdparty/rexglue-sdk/src/system/CMakeLists.txt index 3bf813fd..35b1727e 100644 --- a/thirdparty/rexglue-sdk/src/system/CMakeLists.txt +++ b/thirdparty/rexglue-sdk/src/system/CMakeLists.txt @@ -35,6 +35,7 @@ set(REXSYSTEM_SOURCES xam/user_profile.cpp # Formerly rexruntime: module loading, CPU execution, thread state + diag_crash_handler.cpp ppc_types.cpp elf_module.cpp entry_table.cpp diff --git a/thirdparty/rexglue-sdk/src/system/diag_crash_handler.cpp b/thirdparty/rexglue-sdk/src/system/diag_crash_handler.cpp new file mode 100644 index 00000000..49e04159 --- /dev/null +++ b/thirdparty/rexglue-sdk/src/system/diag_crash_handler.cpp @@ -0,0 +1,721 @@ +/** + * @file system/diag_crash_handler.cpp + * @brief Crash reporting implementation (Windows; POSIX stubs below). + * + * Everything on the crash path follows the broken-process rules: all buffers + * are preallocated at Install, formatting uses fmt::format_to_n into those + * buffers (no heap), files are written with raw Win32 APIs, no locks are + * taken (the log tail uses try_lock and is skipped if held), a re-entry + * guard turns a fault inside the handler into an immediate TerminateProcess, + * and the file is written incrementally most-valuable-first so a handler + * that dies partway still leaves the useful part on disk. + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#if REX_PLATFORM_WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +#include + +namespace rex::diag::crash { + +namespace { + +constexpr size_t kScratchSize = 256 * 1024; +constexpr size_t kLogTailLines = 200; +constexpr int kMaxGuestFrames = 32; +constexpr int kMaxHostFrames = 48; + +struct HostFunctionEntry { + uintptr_t host; + uint32_t guest; +}; + +struct CrashState { + bool installed = false; + std::atomic in_handler{false}; + std::atomic report_written{false}; + std::atomic orderly_shutdown{false}; + + // Fixed at install; used verbatim by the handler. + wchar_t directory[MAX_PATH] = {}; + char app_name[64] = {}; + char build_title[128] = {}; + char build_commit[64] = {}; + char build_timestamp[64] = {}; + char session_id[64] = {}; + + // Guest memory bounds for classification + safe backchain reads. + std::atomic guest_base{0}; + std::atomic guest_extent{0}; + + // Guest function table sorted by host address (built at registration, + // never touched again). Lookup at crash time is a binary search. + std::vector functions_by_host; + + rex::LogCaptureSink* log_tail_sink = nullptr; + ContextProvider context_provider = nullptr; + + uintptr_t exe_base = 0; + uintptr_t exe_size = 0; + + // The crash path's only working memory. + char scratch[kScratchSize]; + char log_tail[64 * 1024]; + +}; + +CrashState& S() { + static CrashState state; + return state; +} + +using GetThreadDescriptionFn = HRESULT(WINAPI*)(HANDLE, PWSTR*); + +// --------------------------------------------------------------------------- +// Crash-safe append formatting into the scratch buffer. + +struct Writer { + char* data; + size_t capacity; + size_t size = 0; + + void Append(const char* text, size_t len) { + if (size + len >= capacity) + len = capacity - size - 1; + std::memcpy(data + size, text, len); + size += len; + } + + void Append(const char* text) { Append(text, std::strlen(text)); } + + template + void Format(fmt::format_string format, Args&&... args) { + auto result = fmt::format_to_n(data + size, capacity - size - 1, format, + std::forward(args)...); + size += (std::min)(result.size, capacity - size - 1); + } + + void Clear() { size = 0; } +}; + +// --------------------------------------------------------------------------- +// File output: raw Win32, incremental, flushed after every section. + +HANDLE OpenCrashFile() { + auto& s = S(); + SYSTEMTIME time; + GetLocalTime(&time); + wchar_t path[MAX_PATH + 64]; + int dir_length = 0; + while (s.directory[dir_length] && dir_length < MAX_PATH) + ++dir_length; + std::memcpy(path, s.directory, dir_length * sizeof(wchar_t)); + wchar_t name[64]; + int name_length = + _snwprintf_s(name, _countof(name), _TRUNCATE, L"\\crash-%04u%02u%02u-%02u%02u%02u.txt", + time.wYear, time.wMonth, time.wDay, time.wHour, time.wMinute, time.wSecond); + if (name_length <= 0) + return INVALID_HANDLE_VALUE; + std::memcpy(path + dir_length, name, (name_length + 1) * sizeof(wchar_t)); + return CreateFileW(path, GENERIC_WRITE, FILE_SHARE_READ, nullptr, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); +} + +void WriteAndFlush(HANDLE file, Writer& writer) { + if (file == INVALID_HANDLE_VALUE || writer.size == 0) + return; + DWORD written = 0; + WriteFile(file, writer.data, static_cast(writer.size), &written, nullptr); + FlushFileBuffers(file); + writer.Clear(); +} + +// --------------------------------------------------------------------------- +// Guest-side: safe reads, function naming, backchain walk. + +bool GuestAddressInRange(uintptr_t host_address) { + auto& s = S(); + uintptr_t base = s.guest_base.load(std::memory_order_relaxed); + uint64_t extent = s.guest_extent.load(std::memory_order_relaxed); + return base && host_address >= base && host_address < base + extent; +} + +// Read one big-endian u32 from guest memory without faulting: the guest map +// is reserved but only partially committed, so every read is probed first. +bool SafeReadGuestU32(uint32_t guest_address, uint32_t* out) { + auto& s = S(); + uintptr_t base = s.guest_base.load(std::memory_order_relaxed); + uint64_t extent = s.guest_extent.load(std::memory_order_relaxed); + if (!base || guest_address + 4ull > extent) + return false; + const void* host = reinterpret_cast(base + guest_address); + MEMORY_BASIC_INFORMATION info; + if (!VirtualQuery(host, &info, sizeof(info)) || info.State != MEM_COMMIT || + (info.Protect & (PAGE_NOACCESS | PAGE_GUARD))) { + return false; + } + uint32_t value; + std::memcpy(&value, host, 4); + *out = _byteswap_ulong(value); + return true; +} + +// Host code address -> the generated function containing it, via the +// host-sorted table. Returns nullptr when the address is not inside +// generated code. The caller needs both the guest address and the function +// start (for the +offset), so hand back the entry rather than looking it up +// twice. +const HostFunctionEntry* GuestFunctionForHostAddress(uintptr_t host_address) { + const auto& table = S().functions_by_host; + if (table.empty()) + return nullptr; + size_t lo = 0, hi = table.size(); + while (lo < hi) { + size_t mid = (lo + hi) / 2; + if (table[mid].host <= host_address) + lo = mid + 1; + else + hi = mid; + } + if (lo == 0) + return nullptr; + const HostFunctionEntry* entry = &table[lo - 1]; + // Generated functions are well under 1 MB; a hit further away than that + // means the address is not inside generated code at all. + if (host_address - entry->host > (1u << 20)) + return nullptr; + return entry; +} + +void AppendHostFrame(Writer& writer, uintptr_t pc) { + auto& s = S(); + if (!s.exe_base || pc < s.exe_base || (s.exe_size && pc >= s.exe_base + s.exe_size)) { + writer.Format(" 0x{:016X} (system module)\n", pc); + return; + } + if (const HostFunctionEntry* entry = GuestFunctionForHostAddress(pc)) { + writer.Format(" exe+0x{:08X} rex_sub_{:08X}+0x{:X}\n", pc - s.exe_base, + entry->guest, pc - entry->host); + } else { + writer.Format(" exe+0x{:08X} (host code)\n", pc - s.exe_base); + } +} + +// The guest call stack straight from guest memory: 0(r1) is the caller's +// stack pointer, and the current function's saved LR sits 8 bytes below the +// caller's frame base (generated prologues do `stw r12,-8(r1)` before +// `stwu r1,-N(r1)`). Needs no symbols and no unwind tables. +void AppendGuestBackchain(Writer& writer, const PPCContext* context) { + uint32_t stack_pointer = context->r1.u32; + writer.Format(" lr 0x{:08X} (return address of the innermost frame)\n", + static_cast(context->lr)); + for (int frame = 0; frame < kMaxGuestFrames; ++frame) { + uint32_t caller_sp = 0; + if (!SafeReadGuestU32(stack_pointer, &caller_sp) || caller_sp == 0 || + caller_sp <= stack_pointer) { + writer.Format(" (backchain ends at sp 0x{:08X})\n", stack_pointer); + break; + } + uint32_t saved_lr = 0; + if (SafeReadGuestU32(caller_sp - 8, &saved_lr) && saved_lr) { + writer.Format(" sp 0x{:08X} return 0x{:08X}\n", caller_sp, saved_lr); + } else { + writer.Format(" sp 0x{:08X} (no readable saved lr)\n", caller_sp); + } + stack_pointer = caller_sp; + } +} + +void AppendGuestRegisters(Writer& writer, const PPCContext* context) { + const PPCRegister* gprs[32] = { + &context->r0, &context->r1, &context->r2, &context->r3, &context->r4, + &context->r5, &context->r6, &context->r7, &context->r8, &context->r9, + &context->r10, &context->r11, &context->r12, &context->r13, &context->r14, + &context->r15, &context->r16, &context->r17, &context->r18, &context->r19, + &context->r20, &context->r21, &context->r22, &context->r23, &context->r24, + &context->r25, &context->r26, &context->r27, &context->r28, &context->r29, + &context->r30, &context->r31}; + for (int i = 0; i < 32; i += 4) { + writer.Format(" r{:<2} {:016X} r{:<2} {:016X} r{:<2} {:016X} r{:<2} {:016X}\n", i, + gprs[i]->u64, i + 1, gprs[i + 1]->u64, i + 2, gprs[i + 2]->u64, i + 3, + gprs[i + 3]->u64); + } + uint32_t cr = (context->cr0.raw() << 28) | (context->cr1.raw() << 24) | + (context->cr2.raw() << 20) | (context->cr3.raw() << 16) | + (context->cr4.raw() << 12) | (context->cr5.raw() << 8) | + (context->cr6.raw() << 4) | context->cr7.raw(); + writer.Format(" lr {:016X} ctr {:016X} cr {:08X} xer so={} ov={} ca={}\n", context->lr, + context->ctr.u64, cr, context->xer.so, context->xer.ov, context->xer.ca); +} + +// --------------------------------------------------------------------------- +// Host-side stack walk from a CONTEXT, no dbghelp: RtlLookupFunctionEntry + +// RtlVirtualUnwind are exported by ntdll/kernel32 and safe here. + +void AppendHostStack(Writer& writer, CONTEXT* context) { + CONTEXT unwind_context = *context; + for (int frame = 0; frame < kMaxHostFrames && unwind_context.Rip; ++frame) { + AppendHostFrame(writer, static_cast(unwind_context.Rip)); + DWORD64 image_base = 0; + PRUNTIME_FUNCTION function_entry = + RtlLookupFunctionEntry(unwind_context.Rip, &image_base, nullptr); + if (!function_entry) { + // Leaf function: return address is at RSP. + unwind_context.Rip = *reinterpret_cast(unwind_context.Rsp); + unwind_context.Rsp += 8; + continue; + } + PVOID handler_data = nullptr; + DWORD64 establisher_frame = 0; + RtlVirtualUnwind(UNW_FLAG_NHANDLER, image_base, unwind_context.Rip, function_entry, + &unwind_context, &handler_data, &establisher_frame, nullptr); + } +} + +const char* ExceptionName(DWORD code) { + switch (code) { + case EXCEPTION_ACCESS_VIOLATION: + return "ACCESS_VIOLATION"; + case EXCEPTION_STACK_OVERFLOW: + return "STACK_OVERFLOW"; + case EXCEPTION_ILLEGAL_INSTRUCTION: + return "ILLEGAL_INSTRUCTION"; + case EXCEPTION_INT_DIVIDE_BY_ZERO: + return "INT_DIVIDE_BY_ZERO"; + case EXCEPTION_PRIV_INSTRUCTION: + return "PRIV_INSTRUCTION"; + case EXCEPTION_IN_PAGE_ERROR: + return "IN_PAGE_ERROR"; + case EXCEPTION_DATATYPE_MISALIGNMENT: + return "DATATYPE_MISALIGNMENT"; + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + return "FLT_DIVIDE_BY_ZERO"; + case EXCEPTION_FLT_INVALID_OPERATION: + return "FLT_INVALID_OPERATION"; + default: + return "EXCEPTION"; + } +} + +// --------------------------------------------------------------------------- +// The report builder. `kind` names the trigger; `pointers` is null for +// non-exception triggers (abort/terminate/...), in which case the current +// context is captured for the host walk. + +void WriteCrashReport(const char* kind, EXCEPTION_POINTERS* pointers, const char* extra) { + auto& s = S(); + if (s.report_written.exchange(true, std::memory_order_acq_rel)) + return; + + HANDLE file = OpenCrashFile(); + Writer writer{s.scratch, kScratchSize}; + + // --- Header: crash nature + build identity. The single most important + // block - everything below it is best-effort. + writer.Format("=== {} CRASH REPORT ===\n", s.app_name); + writer.Format("Session {}\n", s.session_id); + writer.Format("Build {} commit {} built {}\n", s.build_title, s.build_commit, + s.build_timestamp); + writer.Format("Trigger {}\n", kind); + if (extra && extra[0]) + writer.Format("Detail {}\n", extra); + + uintptr_t fault_address = 0; + bool is_access_violation = false; + const char* access_kind = ""; + if (pointers) { + DWORD code = pointers->ExceptionRecord->ExceptionCode; + uintptr_t exception_address = + reinterpret_cast(pointers->ExceptionRecord->ExceptionAddress); + if (s.exe_base && exception_address >= s.exe_base && + (!s.exe_size || exception_address < s.exe_base + s.exe_size)) { + writer.Format("Exception {} (0x{:08X}) at exe+0x{:X}\n", ExceptionName(code), code, + exception_address - s.exe_base); + } else { + writer.Format("Exception {} (0x{:08X}) at 0x{:016X} (outside exe)\n", + ExceptionName(code), code, exception_address); + } + if ((code == EXCEPTION_ACCESS_VIOLATION || code == EXCEPTION_IN_PAGE_ERROR) && + pointers->ExceptionRecord->NumberParameters >= 2) { + is_access_violation = true; + fault_address = static_cast(pointers->ExceptionRecord->ExceptionInformation[1]); + switch (pointers->ExceptionRecord->ExceptionInformation[0]) { + case 0: + access_kind = "reading"; + break; + case 1: + access_kind = "writing"; + break; + case 8: + access_kind = "executing"; + break; + default: + access_kind = "accessing"; + break; + } + writer.Format("Fault {} address 0x{:016X}\n", access_kind, fault_address); + } + } + + // --- Fault classification: the one line that routes a report. + if (is_access_violation) { + uintptr_t base = s.guest_base.load(std::memory_order_relaxed); + if (GuestAddressInRange(fault_address)) { + writer.Format( + "Classification: GUEST pointer - the faulting address is guest 0x{:08X} inside " + "the guest memory map (a guest-side pointer bug or corrupted guest state)\n", + static_cast(fault_address - base)); + } else if (base && fault_address < 0x10000) { + writer.Append( + "Classification: null/near-null HOST pointer - our code, not guest data\n"); + } else { + writer.Append("Classification: HOST address - our code, not a guest pointer\n"); + } + } + + // --- Thread identity. + writer.Format("Thread id {}", GetCurrentThreadId()); + { + HMODULE kernel32 = GetModuleHandleW(L"kernel32.dll"); + auto get_description = kernel32 ? reinterpret_cast( + GetProcAddress(kernel32, "GetThreadDescription")) + : nullptr; + PWSTR description = nullptr; + if (get_description && SUCCEEDED(get_description(GetCurrentThread(), &description)) && + description) { + char narrow[128]; + int i = 0; + for (; i < 127 && description[i]; ++i) + narrow[i] = description[i] < 128 ? static_cast(description[i]) : '?'; + narrow[i] = 0; + writer.Format(" \"{}\"", narrow); + LocalFree(description); + } + } + writer.Append("\n"); + WriteAndFlush(file, writer); + + // --- Guest context: registers + backchain, straight from the thread-local + // PPCContext. No symbols involved. + auto* thread_state = rex::runtime::ThreadState::Get(); + PPCContext* guest_context = thread_state ? thread_state->context() : nullptr; + if (guest_context) { + writer.Append("\n[GUEST STACK]\n"); + writer.Format(" Innermost guest function: walk the host stack below; ctx.lr 0x{:08X} " + "is the last guest return address taken\n", + static_cast(guest_context->lr)); + AppendGuestBackchain(writer, guest_context); + writer.Append("\n[GUEST REGISTERS]\n"); + AppendGuestRegisters(writer, guest_context); + } else { + writer.Append("\n[GUEST STACK]\nno guest context bound to this thread (host-only " + "thread)\n"); + } + WriteAndFlush(file, writer); + + // --- Host stack: raw RVAs for the offline symbolizer, with guest function + // names attached where frames land inside generated code. + writer.Append("\n[HOST STACK] (exe+RVA; decode offline: tools/symbolize_crash.py)\n"); + if (pointers) { + CONTEXT context_copy = *pointers->ContextRecord; + AppendHostStack(writer, &context_copy); + } else { + CONTEXT context = {}; + context.ContextFlags = CONTEXT_FULL; + RtlCaptureContext(&context); + AppendHostStack(writer, &context); + } + WriteAndFlush(file, writer); + + // --- Memory state. + writer.Append("\n[MEMORY]\n"); + { + MEMORYSTATUSEX memory_status = {}; + memory_status.dwLength = sizeof(memory_status); + if (GlobalMemoryStatusEx(&memory_status)) { + writer.Format(" System: {} MB physical, {} MB available, load {}%\n", + memory_status.ullTotalPhys >> 20, memory_status.ullAvailPhys >> 20, + memory_status.dwMemoryLoad); + } + PROCESS_MEMORY_COUNTERS counters = {}; + counters.cb = sizeof(counters); + if (K32GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters))) { + writer.Format(" Process: working set {} MB (peak {} MB), pagefile {} MB\n", + counters.WorkingSetSize >> 20, counters.PeakWorkingSetSize >> 20, + counters.PagefileUsage >> 20); + } + uintptr_t base = s.guest_base.load(std::memory_order_relaxed); + if (base) { + writer.Format(" Guest map: host base 0x{:016X}, extent {} MB\n", base, + s.guest_extent.load(std::memory_order_relaxed) >> 20); + } + } + + // --- Loaded modules: injected overlays/capture layers are a real crash + // source (renderdoc.dll has form here). + writer.Append("\n[MODULES]\n"); + { + HMODULE modules[128]; + DWORD needed = 0; + if (K32EnumProcessModules(GetCurrentProcess(), modules, sizeof(modules), &needed)) { + DWORD count = (std::min)(needed / static_cast(sizeof(HMODULE)), + static_cast(_countof(modules))); + for (DWORD i = 0; i < count; ++i) { + char module_name[MAX_PATH]; + if (K32GetModuleFileNameExA(GetCurrentProcess(), modules[i], module_name, + sizeof(module_name))) { + const char* base_name = std::strrchr(module_name, '\\'); + writer.Format(" 0x{:016X} {}\n", reinterpret_cast(modules[i]), + base_name ? base_name + 1 : module_name); + } + } + } + } + WriteAndFlush(file, writer); + + // --- Optional embedded context. The crash file must answer as much as it + // can ALONE, because the logs are replaced on the next launch. Anything + // that registered a provider (a session report, say) is copied in verbatim + // here; with no provider the section is simply absent. + if (s.context_provider) { + const char* context_data = nullptr; + size_t context_size = 0; + s.context_provider(&context_data, &context_size); + if (context_data && context_size) { + writer.Append("\n[CONTEXT]\n"); + WriteAndFlush(file, writer); + DWORD written = 0; + WriteFile(file, context_data, static_cast(context_size), &written, nullptr); + } + } + + // --- Log tail (best effort - never blocks on a held lock). + writer.Append("\n[LOG TAIL]\n"); + WriteAndFlush(file, writer); + if (s.log_tail_sink) { + size_t tail_size = + s.log_tail_sink->CopyTailForCrash(s.log_tail, sizeof(s.log_tail), kLogTailLines); + if (tail_size) { + DWORD written = 0; + WriteFile(file, s.log_tail, static_cast(tail_size), &written, nullptr); + } else { + writer.Append("(log tail unavailable - its lock was held at crash time)\n"); + WriteAndFlush(file, writer); + } + } else { + writer.Append("(no log capture sink registered)\n"); + WriteAndFlush(file, writer); + } + + writer.Append("\n=== END OF CRASH REPORT ===\n"); + WriteAndFlush(file, writer); + if (file != INVALID_HANDLE_VALUE) + CloseHandle(file); +} + +// Guarded entry: any fault inside the crash path terminates immediately +// instead of looping. +void HandleFatalEvent(const char* kind, EXCEPTION_POINTERS* pointers, const char* extra) { + auto& s = S(); + if (s.in_handler.exchange(true, std::memory_order_acq_rel)) { + TerminateProcess(GetCurrentProcess(), 0xC0DEDEAD); + } + __try { + WriteCrashReport(kind, pointers, extra); + } __except (EXCEPTION_EXECUTE_HANDLER) { + // The handler itself died; the incremental flushes preserved whatever + // was written before this point. + } + s.in_handler.store(false, std::memory_order_release); +} + +// --------------------------------------------------------------------------- +// The hooks. + +LONG WINAPI UnhandledFilter(EXCEPTION_POINTERS* pointers) { + HandleFatalEvent("unhandled exception", pointers, nullptr); + return EXCEPTION_EXECUTE_HANDLER; +} + +void AbortSignalHandler(int) { + HandleFatalEvent("abort (assert_always / rex_assert_fail / CRT abort)", nullptr, nullptr); + _exit(3); +} + +void TerminateHandler() { + HandleFatalEvent("std::terminate (unhandled C++ exception or noexcept violation)", nullptr, + nullptr); + _exit(3); +} + +void PurecallHandler() { + HandleFatalEvent("pure virtual call", nullptr, nullptr); + _exit(3); +} + +void InvalidParameterHandler(const wchar_t*, const wchar_t*, const wchar_t*, unsigned int, + uintptr_t) { + // The CRT passes details only in debug builds; the stacks tell the story. + HandleFatalEvent("invalid CRT parameter", nullptr, nullptr); + _exit(3); +} + +void AtExitHook() { + auto& s = S(); + if (s.orderly_shutdown.load(std::memory_order_acquire) || + s.report_written.load(std::memory_order_acquire)) { + return; + } + // exit()/ExitProcess without the shutdown path: no fault context, but the + // exiting thread's stack usually names the caller. + HandleFatalEvent("unexpected exit (exit()/ExitProcess without the shutdown path)", nullptr, + nullptr); +} + +} // namespace + +void Install(const InstallOptions& options) { + auto& s = S(); + if (s.installed) + return; + + auto copy_string = [](char* dst, size_t cap, const std::string& src) { + size_t n = (std::min)(src.size(), cap - 1); + std::memcpy(dst, src.data(), n); + dst[n] = 0; + }; + copy_string(s.app_name, sizeof(s.app_name), options.app_name); + copy_string(s.build_title, sizeof(s.build_title), options.build_title); + copy_string(s.build_commit, sizeof(s.build_commit), options.build_commit); + copy_string(s.build_timestamp, sizeof(s.build_timestamp), options.build_timestamp); + // A session id ties a crash file to the run that produced it. A caller that + // already mints one passes it in; otherwise generate the same shape here so + // the field is never empty. + if (!options.session_id.empty()) { + copy_string(s.session_id, sizeof(s.session_id), options.session_id); + } else { + SYSTEMTIME now; + GetLocalTime(&now); + _snprintf_s(s.session_id, sizeof(s.session_id), _TRUNCATE, "%04u%02u%02u-%02u%02u%02u-%lu", + now.wYear, now.wMonth, now.wDay, now.wHour, now.wMinute, now.wSecond, + GetCurrentProcessId()); + } + { + auto wide = options.directory.wstring(); + size_t n = (std::min)(wide.size(), static_cast(MAX_PATH - 1)); + std::memcpy(s.directory, wide.data(), n * sizeof(wchar_t)); + s.directory[n] = 0; + } + s.exe_base = reinterpret_cast(GetModuleHandleW(nullptr)); + { + MODULEINFO module_info = {}; + if (K32GetModuleInformation(GetCurrentProcess(), + reinterpret_cast(s.exe_base), &module_info, + sizeof(module_info))) { + s.exe_size = module_info.SizeOfImage; + } + } + + SetUnhandledExceptionFilter(UnhandledFilter); + std::signal(SIGABRT, AbortSignalHandler); + std::set_terminate(TerminateHandler); + _set_purecall_handler(PurecallHandler); + _set_invalid_parameter_handler(InvalidParameterHandler); + std::atexit(AtExitHook); + PrepareCurrentThread(); + + s.installed = true; +} + +void SetGuestMemoryBounds(const void* host_base, uint64_t extent_bytes) { + auto& s = S(); + s.guest_base.store(reinterpret_cast(host_base), std::memory_order_relaxed); + s.guest_extent.store(extent_bytes, std::memory_order_relaxed); +} + +void SetGuestFunctionTable(const PPCFuncMapping* mappings) { + auto& s = S(); + if (!mappings) + return; + std::vector table; + for (const PPCFuncMapping* m = mappings; m->guest; ++m) { + if (m->host) { + table.push_back({reinterpret_cast(m->host), + static_cast(m->guest)}); + } + } + std::sort(table.begin(), table.end(), + [](const HostFunctionEntry& a, const HostFunctionEntry& b) { return a.host < b.host; }); + s.functions_by_host = std::move(table); +} + +void SetLogTailSink(rex::LogCaptureSink* sink) { + S().log_tail_sink = sink; +} + +void SetContextProvider(ContextProvider provider) { + S().context_provider = provider; +} + +void PrepareCurrentThread() { + // Reserve stack for the handler so a stack-overflow report can be written. + ULONG stack_size = 64 * 1024; + SetThreadStackGuarantee(&stack_size); +} + +void NotifyOrderlyShutdown() { + S().orderly_shutdown.store(true, std::memory_order_release); +} + +} // namespace rex::diag::crash + +#else // !REX_PLATFORM_WIN32 + +// POSIX: not implemented yet. The seam is this file - a signal-based +// implementation (SIGSEGV/SIGABRT + sigaltstack) drops in behind the same +// header; the guest side (PPCContext + backchain) is already +// platform-neutral. + +namespace rex::diag::crash { + +void Install(const InstallOptions&) {} +void SetGuestMemoryBounds(const void*, uint64_t) {} +void SetGuestFunctionTable(const PPCFuncMapping*) {} +void SetLogTailSink(rex::LogCaptureSink*) {} +void SetContextProvider(ContextProvider) {} +void PrepareCurrentThread() {} +void NotifyOrderlyShutdown() {} + +} // namespace rex::diag::crash + +#endif // REX_PLATFORM_WIN32 diff --git a/thirdparty/rexglue-sdk/src/system/xthread.cpp b/thirdparty/rexglue-sdk/src/system/xthread.cpp index f64620e4..f6de1d47 100644 --- a/thirdparty/rexglue-sdk/src/system/xthread.cpp +++ b/thirdparty/rexglue-sdk/src/system/xthread.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -321,6 +322,7 @@ X_STATUS XThread::Create() { params.create_suspended = true; thread_ = rex::thread::Thread::Create(params, [this]() { rex::initialize_seh_thread(); + rex::diag::crash::PrepareCurrentThread(); runtime::ThreadState::Bind(thread_state_.get()); // Set thread ID override. This is used by logging. diff --git a/tools/symbolize_crash.py b/tools/symbolize_crash.py new file mode 100644 index 00000000..a530ca95 --- /dev/null +++ b/tools/symbolize_crash.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Decode an ac6recomp crash report to exact guest instructions, offline. + +The crash handler ships raw host RVAs (exe+0x...) because users don't have +the PDB. This tool runs where the PDB exists (a dev machine; the +reproducible build regenerates it byte-identical for any released commit) +and rewrites each host frame as: + + exe+0x00123456 rex_sub_8209ABCD generated/ac6recomp_recomp.7.cpp:1234 + guest 0x8209ABF0: lwz r11,0(r3) <- the exact guest instruction + +Usage: + python tools/symbolize_crash.py + [--exe out/build/win-amd64-relwithdebinfo/ac6recomp.exe] + [--generated generated] [--force] + +Requires llvm-symbolizer (and llvm-readobj for the PDB match check) on PATH +or in C:/Program Files/LLVM/bin. The PDB must MATCH the exe (same build): +the tool compares the exe's RSDS debug-directory GUID against the PDB's and +refuses to print symbols from a mismatched pair unless --force is given - +wrong symbols are worse than none. +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys + +LLVM_DIRS = [r"C:\Program Files\LLVM\bin", "/usr/bin", "/usr/local/bin"] + + +def find_tool(name): + path = shutil.which(name) + if path: + return path + for d in LLVM_DIRS: + candidate = os.path.join(d, name + (".exe" if os.name == "nt" else "")) + if os.path.isfile(candidate): + return candidate + return None + + +def pe_pdb_guid(readobj, exe): + """The exe's RSDS GUID+age and referenced PDB path, via llvm-readobj.""" + out = subprocess.run([readobj, "--coff-debug-directory", exe], + capture_output=True, text=True).stdout + guid = re.search(r"PDBGUID:\s*\(([^)]*)\)", out) + age = re.search(r"PDBAge:\s*(\d+)", out) + pdb = re.search(r"PDBFileName:\s*(.+)", out) + if not guid: + # Older llvm-readobj prints "GUID: {...}" + guid = re.search(r"GUID:\s*\{?([0-9A-Fa-f-]+)\}?", out) + return (guid.group(1).strip() if guid else None, + age.group(1) if age else None, + pdb.group(1).strip() if pdb else None) + + +def pdb_guid(pdbutil, pdb_path): + out = subprocess.run([pdbutil, "dump", "--summary", pdb_path], + capture_output=True, text=True).stdout + guid = re.search(r"GUID:\s*\{?([0-9A-Fa-f-]+)\}?", out) + age = re.search(r"Age:\s*(\d+)", out) + return (guid.group(1).strip() if guid else None, age.group(1) if age else None) + + +def normalize_guid(g): + if not g: + return None + return re.sub(r"[^0-9A-F]", "", g.upper()) + + +def crash_build_identity(crash_path): + """The 'Build ... commit built ' line the handler writes.""" + with open(crash_path, "r", errors="replace") as f: + for _ in range(40): + line = f.readline() + if not line: + break + m = re.match(r"Build\s+(\S+)\s+commit\s+(\S+)\s+built\s+(\S+)", line.strip()) + if m: + return m.group(2), m.group(3) + return None, None + + +def exe_contains_tokens(exe_path, tokens): + """Are these build strings compiled into this exe? They come from + version.h, so a binary from another build carries different ones.""" + try: + with open(exe_path, "rb") as f: + blob = f.read() + except OSError: + return None + return all(t.encode("ascii", "ignore") in blob for t in tokens if t) + + +MNEMONIC_RE = re.compile(r"^\s*//\s+(\S.*)$") +LABEL_RE = re.compile(r"^\s*loc_([0-9A-Fa-f]{8}):") +LR_RE = re.compile(r"ctx\.lr\s*=\s*0x([0-9A-Fa-f]{8})") + + +def guest_instruction_at(generated_root, rel_file, line_number): + """The PPC mnemonic comment directly above `line_number`, plus the + nearest preceding guest-address anchor (loc_ label or ctx.lr store).""" + path = rel_file + if not os.path.isabs(path): + path = os.path.join(generated_root, os.path.basename(rel_file)) + if not os.path.isfile(path): + return None, None + try: + with open(path, "r", errors="replace") as f: + lines = f.readlines() + except OSError: + return None, None + if line_number < 1 or line_number > len(lines): + return None, None + mnemonic = None + anchor = None + for i in range(line_number - 1, max(-1, line_number - 400), -1): + text = lines[i] + if mnemonic is None: + m = MNEMONIC_RE.match(text) + if m and not m.group(1).startswith(("=", "-", "TODO")): + mnemonic = m.group(1).strip() + if anchor is None: + m = LABEL_RE.match(text) + if m: + anchor = "after loc_%s" % m.group(1).upper() + else: + m = LR_RE.search(text) + if m: + anchor = "after call site lr=0x%s" % m.group(1).upper() + if mnemonic and anchor: + break + # A function boundary ends the walk - anything above belongs elsewhere. + if text.startswith("PPC_FUNC_IMPL("): + break + return mnemonic, anchor + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("crash", help="crash-....txt written by the game") + parser.add_argument("--exe", default="out/build/win-amd64-relwithdebinfo/ac6recomp.exe") + parser.add_argument("--pdb", default=None, help="default: next to the exe") + parser.add_argument("--generated", default="generated") + parser.add_argument("--force", action="store_true", + help="symbolize even if the PDB does not match the exe") + args = parser.parse_args() + + symbolizer = find_tool("llvm-symbolizer") + if not symbolizer: + sys.exit("llvm-symbolizer not found (install LLVM or add it to PATH)") + pdb_path = args.pdb or os.path.splitext(args.exe)[0] + ".pdb" + + # --- PDB<->exe match check. Wrong symbols are worse than none. + readobj = find_tool("llvm-readobj") + pdbutil = find_tool("llvm-pdbutil") + if readobj and pdbutil and os.path.isfile(pdb_path): + exe_guid, exe_age, exe_pdb_name = pe_pdb_guid(readobj, args.exe) + got_guid, got_age = pdb_guid(pdbutil, pdb_path) + if normalize_guid(exe_guid) and normalize_guid(got_guid) and \ + normalize_guid(exe_guid) != normalize_guid(got_guid): + print("*" * 72) + print("PDB MISMATCH: %s does not belong to %s" % (pdb_path, args.exe)) + print(" exe RSDS GUID: %s (age %s, references %s)" % (exe_guid, exe_age, + exe_pdb_name)) + print(" pdb GUID: %s (age %s)" % (got_guid, got_age)) + print(" Rebuild the released commit (reproducible build) to regenerate the") + print(" matching PDB. Symbols below would be WRONG.") + print("*" * 72) + if not args.force: + sys.exit(2) + else: + print("note: PDB match check skipped (llvm-readobj/llvm-pdbutil or the PDB " + "itself unavailable) - trust the symbols accordingly") + + # --- Does this exe actually correspond to this crash file? The PDB check + # above only proves the PDB belongs to the exe; decoding a crash from a + # DIFFERENT build against it yields confident, wrong answers. + crash_commit, crash_built = crash_build_identity(args.crash) + if crash_commit or crash_built: + present = exe_contains_tokens(args.exe, [crash_commit, crash_built]) + if present is False: + print("*" * 72) + print("BUILD MISMATCH: %s was not produced by %s" % (args.crash, args.exe)) + print(" crash file says: commit %s built %s" % (crash_commit, crash_built)) + print(" those build strings are not present in that exe.") + print(" Rebuild that commit and point --exe/--pdb at it.") + print(" Symbols below would be WRONG.") + print("*" * 72) + if not args.force: + sys.exit(3) + else: + print("note: the crash file carries no build identity line - cannot " + "confirm it belongs to this exe") + + with open(args.crash, "r", errors="replace") as f: + crash_lines = f.readlines() + + rva_re = re.compile(r"exe\+0x([0-9A-Fa-f]+)") + decoded_any = False + for line in crash_lines: + stripped = line.rstrip("\n") + m = rva_re.search(stripped) + if not m: + print(stripped) + continue + rva = int(m.group(1), 16) + result = subprocess.run( + [symbolizer, "--obj=" + args.exe, "--relative-address", + "--no-inlines", "--output-style=LLVM", "0x%X" % rva], + capture_output=True, text=True).stdout.strip().splitlines() + function = result[0].strip() if result else "??" + location = result[1].strip() if len(result) > 1 else "??" + print(stripped) + print(" => %s %s" % (function, location)) + loc_match = re.match(r"(.+):(\d+):\d*", location) + if loc_match and "recomp" in loc_match.group(1): + mnemonic, anchor = guest_instruction_at(args.generated, loc_match.group(1), + int(loc_match.group(2))) + if mnemonic: + extra = (" (%s)" % anchor) if anchor else "" + print(" => guest instruction: %s%s" % (mnemonic, extra)) + decoded_any = True + + if not decoded_any: + print("\n(no host frame resolved into generated code - the fault may be " + "entirely host-side; the crash file's own rex_sub_* naming and the " + "guest backchain still apply)") + + +if __name__ == "__main__": + main()