diff --git a/ac6recomp_config.toml b/ac6recomp_config.toml index b146312e..5aeba91a 100644 --- a/ac6recomp_config.toml +++ b/ac6recomp_config.toml @@ -10670,3 +10670,58 @@ registers = ["r3", "r4", "r5", "r6", "ctr"] address = 0x821CCC5C name = "ac6PacDecoderDumpHook" registers = ["r4", "r10", "r11", "r31"] + +# ---------------------------------------------------------------------------- +# Codec internal tracing probes. +# +# Wired at the entries of every PPC function involved in mode-1 decoding so we +# can observe the runtime state at each transition. Gated by env var +# AC6_TRACE_CODEC_INTERNALS=1 and rate-limited (first 3 mode-1 invocations +# only) so log volume stays bounded. Used to confirm whether the bytes the +# codec actually consumes match the .compressed.bin files we dump. +# +# Hook pipeline (top-down): +# sub_822CF510 codec dispatcher -> ac6PacCodecDispatchProbe +# sub_822CF2F8 mode-1 entry -> ac6PacMode1EntryProbe (one-shot ROM dump +# of static-Huffman tables at 0x8243D1F8 / +# 0x8243DDF8) +# sub_822CCB50 bit fetcher -> ac6PacBitFetcherProbe +# sub_822CEAC0 dynamic header -> ac6PacDynamicHeaderProbe +# sub_822CDB38 tree builder -> ac6PacTreeBuilderProbe (dumps the 19 +# u16 code-length array at r3+108) +# sub_822CD068 / sub_822CD758 -> ac6PacBlockConsumerProbe +# ---------------------------------------------------------------------------- +[[midasm_hook]] +address = 0x822CF510 +name = "ac6PacCodecDispatchProbe" +registers = ["r3"] + +[[midasm_hook]] +address = 0x822CF2F8 +name = "ac6PacMode1EntryProbe" +registers = ["r3"] + +[[midasm_hook]] +address = 0x822CCB50 +name = "ac6PacBitFetcherProbe" +registers = ["r3", "r4", "r5"] + +[[midasm_hook]] +address = 0x822CEAC0 +name = "ac6PacDynamicHeaderProbe" +registers = ["r3", "r4"] + +[[midasm_hook]] +address = 0x822CDB38 +name = "ac6PacTreeBuilderProbe" +registers = ["r3"] + +[[midasm_hook]] +address = 0x822CD068 +name = "ac6PacBlockConsumerProbe" +registers = ["r3"] + +[[midasm_hook]] +address = 0x822CD758 +name = "ac6PacBlockConsumerProbe" +registers = ["r3"] diff --git a/cmake/rexglue_bootstrap.cmake b/cmake/rexglue_bootstrap.cmake index ef3787b2..ad979d8f 100644 --- a/cmake/rexglue_bootstrap.cmake +++ b/cmake/rexglue_bootstrap.cmake @@ -17,7 +17,7 @@ else() if(REXSDK_VERSION) find_package(rexglue ${REXSDK_VERSION} EXACT QUIET CONFIG) else() - find_package(rexglue 0.7.4 QUIET CONFIG) + find_package(rexglue 0.8.0 QUIET CONFIG) endif() if(NOT rexglue_FOUND) message(FATAL_ERROR diff --git a/default.xex.i64 b/default.xex.i64 new file mode 100644 index 00000000..8547992e Binary files /dev/null and b/default.xex.i64 differ diff --git a/docs/ac6_asset_pipeline.md b/docs/ac6_asset_pipeline.md index fb7a0681..a3133e7a 100644 --- a/docs/ac6_asset_pipeline.md +++ b/docs/ac6_asset_pipeline.md @@ -10,19 +10,36 @@ It automates these stages: 4. `SWG` UI metadata parsing 5. `NTXR` texture export -## Important Limitation +## Offline mode-1 decompression (solved) -The pipeline can process **all PAC entries and all runtime dumps you already have**, but it **cannot yet offline-decompress every compressed PAC entry in the archive** by itself. +The pipeline can now **decompress every compressed PAC entry fully offline**, with +no need to launch the game. The AC6 "mode-1" codec has been reverse-engineered: -That means: +1. **Descramble**: stored bytes are XORed with an 8-byte repeating pad. The pad + for a DATA.TBL entry is derived from the entry's table index: -- Raw PAC entries are handled in one pass. -- Runtime-decoded assets are handled in one pass. -- If a compressed asset has never been decoded by the game and never appeared in `out/ac6_pac_runtime_dump`, this pipeline cannot currently materialize it. + ``` + pad(index) = pi_words[2*(index % 256) + 1] ++ pi_words[2*(index % 256) + 2] + ``` -So the pipeline is "all at once" for the **current corpus**, not yet "decode the entire PAC archive from scratch with no runtime help". + where `pi_words` are the big-endian base-2^32 words of the fractional part of + pi (`0x243F6A88 0x85A308D3 0x13198A2E ...`). The game generates these at runtime + via Machin's formula (`4*arctan(1/5) - arctan(1/239)`); the offline tool simply + computes pi to the needed precision. -If you want truly complete one-shot extraction of every compressed PAC asset without launching the game, the remaining missing piece is an offline implementation of AC6's mode-1 decompressor. +2. **Inflate**: the descrambled bytes are raw DEFLATE (RFC 1951, no zlib/gzip + wrapper) -- `zlib.decompress(data, wbits=-15)`. + +This is implemented in `tools/ac6_mode1_codec.py` and wired into the extractor: + +```powershell +python .\tools\extract_ac6_pac.py --decompress +``` + +Compressed entries are written as `files/DATA0x/compressed/.decompressed.bin` +and the manifest records `decompressed_count` / `decode_failures`. Playing the game +to pre-populate `out/ac6_pac_runtime_dump` is no longer required for compressed-entry +extraction; it remains useful only for assets synthesized at runtime. ## Prerequisites diff --git a/src/ac6_pac_decode_dump.cpp b/src/ac6_pac_decode_dump.cpp index 7ce11808..1b658b1b 100644 --- a/src/ac6_pac_decode_dump.cpp +++ b/src/ac6_pac_decode_dump.cpp @@ -441,6 +441,23 @@ void Ac6DumpPacDecodedEntry(uint16_t entry_index, uint8_t codec_mode, uint32_t c path.string()); } +std::vector Ac6PeekCompressedHead(uint32_t entry_index, std::size_t max_bytes) { + if (max_bytes == 0) return {}; + auto rec = ac6_pac_index::GetByIndex(entry_index); + if (!rec || rec->compressed_size == 0) return {}; + + const uint32_t want = static_cast( + std::min(max_bytes, static_cast(rec->compressed_size))); + const uint32_t start = rec->offset; + if (want == 0 || start > UINT32_MAX - want) return {}; + const uint32_t end = start + want; + + std::scoped_lock lock(ArchiveMutex()); + auto& archive = GetArchive(rec->is_data01); + if (!IsRangeCovered(archive.chunks, start, end)) return {}; + return GatherRange(archive.chunks, start, end); +} + void Ac6OnPacReadCompleted(std::string_view path, uint32_t guest_buffer, uint64_t file_offset, uint32_t bytes_read) { if (!DumpingEnabled() || guest_buffer == 0 || bytes_read == 0) return; diff --git a/src/ac6_pac_decode_dump.h b/src/ac6_pac_decode_dump.h index 2ba7034a..ce998687 100644 --- a/src/ac6_pac_decode_dump.h +++ b/src/ac6_pac_decode_dump.h @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include // Writes a single decoded PAC entry to the runtime dump directory. // Filename format: entry__mode_c_u_off.bin @@ -9,6 +11,14 @@ void Ac6DumpPacDecodedEntry(uint16_t entry_index, uint8_t codec_mode, uint32_t c uint32_t decompressed_size, uint32_t source_offset, const uint8_t* host_data); +// Returns up to `max_bytes` of the compressed source for the given DATA.TBL +// entry, drawing from the chunks recorded by Ac6OnPacReadCompleted. Returns +// an empty vector if DATA.TBL isn't loaded yet, the entry index is unknown, +// the entry has zero compressed size, or the relevant byte range hasn't been +// streamed in yet. Safe to call from any thread; takes the same mutex used +// by the dump path so it can race with in-flight reads. +std::vector Ac6PeekCompressedHead(uint32_t entry_index, std::size_t max_bytes); + // Hook called from the kernel-side NtReadFile completion path for any read // targeting an AC6 PAC archive (DATA00.PAC, DATA01.PAC) or DATA.TBL itself. // - For DATA.TBL reads: parses and caches the index. diff --git a/src/ac6_pac_decoder_probe.cpp b/src/ac6_pac_decoder_probe.cpp index 5151439c..5a44795a 100644 --- a/src/ac6_pac_decoder_probe.cpp +++ b/src/ac6_pac_decoder_probe.cpp @@ -1,4 +1,5 @@ #include "ac6_pac_decoder_probe.h" +#include "ac6_pac_decode_dump.h" #include #include @@ -7,11 +8,18 @@ #include #include +#include #include #include +#include +#include #include +#include +#include #include +#include #include +#include #include #include @@ -176,6 +184,39 @@ void ac6PacWorkerL2DispatchHook(PPCRegister& r3, PPCRegister& r4, PPCRegister& r } } +namespace { + +bool DumpEnabledEnv() { + static const bool enabled = [] { + const char* v = std::getenv("AC6_DUMP_PAC_DECODED"); + return v && v[0] && std::string_view(v) != "0"; + }(); + return enabled; +} + +std::mutex& DecoderSeenMutex() { + static std::mutex m; + return m; +} + +std::unordered_set& DecoderSeenEntries() { + static std::unordered_set s; + return s; +} + +std::string HexPreviewBytes(const uint8_t* data, std::size_t len) { + static constexpr char kHex[] = "0123456789abcdef"; + std::string out; + out.reserve(len * 2); + for (std::size_t i = 0; i < len; ++i) { + out.push_back(kHex[(data[i] >> 4) & 0xF]); + out.push_back(kHex[data[i] & 0xF]); + } + return out; +} + +} // namespace + void ac6PacDecoderDumpHook(PPCRegister& r4, PPCRegister& r10, PPCRegister& r11, PPCRegister& r31) { auto* memory = REX_KERNEL_MEMORY(); @@ -201,6 +242,298 @@ void ac6PacDecoderDumpHook(PPCRegister& r4, PPCRegister& r10, PPCRegister& r11, const auto* host = memory->TranslateVirtual(r4.u32); if (!host) return; - Ac6DumpPacDecodedEntry(static_cast(r10.u32 & 0xFFFFu), - codec, csize, usize, source_offset, host); + const uint16_t entry_index = static_cast(r10.u32 & 0xFFFFu); + + // Per-distinct-entry RE log. Dedup on (csize,usize) because r10&0xFFFF is a + // runtime tag that collides across distinct TOC entries. Captures the + // descramble pad and its seed so we can crack the per-entry XOR key: + // codec_ctx = streamer(r31) + 344 + // pad (8B) @ codec_ctx + 22380 == r31 + 22724 + // seed (u32) @ codec_ctx + 22392 == r31 + 22736 + // The pad is generated by sub_822CF018(seed) and applied as a repeating + // 8-byte XOR over the codec input. Logging (csize,usize,seed,pad) for every + // distinct entry in one play session gives us the seed->pad table and the + // seed=f(entry) relationship needed for a fully offline decoder. + if (DumpEnabledEnv()) { + const uint64_t dedup_key = (static_cast(csize) << 32) | usize; + bool first = false; + { + std::scoped_lock lock(DecoderSeenMutex()); + first = DecoderSeenEntries().insert(dedup_key).second; + } + if (first) { + constexpr std::size_t kPreviewBytes = 32; + + std::string record_hex; + const uint32_t record_end_check = + r11.u32 > UINT32_MAX - kPreviewBytes ? 0 : r11.u32 + kPreviewBytes - 1; + if (record_end_check && memory->LookupHeap(r11.u32) && + memory->LookupHeap(record_end_check)) { + const auto* rec_host = memory->TranslateVirtual(r11.u32); + if (rec_host) record_hex = HexPreviewBytes(rec_host, kPreviewBytes); + } + + const std::size_t out_preview = + usize < kPreviewBytes ? static_cast(usize) : kPreviewBytes; + const std::string out_hex = HexPreviewBytes(host, out_preview); + + std::vector compressed_head = + Ac6PeekCompressedHead(entry_index, kPreviewBytes); + const std::string in_hex = + compressed_head.empty() + ? std::string{} + : HexPreviewBytes(compressed_head.data(), compressed_head.size()); + + // Descramble pad + seed, read from the codec sub-struct at r31+344. + const uint32_t seed = load_u32_be(r31.u32 + 22736); + std::string pad_hex; + const uint32_t pad_addr = r31.u32 + 22724; + if (memory->LookupHeap(pad_addr) && memory->LookupHeap(pad_addr + 7)) { + const auto* pad = memory->TranslateVirtual(pad_addr); + if (pad) pad_hex = HexPreviewBytes(pad, 8); + } + + REXFS_INFO( + "[AC6 PAC DECODER] entry={} mode={} csize=0x{:x} usize=0x{:x} src_off=0x{:x} " + "seed=0x{:08x} pad={} r4=0x{:08X} r11=0x{:08X} record_hex={} in_head={} out_head={}", + entry_index, uint32_t(codec), csize, usize, source_offset, seed, + pad_hex.empty() ? "" : pad_hex, r4.u32, r11.u32, + record_hex.empty() ? "" : record_hex, + in_hex.empty() ? "" : in_hex, + out_hex.empty() ? "" : out_hex); + } + } + + Ac6DumpPacDecodedEntry(entry_index, codec, csize, usize, source_offset, host); +} + +// ============================================================================ +// Codec internal probes +// ---------------------------------------------------------------------------- +// Gated by AC6_TRACE_CODEC_INTERNALS=1. Rate-limited to the first +// kMaxTracedMode1Calls invocations of the mode-1 decoder so log volume stays +// bounded. +// ============================================================================ + +namespace { + +bool CodecTraceEnabledEnv() { + static const bool enabled = [] { + const char* v = std::getenv("AC6_TRACE_CODEC_INTERNALS"); + return v && v[0] && std::string_view(v) != "0"; + }(); + return enabled; +} + +std::atomic& Mode1CallCount() { + static std::atomic c{0}; + return c; +} + +// First-N detailed-internals trace: bit-fetcher / tree-builder / dynamic-header +// / block-consumer probes only fire while Mode1CallCount <= this. Three +// invocations is enough to capture one entry's worth of internal activity. +constexpr int kMaxInternalTracedCalls = 3; + +// Mode-1 entry probe (including codec_input dump) fires up to this many times, +// regardless of the internal-trace gate. Lets us collect input buffers from +// many entries in a single play session to derive the per-entry XOR key. +constexpr int kMaxMode1EntryProbeCalls = 100; + +bool CodecTraceActive() { + if (!CodecTraceEnabledEnv()) return false; + return Mode1CallCount().load(std::memory_order_relaxed) <= kMaxInternalTracedCalls; +} + +uint32_t LoadGuestU32BE(rex::memory::Memory* memory, uint32_t va) { + if (!memory || va > UINT32_MAX - 3) return 0; + if (!memory->LookupHeap(va) || !memory->LookupHeap(va + 3)) return 0; + return rex::memory::load_and_swap(memory->TranslateVirtual(va)); +} + +uint64_t LoadGuestU64BE(rex::memory::Memory* memory, uint32_t va) { + if (!memory || va > UINT32_MAX - 7) return 0; + if (!memory->LookupHeap(va) || !memory->LookupHeap(va + 7)) return 0; + return rex::memory::load_and_swap(memory->TranslateVirtual(va)); +} + +uint16_t LoadGuestU16BE(rex::memory::Memory* memory, uint32_t va) { + if (!memory || va > UINT32_MAX - 1) return 0; + if (!memory->LookupHeap(va) || !memory->LookupHeap(va + 1)) return 0; + return rex::memory::load_and_swap(memory->TranslateVirtual(va)); +} + +void DumpStaticHuffmanTablesOnce() { + static std::once_flag flag; + std::call_once(flag, [] { + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + // Addresses confirmed via IDA Pro disassembly of sub_822CF2F8. + const uint32_t kAddrs[2] = {0x827BD1F8u, 0x827BDDF8u}; + for (uint32_t addr : kAddrs) { + constexpr uint32_t kSize = 256; + if (!memory->LookupHeap(addr) || !memory->LookupHeap(addr + kSize - 1)) continue; + const auto* p = memory->TranslateVirtual(addr); + if (!p) continue; + REXFS_INFO("[AC6 PAC CODEC] ROM table @ 0x{:08X} ({} bytes): {}", + addr, kSize, HexPreviewBytes(p, kSize)); + } + }); +} + +// Dumps up to `max_bytes` from the codec's input buffer (base = *(ctx + 56)) +// to disk as codec_input_.bin. Lets us compare the bytes the codec +// actually decompresses against the kernel-recorded .compressed.bin to +// identify any pre-codec transformation (descrambling, XOR, etc.). +void DumpCodecInputBuffer(uint32_t ctx, int call_n, std::size_t max_bytes) { + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + const uint32_t base = LoadGuestU32BE(memory, ctx + 56); + if (base == 0) return; + if (!memory->LookupHeap(base)) return; + const uint32_t avail = LoadGuestU32BE(memory, ctx + 16); // in_end + const std::size_t want = + std::min(max_bytes, avail > 0 ? static_cast(avail) : max_bytes); + if (want == 0) return; + if (!memory->LookupHeap(base + static_cast(want) - 1)) return; + const auto* p = memory->TranslateVirtual(base); + if (!p) return; + + // Anchor output next to the existing PAC runtime dumps so analysis can + // compare byte-by-byte against entry_*.compressed.bin in the same dir. + static const std::filesystem::path kDumpRoot = [] { + std::error_code ec; + auto cwd = std::filesystem::current_path(ec); + std::filesystem::path candidate = (ec ? std::filesystem::path() : cwd) / + "out" / "ac6_pac_runtime_dump"; + // Walk up to find repo root if cwd is somewhere deeper. + for (auto cur = candidate.parent_path().parent_path(); + !cur.empty(); cur = cur.parent_path()) { + if (std::filesystem::exists(cur / "tools" / "run_ac6_asset_pipeline.py", ec)) { + candidate = cur / "out" / "ac6_pac_runtime_dump"; + break; + } + if (cur.has_parent_path() && cur == cur.parent_path()) break; + } + return candidate; + }(); + + std::error_code ec; + std::filesystem::create_directories(kDumpRoot, ec); + std::ostringstream name; + name << "codec_input_" << call_n << "_base0x" << std::hex << base << "_size0x" << avail + << ".bin"; + const auto path = kDumpRoot / name.str(); + + std::ofstream f(path, std::ios::binary | std::ios::trunc); + if (!f) { + REXFS_ERROR("[AC6 PAC CODEC] failed to open codec input dump {}", path.string()); + return; + } + f.write(reinterpret_cast(p), static_cast(want)); + REXFS_INFO("[AC6 PAC CODEC] dumped codec input #{} base=0x{:08X} size=0x{:x} " + "(first {} bytes) path={}", call_n, base, avail, want, path.string()); +} + +} // namespace + +void ac6PacCodecDispatchProbe(PPCRegister& r3) { + if (!CodecTraceActive()) return; + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + const uint32_t ctx = r3.u32; + uint8_t codec_mode = 0; + if (memory->LookupHeap(ctx + 68)) { + codec_mode = *memory->TranslateVirtual(ctx + 68); + } + REXFS_INFO("[AC6 PAC CODEC] dispatch ctx=0x{:08X} codec_mode={} disp_state=0x{:x}", + ctx, uint32_t(codec_mode), LoadGuestU32BE(memory, ctx + 88)); +} + +void ac6PacMode1EntryProbe(PPCRegister& r3) { + if (!CodecTraceEnabledEnv()) return; + const int call_n = Mode1CallCount().fetch_add(1, std::memory_order_relaxed) + 1; + if (call_n > kMaxMode1EntryProbeCalls) return; + + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + + DumpStaticHuffmanTablesOnce(); + + const uint32_t ctx = r3.u32; + REXFS_INFO("[AC6 PAC CODEC] mode1_entry #{} ctx=0x{:08X} state=0x{:x} " + "in_ptr=0x{:x} in_end=0x{:x} out_pos=0x{:x} bit_buf=0x{:016x} bit_count={} " + "base_ptr=0x{:08X}", + call_n, ctx, + LoadGuestU32BE(memory, ctx + 84), + LoadGuestU64BE(memory, ctx + 8), + LoadGuestU64BE(memory, ctx + 16), + LoadGuestU64BE(memory, ctx + 32), + LoadGuestU64BE(memory, ctx + 72), + LoadGuestU32BE(memory, ctx + 80), + LoadGuestU32BE(memory, ctx + 56)); + + // Dump the codec input buffer on state==0 (initial chunk of a new entry) + // so we can diff it against the kernel-recorded .compressed.bin to find + // any transformation between PAC archive bytes and the decoder's input. + const uint32_t state = LoadGuestU32BE(memory, ctx + 84); + if (state == 0) { + DumpCodecInputBuffer(ctx, call_n, 4096); + } +} + +void ac6PacBitFetcherProbe(PPCRegister& r3, PPCRegister& r4, PPCRegister& r5) { + if (!CodecTraceActive()) return; + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + const uint32_t ctx = r3.u32; + const uint32_t cursor_ptr = r4.u32; + REXFS_INFO("[AC6 PAC CODEC] bit_fetch bits_req={} cursor_ptr=0x{:x} " + "cursor=0x{:x} bit_buf=0x{:016x} bit_count={}", + r5.u32, cursor_ptr, + LoadGuestU64BE(memory, cursor_ptr), + LoadGuestU64BE(memory, ctx + 72), + LoadGuestU32BE(memory, ctx + 80)); +} + +void ac6PacDynamicHeaderProbe(PPCRegister& r3, PPCRegister& r4) { + if (!CodecTraceActive()) return; + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + const uint32_t ctx = r3.u32; + REXFS_INFO("[AC6 PAC CODEC] dynamic_header_entry ctx=0x{:08X} cursor_arg=0x{:x} " + "bit_buf=0x{:016x} bit_count={}", + ctx, r4.u32, + LoadGuestU64BE(memory, ctx + 72), + LoadGuestU32BE(memory, ctx + 80)); +} + +void ac6PacTreeBuilderProbe(PPCRegister& r3) { + if (!CodecTraceActive()) return; + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + const uint32_t ctx = r3.u32; + const uint32_t arr_base = ctx + 108; + constexpr uint32_t kEntries = 19; + std::string lens; + lens.reserve(kEntries * 4); + for (uint32_t i = 0; i < kEntries; ++i) { + if (i) lens.push_back(','); + lens += std::to_string(LoadGuestU16BE(memory, arr_base + i * 2)); + } + REXFS_INFO("[AC6 PAC CODEC] tree_builder_entry ctx=0x{:08X} cl_array=[{}]", ctx, lens); +} + +void ac6PacBlockConsumerProbe(PPCRegister& r3) { + if (!CodecTraceActive()) return; + auto* memory = REX_KERNEL_MEMORY(); + if (!memory) return; + const uint32_t ctx = r3.u32; + REXFS_INFO("[AC6 PAC CODEC] block_consumer_entry ctx=0x{:08X} state=0x{:x} " + "out_pos=0x{:x} bit_buf=0x{:016x} bit_count={}", + ctx, + LoadGuestU32BE(memory, ctx + 84), + LoadGuestU64BE(memory, ctx + 32), + LoadGuestU64BE(memory, ctx + 72), + LoadGuestU32BE(memory, ctx + 80)); } diff --git a/src/ac6_pac_decoder_probe.h b/src/ac6_pac_decoder_probe.h index ad163fcd..346da702 100644 --- a/src/ac6_pac_decoder_probe.h +++ b/src/ac6_pac_decoder_probe.h @@ -45,3 +45,29 @@ void ac6PacDecoderDumpHook(PPCRegister& r4, PPCRegister& r10, PPCRegister& r11, void Ac6DumpPacDecodedEntry(uint16_t entry_index, uint8_t codec_mode, uint32_t compressed_size, uint32_t decompressed_size, uint32_t source_offset, const uint8_t* host_data); + +// --------------------------------------------------------------------------- +// Codec-internals tracing probes. +// +// Each probe is a mid-asm hook on a specific PPC function entry inside the +// mode-1 decoder pipeline. All probes are gated by env var +// AC6_TRACE_CODEC_INTERNALS=1 and rate-limited to the first 3 mode-1 decoder +// invocations so log volume stays manageable. +// +// Decoder pipeline (top-down): +// sub_822CF510 codec dispatcher -> ac6PacCodecDispatchProbe +// sub_822CF2F8 mode-1 entry -> ac6PacMode1EntryProbe (one-shot ROM dump) +// sub_822CCB50 bit fetcher -> ac6PacBitFetcherProbe +// sub_822CEAC0 dynamic header -> ac6PacDynamicHeaderProbe +// sub_822CDB38 tree builder -> ac6PacTreeBuilderProbe +// sub_822CD068 / sub_822CD758 -> ac6PacBlockConsumerProbe +// +// The bit-fetcher probe is the highest-value: it logs every byte the codec +// consumes, which lets us verify whether the .compressed.bin bytes are read +// verbatim or transformed before reaching the bit buffer. +void ac6PacCodecDispatchProbe(PPCRegister& r3); +void ac6PacMode1EntryProbe(PPCRegister& r3); +void ac6PacBitFetcherProbe(PPCRegister& r3, PPCRegister& r4, PPCRegister& r5); +void ac6PacDynamicHeaderProbe(PPCRegister& r3, PPCRegister& r4); +void ac6PacTreeBuilderProbe(PPCRegister& r3); +void ac6PacBlockConsumerProbe(PPCRegister& r3); diff --git a/thirdparty/rexglue-sdk/.github/CODEOWNERS b/thirdparty/rexglue-sdk/.github/CODEOWNERS new file mode 100644 index 00000000..c8bb481f --- /dev/null +++ b/thirdparty/rexglue-sdk/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Global owners +* @tomcl7 diff --git a/thirdparty/rexglue-sdk/CMakeLists.txt b/thirdparty/rexglue-sdk/CMakeLists.txt index b76ad098..a53b36f2 100644 --- a/thirdparty/rexglue-sdk/CMakeLists.txt +++ b/thirdparty/rexglue-sdk/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.25) project(ReXGlue - VERSION 0.7.1 + VERSION 0.8.0 LANGUAGES C CXX DESCRIPTION "Xbox 360 Recompilation SDK" ) diff --git a/thirdparty/rexglue-sdk/include/native/ui/d3d12/d3d12_util.h b/thirdparty/rexglue-sdk/include/native/ui/d3d12/d3d12_util.h index 3ba9996a..41b56ee6 100644 --- a/thirdparty/rexglue-sdk/include/native/ui/d3d12/d3d12_util.h +++ b/thirdparty/rexglue-sdk/include/native/ui/d3d12/d3d12_util.h @@ -21,9 +21,9 @@ namespace util { using DescriptorCpuGpuHandlePair = std::pair; -extern const D3D12_HEAP_PROPERTIES kHeapPropertiesDefault; -extern const D3D12_HEAP_PROPERTIES kHeapPropertiesUpload; -extern const D3D12_HEAP_PROPERTIES kHeapPropertiesReadback; +inline constexpr D3D12_HEAP_PROPERTIES kHeapPropertiesDefault = {D3D12_HEAP_TYPE_DEFAULT}; +inline constexpr D3D12_HEAP_PROPERTIES kHeapPropertiesUpload = {D3D12_HEAP_TYPE_UPLOAD}; +inline constexpr D3D12_HEAP_PROPERTIES kHeapPropertiesReadback = {D3D12_HEAP_TYPE_READBACK}; template bool ReleaseAndNull(T& object) { diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/analyze.h b/thirdparty/rexglue-sdk/include/rex/codegen/analyze.h index 7926d7a4..d80914d5 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/analyze.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/analyze.h @@ -12,6 +12,7 @@ #pragma once #include +#include #include namespace rex::codegen { @@ -31,6 +32,6 @@ namespace rex::codegen { * @param ctx CodegenContext with binary and config loaded * @return Success if graph is valid, error otherwise */ -Result Analyze(CodegenContext& ctx); +Result Analyze(CodegenContext& ctx, ProgressReporter* reporter = nullptr); } // namespace rex::codegen diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/code_emitter.h b/thirdparty/rexglue-sdk/include/rex/codegen/code_emitter.h index d41325f0..d1079fdd 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/code_emitter.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/code_emitter.h @@ -19,7 +19,7 @@ namespace rex::codegen { // Forward declarations -class RecompilerConfig; +struct RecompilerConfig; /** * @brief CSR (Control/Status Register) state for FPU denormal handling. diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/codegen.h b/thirdparty/rexglue-sdk/include/rex/codegen/codegen.h index b7b9cfee..0eacb698 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/codegen.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/codegen.h @@ -50,15 +50,10 @@ class CodegenPipeline { */ static Result Create(const std::filesystem::path& configPath); - /** - * Run the full pipeline: Analyze -> Recompile. - * - * @param force If true, generate output even with validation errors - * @return Success or error with description - */ Result Run(bool force = false); + Result RunAnalyze(); + Result RunWrite(bool force = false); - /// Access context for CLI needs (output path, project name, etc.) CodegenContext& context() { return *ctx_; } const CodegenContext& context() const { return *ctx_; } diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/codegen_writer.h b/thirdparty/rexglue-sdk/include/rex/codegen/codegen_writer.h index bcee935b..56d0290b 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/codegen_writer.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/codegen_writer.h @@ -32,6 +32,18 @@ class CodegenWriter { /// Run the full output pipeline: validate, clean old files, generate, flush. bool write(bool force); + /** + * Basenames of files removed by the pre-emit cleanup sweep during write(). + * Populated only after write() completes. Empty otherwise. + */ + const std::vector& deletedFiles() const { return deletedFiles_; } + + /** + * Basenames of files written to disk during write() (via FlushPendingWrites). + * Populated only after write() completes. Empty otherwise. + */ + const std::vector& writtenFiles() const { return writtenFiles_; } + private: CodegenContext& ctx_; Runtime* runtime_; @@ -39,6 +51,8 @@ class CodegenWriter { std::string out; size_t cppFileIndex = 0; std::vector> pendingWrites; + std::vector deletedFiles_; + std::vector writtenFiles_; template void print(fmt::format_string fmt, Args&&... args) { diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/config.h b/thirdparty/rexglue-sdk/include/rex/codegen/config.h index 343af1bb..e8d3b47c 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/config.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/config.h @@ -13,11 +13,14 @@ #include #include +#include #include #include #include #include +#include + #include // For JumpTable namespace rex::codegen { @@ -95,6 +98,10 @@ struct RecompilerConfig { uint32_t dataRegionThreshold = 16; ///< Consecutive invalid instructions to mark as data region uint32_t largeFunctionThreshold = 1048576; ///< 1MB - warn if function exceeds this size + // Optional override for DLL module flag. If unset, the orchestrator infers + // from the module's position in the manifest (entrypoint = false, modules = true). + std::optional isDll; + // === Manual overrides === std::unordered_map functions; ///< Function/chunk configuration std::unordered_map switchTables; @@ -127,6 +134,13 @@ struct RecompilerConfig { */ bool Load(const std::string_view& configFilePath); + /** + * Load configuration from an in-memory TOML table (e.g. an inline binary + * entry inside a manifest). Includes referenced from the table resolve + * relative to `base_dir`. Same merge semantics as Load(). + */ + bool LoadFromTable(const toml::table& tbl, const std::filesystem::path& base_dir); + /// Validation result containing warnings and errors. struct ValidationResult { bool valid = true; ///< true if no errors (warnings OK) diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/function_node.h b/thirdparty/rexglue-sdk/include/rex/codegen/function_node.h index af5dd89a..a335b37e 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/function_node.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/function_node.h @@ -94,8 +94,8 @@ class FunctionNode { /// Emit C++ code for this function /// Requires: state() == kSealed - /// For imports: emits PPC_IMPORT macro - /// For normal functions: emits PPC_FUNC with blocks and instructions + /// For imports: emits REX_IMPORT macro + /// For normal functions: emits REX_FUNC with blocks and instructions std::string emitCpp(const EmitContext& ctx) const; //========================================================================= diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/manifest.h b/thirdparty/rexglue-sdk/include/rex/codegen/manifest.h new file mode 100644 index 00000000..395d65d2 --- /dev/null +++ b/thirdparty/rexglue-sdk/include/rex/codegen/manifest.h @@ -0,0 +1,70 @@ +/** + * @file rex/codegen/manifest.h + * @brief Manifest TOML parser for multi-binary projects + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace rex::codegen { + +/** + * Codegen settings for a single binary inside a manifest. The entrypoint + * uses an empty `guestPath`; module entries set it to the canonicalized + * guest-visible path the host runtime resolves against. + */ +struct BinaryConfig { + RecompilerConfig recompiler; + std::string guestPath; +}; + +/** + * Canonicalize a module guest path: device-stripped, slashes/case normalized, + * with `/assets/` stripped when a matching project name is given. + */ +std::string CanonicalizeModuleGuestPath(std::string_view path, std::string_view project_name = {}); + +/** + * Parsed manifest TOML. Construct via Load(); treat as read-only after. + */ +struct ManifestConfig { + std::string projectName; + std::optional sdkVersion; ///< Last SDK that ran codegen on this project + std::optional gameRoot; ///< Game asset root, relative to manifestDir. + ///< Set by `rexglue init` to anchor DLL guest paths. + std::filesystem::path manifestDir; ///< Directory containing the manifest + BinaryConfig entrypoint; ///< Entrypoint codegen settings (inline) + std::vector modules; ///< DLL module codegen settings (inline) + + /** + * Load a manifest TOML file. Returns nullopt on parse failure. + */ + static std::optional Load(const std::filesystem::path& path); + + /** + * True when `path` parses as a manifest (i.e. has a `[project]` section). + */ + static bool IsManifest(const std::filesystem::path& path); + + /** + * Insert or overwrite [project].sdk_version in the manifest file at `path`. + * Preserves the rest of the file's content. Returns false on parse or write + * failure. + */ + static bool WriteSdkVersionStamp(const std::filesystem::path& path, std::string_view version); +}; + +} // namespace rex::codegen diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/phases.h b/thirdparty/rexglue-sdk/include/rex/codegen/phases.h index a824bbf0..e96f7ec4 100644 --- a/thirdparty/rexglue-sdk/include/rex/codegen/phases.h +++ b/thirdparty/rexglue-sdk/include/rex/codegen/phases.h @@ -14,26 +14,27 @@ #include #include +#include #include namespace rex::codegen::phases { /// Register PDATA, CONFIG, helpers, imports into FunctionGraph -VoidResult Register(CodegenContext& ctx); +VoidResult Register(CodegenContext& ctx, ProgressReporter* reporter = nullptr); /// Scan binary for bl targets, thunks, bctr locations -VoidResult Scan(CodegenContext& ctx); +VoidResult Scan(CodegenContext& ctx, ProgressReporter* reporter = nullptr); /// Discover function blocks from candidates -VoidResult Discover(CodegenContext& ctx); +VoidResult Discover(CodegenContext& ctx, ProgressReporter* reporter = nullptr); /// Vacancy-based function expansion and sealing -VoidResult Merge(CodegenContext& ctx); +VoidResult Merge(CodegenContext& ctx, ProgressReporter* reporter = nullptr); /// Find uncovered code regions and register them as functions -VoidResult GapFill(CodegenContext& ctx); +VoidResult GapFill(CodegenContext& ctx, ProgressReporter* reporter = nullptr); /// Validate all call targets resolve -VoidResult Validate(CodegenContext& ctx); +VoidResult Validate(CodegenContext& ctx, ProgressReporter* reporter = nullptr); } // namespace rex::codegen::phases diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/progress_reporter.h b/thirdparty/rexglue-sdk/include/rex/codegen/progress_reporter.h new file mode 100644 index 00000000..fdaacd32 --- /dev/null +++ b/thirdparty/rexglue-sdk/include/rex/codegen/progress_reporter.h @@ -0,0 +1,49 @@ +/** + * @file rex/codegen/progress_reporter.h + * @brief Abstract callback interface for codegen pipeline progress + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#pragma once + +#include +#include +#include + +namespace rex::codegen { + +/** + * Optional progress callback invoked by the codegen pipeline at module + * and phase boundaries. CLI consumers implement this to drive a progress + * view; library-internal callers (tests, headless callers) can pass + * nullptr to opt out. + * + * All methods are called from the thread that drove the pipeline; no + * cross-thread synchronization is implied. + */ +class ProgressReporter { + public: + virtual ~ProgressReporter() = default; + + /** A new module's analysis+write cycle is starting. `index` is 0-based. */ + virtual void moduleStarted(std::string_view name, std::size_t index, std::size_t total) = 0; + + /** A named phase within the current module is starting. */ + virtual void phaseChanged(std::string_view name) = 0; + + /** The current module finished successfully. */ + virtual void moduleFinished(std::chrono::milliseconds elapsed) = 0; + + /** Project-level (non-module) emit phase started, e.g. "module_registry". */ + virtual void projectPhaseStarted(std::string_view name) = 0; + + /** Project-level emit phase finished. */ + virtual void projectPhaseFinished() = 0; +}; + +} // namespace rex::codegen diff --git a/thirdparty/rexglue-sdk/include/rex/codegen/project_recompiler.h b/thirdparty/rexglue-sdk/include/rex/codegen/project_recompiler.h new file mode 100644 index 00000000..ece295ff --- /dev/null +++ b/thirdparty/rexglue-sdk/include/rex/codegen/project_recompiler.h @@ -0,0 +1,53 @@ +/** + * @file rex/codegen/project_recompiler.h + * @brief Project-level recompiler driving manifest-based multi-binary codegen + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace rex::codegen { + +struct ProjectRecompilerOptions { + std::vector targets; // empty = all + bool force = false; + bool enableExceptionHandlers = false; + ProgressReporter* reporter = nullptr; +}; + +class ProjectRecompiler { + public: + explicit ProjectRecompiler(ManifestConfig manifest); + Result Run(const ProjectRecompilerOptions& opts); + + /** + * Aggregated basenames of files removed across all modules during the most + * recent Run() call. Empty until Run() completes successfully. + */ + const std::vector& deletedFiles() const { return deletedFiles_; } + + /** + * Aggregated basenames of files written across all modules during the most + * recent Run() call. Empty until Run() completes successfully. + */ + const std::vector& writtenFiles() const { return writtenFiles_; } + + private: + ManifestConfig manifest_; + std::vector deletedFiles_; + std::vector writtenFiles_; +}; + +} // namespace rex::codegen diff --git a/thirdparty/rexglue-sdk/include/rex/cvar.h b/thirdparty/rexglue-sdk/include/rex/cvar.h index 91716cdc..c6abbfb1 100644 --- a/thirdparty/rexglue-sdk/include/rex/cvar.h +++ b/thirdparty/rexglue-sdk/include/rex/cvar.h @@ -12,7 +12,7 @@ * REXCVAR_DEFINE_STRING(my_string, "default", "Category", "A string setting"); * @endcode * - * Available types: BOOL, INT32, INT64, UINT32, UINT64, DOUBLE, STRING + * Available types: BOOL, INT32, INT64, UINT32, UINT64, DOUBLE, STRING, COMMAND * * @section cvar_declaring Declaring CVars (for use in other files) * @@ -114,7 +114,7 @@ void SaveConfig(const std::filesystem::path& config_path); // Flag Registry //============================================================================= -enum class FlagType { Boolean, Int32, Int64, Uint32, Uint64, Double, String }; +enum class FlagType { Boolean, Int32, Int64, Uint32, Uint64, Double, String, Command }; // Lifecycle: when can this flag be modified? enum class Lifecycle { @@ -141,6 +141,7 @@ struct FlagEntry { std::string description; std::function setter; std::function getter; + std::function command_callback; Lifecycle lifecycle = Lifecycle::kHotReload; Constraints constraints; std::string default_value; @@ -148,9 +149,45 @@ struct FlagEntry { }; std::vector& GetRegistry(); -void RegisterFlag(FlagEntry entry); + +/** + * Returns the registered entry's index, or nullopt if the name was already + * registered (logged at ERROR). + */ +std::optional RegisterFlag(FlagEntry entry); + +/** + * Removes a flag from the registry. Used by `FlagRegistrar`'s destructor so + * that DLL unload tears down the lambdas captured in each FlagEntry. + */ +void UnregisterFlag(std::string_view name); + bool SetFlagByName(std::string_view name, std::string_view value); std::string GetFlagByName(std::string_view name); + +// Typed registry query. Cross-DLL access path that does not require linking +// the DLL where the cvar is defined. Slower than REXCVAR_GET (string parse + +// hash lookup), so prefer REXCVAR_GET when the defining DLL is already on the +// link line. Returns a value-initialized T when the cvar is missing or its +// stored string fails to parse. +template +T Query(std::string_view name); + +template <> +bool Query(std::string_view name); +template <> +int32_t Query(std::string_view name); +template <> +int64_t Query(std::string_view name); +template <> +uint32_t Query(std::string_view name); +template <> +uint64_t Query(std::string_view name); +template <> +double Query(std::string_view name); +template <> +std::string Query(std::string_view name); + std::vector ListFlags(); std::vector ListFlagsByCategory(std::string_view category); std::vector ListFlagsByLifecycle(Lifecycle lc); @@ -175,54 +212,70 @@ void RegisterChangeCallback(std::string_view name, ChangeCallback callback); /// Unregister all callbacks for a specific CVAR void UnregisterChangeCallbacks(std::string_view name); +/** + * RAII handle for a registered flag. Destructor unregisters by name; on + * duplicate-name registration `owned_name_` is empty so chain methods and + * the destructor become no-ops and the original owner's entry is untouched. + */ struct FlagRegistrar { - FlagEntry* entry_ptr = nullptr; + std::string owned_name_; // empty when registration was rejected explicit FlagRegistrar(FlagEntry e) { - // Register immediately and store pointer to the registered entry for chaining - RegisterFlag(std::move(e)); - auto& registry = GetRegistry(); - entry_ptr = ®istry.back(); + std::string name = e.name; + if (RegisterFlag(std::move(e)).has_value()) { + owned_name_ = std::move(name); + } } - // Move constructor for copy-initialization in macros - FlagRegistrar(FlagRegistrar&& other) noexcept : entry_ptr(other.entry_ptr) { - other.entry_ptr = nullptr; + FlagRegistrar(FlagRegistrar&& other) noexcept : owned_name_(std::move(other.owned_name_)) { + other.owned_name_.clear(); } - // Chain methods are rvalue-ref-qualified to work with temporaries + ~FlagRegistrar() { + if (!owned_name_.empty()) { + UnregisterFlag(owned_name_); + } + } + + // Chain methods mutate the registered entry by name lookup. FlagRegistrar&& range(double min_val, double max_val) && { - entry_ptr->constraints.min = min_val; - entry_ptr->constraints.max = max_val; + apply_([=](FlagEntry& entry) { + entry.constraints.min = min_val; + entry.constraints.max = max_val; + }); return std::move(*this); } FlagRegistrar&& allowed(std::initializer_list values) && { - entry_ptr->constraints.allowed_values = values; + std::vector vals(values); + apply_([vals = std::move(vals)](FlagEntry& entry) { entry.constraints.allowed_values = vals; }); return std::move(*this); } FlagRegistrar&& lifecycle(Lifecycle lc) && { - entry_ptr->lifecycle = lc; + apply_([=](FlagEntry& entry) { entry.lifecycle = lc; }); return std::move(*this); } FlagRegistrar&& debug_only() && { - entry_ptr->is_debug_only = true; + apply_([](FlagEntry& entry) { entry.is_debug_only = true; }); return std::move(*this); } FlagRegistrar&& validator(std::function fn) && { - entry_ptr->constraints.custom_validator = std::move(fn); + apply_([fn = std::move(fn)](FlagEntry& entry) mutable { + entry.constraints.custom_validator = std::move(fn); + }); return std::move(*this); } - ~FlagRegistrar() = default; - // Non-copyable (prevent double registration) FlagRegistrar(const FlagRegistrar&) = delete; FlagRegistrar& operator=(const FlagRegistrar&) = delete; FlagRegistrar& operator=(FlagRegistrar&&) = delete; + + private: + void apply_(std::function fn); }; inline bool ParseDouble(std::string_view s, double& out) { @@ -240,158 +293,214 @@ inline bool ParseDouble(std::string_view s, double& out) { // CVar Macros //============================================================================= -// Declare a cvar (use in files that need access to a cvar defined elsewhere) -#define REXCVAR_DECLARE(type, name) extern type FLAGS_##name +// Declare a cvar (use in headers and TUs that need to read it). +// The accessor function returns a reference to the cvar's storage. Storage +// lives as a static-local inside whichever DLL contains the matching +// REXCVAR_DEFINE_*. Cross-DLL access goes through the import lib. +#define REXCVAR_DECLARE(type, name) type& FLAGS_##name##_storage_() // Get a cvar value -#define REXCVAR_GET(name) (FLAGS_##name) +#define REXCVAR_GET(name) (FLAGS_##name##_storage_()) // Set a cvar value -#define REXCVAR_SET(name, value) (FLAGS_##name = (value)) +#define REXCVAR_SET(name, value) (FLAGS_##name##_storage_() = (value)) + +// Cross-module typed query that goes through the cvar registry by name. +// Use this when the defining DLL is not on the consumer's link line (e.g., +// across one-way subsystem dependencies where adding the reverse link would +// create a cycle). Slower than REXCVAR_GET; prefer REXCVAR_GET when possible. +#define REXCVAR_QUERY(type, name) (::rex::cvar::Query(#name)) // Define cvars (use in one .cpp file per cvar) // The FlagRegistrar registers the flag in its destructor, allowing method chaining. -#define REXCVAR_DEFINE_BOOL(name, default_val, category, desc) \ - bool FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = \ - ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::Boolean, \ - category, \ - desc, \ - [](std::string_view v) { \ - bool val = (v == "true" || v == "1" || v == "yes"); \ - FLAGS_##name = val; \ - return true; \ - }, \ - []() { return FLAGS_##name ? "true" : "false"; }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - (default_val) ? "true" : "false", \ +#define REXCVAR_DEFINE_BOOL(name, default_val, category, desc) \ + bool& FLAGS_##name##_storage_() { \ + static bool storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Boolean, \ + category, \ + desc, \ + [](std::string_view v) { \ + bool val = (v == "true" || v == "1" || v == "yes"); \ + FLAGS_##name##_storage_() = val; \ + return true; \ + }, \ + []() { return FLAGS_##name##_storage_() ? "true" : "false"; }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + (default_val) ? "true" : "false", \ false}) -#define REXCVAR_DEFINE_INT32(name, default_val, category, desc) \ - int32_t FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = \ - ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::Int32, \ - category, \ - desc, \ - [](std::string_view v) { \ - int32_t val = 0; \ - auto [ptr, ec] = \ - std::from_chars(v.data(), v.data() + v.size(), val); \ - if (ec != std::errc()) \ - return false; \ - FLAGS_##name = val; \ - return true; \ - }, \ - []() { return std::to_string(FLAGS_##name); }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - std::to_string(default_val), \ +#define REXCVAR_DEFINE_INT32(name, default_val, category, desc) \ + int32_t& FLAGS_##name##_storage_() { \ + static int32_t storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Int32, \ + category, \ + desc, \ + [](std::string_view v) { \ + int32_t val = 0; \ + auto [ptr, ec] = \ + std::from_chars(v.data(), v.data() + v.size(), val); \ + if (ec != std::errc()) \ + return false; \ + FLAGS_##name##_storage_() = val; \ + return true; \ + }, \ + []() { return std::to_string(FLAGS_##name##_storage_()); }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + std::to_string(default_val), \ false}) -#define REXCVAR_DEFINE_INT64(name, default_val, category, desc) \ - int64_t FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = \ - ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::Int64, \ - category, \ - desc, \ - [](std::string_view v) { \ - int64_t val = 0; \ - auto [ptr, ec] = \ - std::from_chars(v.data(), v.data() + v.size(), val); \ - if (ec != std::errc()) \ - return false; \ - FLAGS_##name = val; \ - return true; \ - }, \ - []() { return std::to_string(FLAGS_##name); }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - std::to_string(default_val), \ +#define REXCVAR_DEFINE_INT64(name, default_val, category, desc) \ + int64_t& FLAGS_##name##_storage_() { \ + static int64_t storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Int64, \ + category, \ + desc, \ + [](std::string_view v) { \ + int64_t val = 0; \ + auto [ptr, ec] = \ + std::from_chars(v.data(), v.data() + v.size(), val); \ + if (ec != std::errc()) \ + return false; \ + FLAGS_##name##_storage_() = val; \ + return true; \ + }, \ + []() { return std::to_string(FLAGS_##name##_storage_()); }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + std::to_string(default_val), \ false}) -#define REXCVAR_DEFINE_UINT32(name, default_val, category, desc) \ - uint32_t FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = \ - ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::Uint32, \ - category, \ - desc, \ - [](std::string_view v) { \ - uint32_t val = 0; \ - auto [ptr, ec] = \ - std::from_chars(v.data(), v.data() + v.size(), val); \ - if (ec != std::errc()) \ - return false; \ - FLAGS_##name = val; \ - return true; \ - }, \ - []() { return std::to_string(FLAGS_##name); }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - std::to_string(default_val), \ +#define REXCVAR_DEFINE_UINT32(name, default_val, category, desc) \ + uint32_t& FLAGS_##name##_storage_() { \ + static uint32_t storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Uint32, \ + category, \ + desc, \ + [](std::string_view v) { \ + uint32_t val = 0; \ + auto [ptr, ec] = \ + std::from_chars(v.data(), v.data() + v.size(), val); \ + if (ec != std::errc()) \ + return false; \ + FLAGS_##name##_storage_() = val; \ + return true; \ + }, \ + []() { return std::to_string(FLAGS_##name##_storage_()); }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + std::to_string(default_val), \ false}) -#define REXCVAR_DEFINE_UINT64(name, default_val, category, desc) \ - uint64_t FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = \ - ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::Uint64, \ - category, \ - desc, \ - [](std::string_view v) { \ - uint64_t val = 0; \ - auto [ptr, ec] = \ - std::from_chars(v.data(), v.data() + v.size(), val); \ - if (ec != std::errc()) \ - return false; \ - FLAGS_##name = val; \ - return true; \ - }, \ - []() { return std::to_string(FLAGS_##name); }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - std::to_string(default_val), \ +#define REXCVAR_DEFINE_UINT64(name, default_val, category, desc) \ + uint64_t& FLAGS_##name##_storage_() { \ + static uint64_t storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Uint64, \ + category, \ + desc, \ + [](std::string_view v) { \ + uint64_t val = 0; \ + auto [ptr, ec] = \ + std::from_chars(v.data(), v.data() + v.size(), val); \ + if (ec != std::errc()) \ + return false; \ + FLAGS_##name##_storage_() = val; \ + return true; \ + }, \ + []() { return std::to_string(FLAGS_##name##_storage_()); }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + std::to_string(default_val), \ false}) -#define REXCVAR_DEFINE_DOUBLE(name, default_val, category, desc) \ - double FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = \ - ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::Double, \ - category, \ - desc, \ - [](std::string_view v) { \ - double val = 0; \ - if (!::rex::cvar::ParseDouble(v, val)) \ - return false; \ - FLAGS_##name = val; \ - return true; \ - }, \ - []() { return std::to_string(FLAGS_##name); }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - std::to_string(default_val), \ +#define REXCVAR_DEFINE_DOUBLE(name, default_val, category, desc) \ + double& FLAGS_##name##_storage_() { \ + static double storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Double, \ + category, \ + desc, \ + [](std::string_view v) { \ + double val = 0; \ + if (!::rex::cvar::ParseDouble(v, val)) \ + return false; \ + FLAGS_##name##_storage_() = val; \ + return true; \ + }, \ + []() { return std::to_string(FLAGS_##name##_storage_()); }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + std::to_string(default_val), \ false}) -#define REXCVAR_DEFINE_STRING(name, default_val, category, desc) \ - std::string FLAGS_##name = (default_val); \ - static auto _cvar_reg_##name = ::rex::cvar::FlagRegistrar({#name, \ - ::rex::cvar::FlagType::String, \ - category, \ - desc, \ - [](std::string_view v) { \ - FLAGS_##name = std::string(v); \ - return true; \ - }, \ - []() { return FLAGS_##name; }, \ - ::rex::cvar::Lifecycle::kHotReload, \ - {}, \ - default_val, \ - false}) +#define REXCVAR_DEFINE_STRING(name, default_val, category, desc) \ + std::string& FLAGS_##name##_storage_() { \ + static std::string storage = (default_val); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::String, \ + category, \ + desc, \ + [](std::string_view v) { \ + FLAGS_##name##_storage_() = std::string(v); \ + return true; \ + }, \ + []() { return FLAGS_##name##_storage_(); }, \ + []() { return; }, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + default_val, \ + false}) + +#define REXCVAR_DEFINE_COMMAND(name, callback, category, desc) \ + std::function& FLAGS_##name##_storage_() { \ + static std::function storage = (callback); \ + return storage; \ + } \ + static auto _cvar_reg_##name = \ + ::rex::cvar::FlagRegistrar({#name, \ + ::rex::cvar::FlagType::Command, \ + category, \ + desc, \ + [](std::string_view) { return false; }, \ + []() { return ""; }, \ + callback, \ + ::rex::cvar::Lifecycle::kHotReload, \ + {}, \ + "", \ + false}) namespace rex::cvar { namespace testing { diff --git a/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_container_file.h b/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_container_file.h index c583e243..b0e0b517 100644 --- a/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_container_file.h +++ b/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_container_file.h @@ -27,11 +27,11 @@ class StfsContainerFile : public File { void Destroy() override; X_STATUS ReadSync(std::span buffer, size_t byte_offset, size_t* out_bytes_read) override; - X_STATUS WriteSync(std::span buffer, size_t byte_offset, - size_t* out_bytes_written) override { + X_STATUS WriteSync(std::span /*buffer*/, size_t /*byte_offset*/, + size_t* /*out_bytes_written*/) override { return X_STATUS_ACCESS_DENIED; } - X_STATUS SetLength(size_t length) override { return X_STATUS_ACCESS_DENIED; } + X_STATUS SetLength(size_t /*length*/) override { return X_STATUS_ACCESS_DENIED; } private: StfsContainerEntry* entry_; diff --git a/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_xbox.h b/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_xbox.h index a2c77098..e7166a5c 100644 --- a/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_xbox.h +++ b/thirdparty/rexglue-sdk/include/rex/filesystem/devices/stfs_xbox.h @@ -29,7 +29,7 @@ using rex::system::XLanguage; // Convert FAT timestamp to 100-nanosecond intervals since January 1, 1601 (UTC) inline uint64_t decode_fat_timestamp(const uint32_t date, const uint32_t time) { - struct tm tm = {0}; + struct tm tm = {}; // 80 is the difference between 1980 (FAT) and 1900 (tm); tm.tm_year = ((0xFE00 & date) >> 9) + 80; tm.tm_mon = ((0x01E0 & date) >> 5) - 1; diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/d3d12/graphics_system.h b/thirdparty/rexglue-sdk/include/rex/graphics/d3d12/graphics_system.h index 58dfc59e..3efe580a 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/d3d12/graphics_system.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/d3d12/graphics_system.h @@ -28,11 +28,8 @@ class D3D12GraphicsSystem : public GraphicsSystem { std::string name() const override; - X_STATUS Setup(runtime::FunctionDispatcher* function_dispatcher, - system::KernelState* kernel_state, ui::WindowedAppContext* app_context, - bool with_presentation) override; - protected: + void CreateProvider(bool with_presentation) override; std::unique_ptr CreateCommandProcessor() override; }; diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/format/ucode.h b/thirdparty/rexglue-sdk/include/rex/graphics/format/ucode.h index 8eb727d9..6011d3c5 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/format/ucode.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/format/ucode.h @@ -235,10 +235,10 @@ struct ControlFlowExecInstruction { uint32_t count_ : 3; uint32_t is_yield_ : 1; uint32_t sequence_ : 12; - uint32_t vc_hi_ : 4; // Vertex cache? + [[maybe_unused]] uint32_t vc_hi_ : 4; // Vertex cache? // Word 1: (16 bits) - uint32_t vc_lo_ : 2; + [[maybe_unused]] uint32_t vc_lo_ : 2; uint32_t : 7; // According to the description of Conditional_Execute_Predicates_No_Stall in // the IPR2015-00325 sequencer specification, the sequencer's control flow @@ -280,10 +280,10 @@ struct ControlFlowCondExecInstruction { uint32_t count_ : 3; uint32_t is_yield_ : 1; uint32_t sequence_ : 12; - uint32_t vc_hi_ : 4; // Vertex cache? + [[maybe_unused]] uint32_t vc_hi_ : 4; // Vertex cache? // Word 1: (16 bits) - uint32_t vc_lo_ : 2; + [[maybe_unused]] uint32_t vc_lo_ : 2; uint32_t bool_address_ : 8; uint32_t condition_ : 1; AddressingMode address_mode_ : 1; @@ -314,10 +314,10 @@ struct ControlFlowCondExecPredInstruction { uint32_t count_ : 3; uint32_t is_yield_ : 1; uint32_t sequence_ : 12; - uint32_t vc_hi_ : 4; // Vertex cache? + [[maybe_unused]] uint32_t vc_hi_ : 4; // Vertex cache? // Word 1: (16 bits) - uint32_t vc_lo_ : 2; + [[maybe_unused]] uint32_t vc_lo_ : 2; uint32_t : 7; uint32_t is_predicate_clean_ : 1; uint32_t condition_ : 1; @@ -457,7 +457,7 @@ struct ControlFlowCondJmpInstruction { // Word 1: (16 bits) uint32_t : 1; - uint32_t direction_ : 1; + [[maybe_unused]] uint32_t direction_ : 1; uint32_t bool_address_ : 8; uint32_t condition_ : 1; AddressingMode address_mode_ : 1; @@ -480,7 +480,7 @@ struct ControlFlowAllocInstruction { // Word 1: (16 bits) uint32_t : 8; - uint32_t is_unserialized_ : 1; + uint32_t : 1; // is_unserialized_ AllocType alloc_type_ : 2; uint32_t : 1; ControlFlowOpcode opcode_ : 4; diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/graphics_system.h b/thirdparty/rexglue-sdk/include/rex/graphics/graphics_system.h index fb4a84be..6ede3215 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/graphics_system.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/graphics_system.h @@ -58,10 +58,11 @@ class GraphicsSystem : public system::IGraphicsSystem { ::rex::ui::GraphicsProvider* provider() const override { return provider_.get(); } ::rex::ui::Presenter* presenter() const override { return presenter_.get(); } - virtual X_STATUS Setup(runtime::FunctionDispatcher* function_dispatcher, - system::KernelState* kernel_state, - ::rex::ui::WindowedAppContext* app_context, bool with_presentation); - virtual void Shutdown(); + X_STATUS SetupPresentation(::rex::ui::WindowedAppContext* app_context) override; + X_STATUS SetupGuestGpu(runtime::FunctionDispatcher* function_dispatcher, + system::KernelState* kernel_state) override; + bool has_presentation() const override { return presenter_ != nullptr; } + void Shutdown() override; // May be called from any thread any number of times, even during recovery // from a device loss. @@ -107,6 +108,10 @@ class GraphicsSystem : public system::IGraphicsSystem { protected: GraphicsSystem(); + // Backends build their provider here. Called lazily from either setup + // entry point; with_presentation is false only on headless guest-GPU paths. + virtual void CreateProvider(bool with_presentation) = 0; + virtual std::unique_ptr CreateCommandProcessor() = 0; static uint32_t ReadRegisterThunk(void* ppc_context, GraphicsSystem* gs, uint32_t addr); @@ -123,6 +128,7 @@ class GraphicsSystem : public system::IGraphicsSystem { system::KernelState* kernel_state_ = nullptr; ::rex::ui::WindowedAppContext* app_context_ = nullptr; std::unique_ptr<::rex::ui::GraphicsProvider> provider_; + bool provider_supports_presentation_ = false; uint32_t interrupt_callback_ = 0; uint32_t interrupt_callback_data_ = 0; diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/dxbc.h b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/dxbc.h index 10240e65..ed4452f4 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/dxbc.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/dxbc.h @@ -50,7 +50,7 @@ class DxbcShader : public Shader { const std::vector& GetTextureBindingsAfterTranslation() const { return texture_bindings_; } - const uint32_t GetUsedTextureMaskAfterTranslation() const { return used_texture_mask_; } + uint32_t GetUsedTextureMaskAfterTranslation() const { return used_texture_mask_; } static constexpr uint32_t kMaxSamplerBindingIndexBits = DxbcShaderTranslator::kMaxSamplerBindingIndexBits; diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/interpreter.h b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/interpreter.h index a29cfa55..9a90df25 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/interpreter.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/interpreter.h @@ -33,9 +33,9 @@ class ShaderInterpreter { class ExportSink { public: virtual ~ExportSink() = default; - virtual void AllocExport(ucode::AllocType type, uint32_t size) {} - virtual void Export(ucode::ExportRegister export_register, const float* value, - uint32_t value_mask) {} + virtual void AllocExport(ucode::AllocType /*type*/, uint32_t /*size*/) {} + virtual void Export(ucode::ExportRegister /*export_register*/, const float* /*value*/, + uint32_t /*value_mask*/) {} }; void SetTraceWriter(TraceWriter* new_trace_writer) { trace_writer_ = new_trace_writer; } diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/shader.h b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/shader.h index fc3de5d6..4cf38547 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/shader.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/shader.h @@ -1005,7 +1005,7 @@ class Shader { std::string ucode_disassembly_; std::vector vertex_bindings_; std::vector texture_bindings_; - ConstantRegisterMap constant_register_map_ = {0}; + ConstantRegisterMap constant_register_map_ = {}; std::set label_addresses_; uint32_t cf_pair_index_bound_ = 0; uint32_t register_static_address_bound_ = 0; diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/translator.h b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/translator.h index 1816e46d..340ced5d 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/translator.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/shader/translator.h @@ -29,13 +29,13 @@ class ShaderTranslator { virtual ~ShaderTranslator(); virtual uint64_t GetDefaultVertexShaderModification( - uint32_t dynamic_addressable_register_count, - Shader::HostVertexShaderType host_vertex_shader_type = + uint32_t /*dynamic_addressable_register_count*/, + Shader::HostVertexShaderType /*host_vertex_shader_type*/ = Shader::HostVertexShaderType::kVertex) const { return 0; } virtual uint64_t GetDefaultPixelShaderModification( - uint32_t dynamic_addressable_register_count) const { + uint32_t /*dynamic_addressable_register_count*/) const { return 0; } @@ -84,51 +84,52 @@ class ShaderTranslator { // Pre-process a control-flow instruction before anything else. virtual void PreProcessControlFlowInstructions( - std::vector instrs) {} + std::vector /*instrs*/) {} // Handles translation for control flow label addresses. // This is triggered once for each label required (due to control flow // operations) before any of the instructions within the target exec. - virtual void ProcessLabel(uint32_t cf_index) {} + virtual void ProcessLabel(uint32_t /*cf_index*/) {} // Handles translation for control flow nop instructions. - virtual void ProcessControlFlowNopInstruction(uint32_t cf_index) {} + virtual void ProcessControlFlowNopInstruction(uint32_t /*cf_index*/) {} // Handles the start of a control flow instruction at the given address. - virtual void ProcessControlFlowInstructionBegin(uint32_t cf_index) {} + virtual void ProcessControlFlowInstructionBegin(uint32_t /*cf_index*/) {} // Handles the end of a control flow instruction that began at the given // address. - virtual void ProcessControlFlowInstructionEnd(uint32_t cf_index) {} + virtual void ProcessControlFlowInstructionEnd(uint32_t /*cf_index*/) {} // Handles translation for control flow exec instructions prior to their // contained ALU/fetch instructions. - virtual void ProcessExecInstructionBegin(const ParsedExecInstruction& instr) {} + virtual void ProcessExecInstructionBegin(const ParsedExecInstruction& /*instr*/) {} // Handles translation for control flow exec instructions after their // contained ALU/fetch instructions. - virtual void ProcessExecInstructionEnd(const ParsedExecInstruction& instr) {} + virtual void ProcessExecInstructionEnd(const ParsedExecInstruction& /*instr*/) {} // Handles translation for loop start instructions. - virtual void ProcessLoopStartInstruction(const ParsedLoopStartInstruction& instr) {} + virtual void ProcessLoopStartInstruction(const ParsedLoopStartInstruction& /*instr*/) {} // Handles translation for loop end instructions. - virtual void ProcessLoopEndInstruction(const ParsedLoopEndInstruction& instr) {} + virtual void ProcessLoopEndInstruction(const ParsedLoopEndInstruction& /*instr*/) {} // Handles translation for function call instructions. - virtual void ProcessCallInstruction(const ParsedCallInstruction& instr) {} + virtual void ProcessCallInstruction(const ParsedCallInstruction& /*instr*/) {} // Handles translation for function return instructions. - virtual void ProcessReturnInstruction(const ParsedReturnInstruction& instr) {} + virtual void ProcessReturnInstruction(const ParsedReturnInstruction& /*instr*/) {} // Handles translation for jump instructions. - virtual void ProcessJumpInstruction(const ParsedJumpInstruction& instr) {} + virtual void ProcessJumpInstruction(const ParsedJumpInstruction& /*instr*/) {} // Handles translation for alloc instructions. Memory exports for eM# // indicated by export_eM must be performed, regardless of the alloc type. - virtual void ProcessAllocInstruction(const ParsedAllocInstruction& instr, uint8_t export_eM) {} + virtual void ProcessAllocInstruction(const ParsedAllocInstruction& /*instr*/, + uint8_t /*export_eM*/) {} // Handles translation for vertex fetch instructions. - virtual void ProcessVertexFetchInstruction(const ParsedVertexFetchInstruction& instr) {} + virtual void ProcessVertexFetchInstruction(const ParsedVertexFetchInstruction& /*instr*/) {} // Handles translation for texture fetch instructions. - virtual void ProcessTextureFetchInstruction(const ParsedTextureFetchInstruction& instr) {} + virtual void ProcessTextureFetchInstruction(const ParsedTextureFetchInstruction& /*instr*/) {} // Handles translation for ALU instructions. // memexport_eM_potentially_written_before needs to be handled by `kill` // instruction to make sure memory exports for the eM# writes earlier in // previous execs and the current exec are done before the invocation becomes // inactive. - virtual void ProcessAluInstruction(const ParsedAluInstruction& instr, - uint8_t memexport_eM_potentially_written_before) {} + virtual void ProcessAluInstruction(const ParsedAluInstruction& /*instr*/, + uint8_t /*memexport_eM_potentially_written_before*/) {} private: void TranslateControlFlowInstruction(const ucode::ControlFlowInstruction& cf); diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/texture/cache.h b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/texture/cache.h index ebd6f9f8..f25850ac 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/texture/cache.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/pipeline/texture/cache.h @@ -83,8 +83,9 @@ class TextureCache { void MarkRangeAsResolved(uint32_t start_unscaled, uint32_t length_unscaled); // Ensures the memory backing the range in the scaled resolve address space is // allocated and returns whether it is. - virtual bool EnsureScaledResolveMemoryCommitted(uint32_t start_unscaled, uint32_t length_unscaled, - uint32_t length_scaled_alignment_log2 = 0) { + virtual bool EnsureScaledResolveMemoryCommitted(uint32_t /*start_unscaled*/, + uint32_t /*length_unscaled*/, + uint32_t /*length_scaled_alignment_log2*/ = 0) { return false; } @@ -484,11 +485,11 @@ class TextureCache { // Whether the signed version of the texture has a different representation on // the host than its unsigned version (for example, if it's a fixed-point // texture emulated with a larger host pixel format). - virtual bool IsSignedVersionSeparateForFormat(TextureKey key) const { return false; } + virtual bool IsSignedVersionSeparateForFormat(TextureKey /*key*/) const { return false; } // Parameters like whether the texture is tiled and its dimensions are checked // externally, the implementation should take only format-related parameters // such as the format itself and the signedness into account. - virtual bool IsScaledResolveSupportedForFormat(TextureKey key) const { return false; } + virtual bool IsScaledResolveSupportedForFormat(TextureKey /*key*/) const { return false; } // For formats with less than 4 components, implementations normally should // replicate the last component into the non-existent ones, similar to what is // done for unused components of operands in shaders by Microsoft's Xbox 360 @@ -542,7 +543,7 @@ class TextureCache { } // Called when something in a texture binding is changed for the // implementation to update the internal dependencies of the binding. - virtual void UpdateTextureBindingsImpl(uint32_t fetch_constant_mask) {} + virtual void UpdateTextureBindingsImpl(uint32_t /*fetch_constant_mask*/) {} private: struct PendingTextureLoad { diff --git a/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/graphics_system.h b/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/graphics_system.h index 0611de5d..ad5bc3eb 100644 --- a/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/graphics_system.h +++ b/thirdparty/rexglue-sdk/include/rex/graphics/vulkan/graphics_system.h @@ -26,9 +26,8 @@ class VulkanGraphicsSystem : public GraphicsSystem { std::string name() const override; - X_STATUS Setup(runtime::FunctionDispatcher* function_dispatcher, - system::KernelState* kernel_state, ui::WindowedAppContext* app_context, - bool with_presentation) override; + protected: + void CreateProvider(bool with_presentation) override; private: std::unique_ptr CreateCommandProcessor() override; diff --git a/thirdparty/rexglue-sdk/include/rex/logging/api.h b/thirdparty/rexglue-sdk/include/rex/logging/api.h index 1e85ab7b..5f84121e 100644 --- a/thirdparty/rexglue-sdk/include/rex/logging/api.h +++ b/thirdparty/rexglue-sdk/include/rex/logging/api.h @@ -184,6 +184,14 @@ void RemoveSink(spdlog::sink_ptr sink); */ void RemoveSink(LogCategoryId category, spdlog::sink_ptr sink); +/** + * Replace the global console sink on every registered logger with `sink`. + * Pass nullptr to remove the console sink without replacement. + * + * @param sink New console sink, or nullptr to remove. + */ +void ReplaceConsoleSink(spdlog::sink_ptr sink); + /** * Update the format pattern on the stdout console sink. * diff --git a/thirdparty/rexglue-sdk/include/rex/ppc/context.h b/thirdparty/rexglue-sdk/include/rex/ppc/context.h index fe3306ae..9aa36c6b 100644 --- a/thirdparty/rexglue-sdk/include/rex/ppc/context.h +++ b/thirdparty/rexglue-sdk/include/rex/ppc/context.h @@ -45,6 +45,10 @@ struct PPCContext; // Function signature for recompiled PPC functions using PPCFunc = void(PPCContext& ctx, uint8_t* base); +namespace rex::runtime { +PPCFunc* ResolveIndirectFunction(uint32_t guest_address); +} // namespace rex::runtime + //============================================================================= // Compiler-Specific Intrinsics //============================================================================= diff --git a/thirdparty/rexglue-sdk/include/rex/ppc/image_info.h b/thirdparty/rexglue-sdk/include/rex/ppc/image_info.h index 170f6b8e..1dad663b 100644 --- a/thirdparty/rexglue-sdk/include/rex/ppc/image_info.h +++ b/thirdparty/rexglue-sdk/include/rex/ppc/image_info.h @@ -15,8 +15,17 @@ struct PPCFuncMapping; +namespace rex::system { +class KernelState; +} + namespace rex { +/** + * Callback for registering recompiled modules with KernelState (multi-binary projects). + */ +using RegisterModulesFunc = void (*)(system::KernelState*); + /// PPC image layout passed from the generated config header into ReXApp. struct PPCImageInfo { uint32_t code_base; @@ -25,6 +34,7 @@ struct PPCImageInfo { uint32_t image_size; const PPCFuncMapping* func_mappings; bool rexcrt_heap = false; ///< Set by codegen when [rexcrt] has heap functions + RegisterModulesFunc register_modules = nullptr; ///< Set by codegen for multi-binary projects }; } // namespace rex diff --git a/thirdparty/rexglue-sdk/include/rex/result.h b/thirdparty/rexglue-sdk/include/rex/result.h index 8e4f4ec6..8e9f2edf 100644 --- a/thirdparty/rexglue-sdk/include/rex/result.h +++ b/thirdparty/rexglue-sdk/include/rex/result.h @@ -34,6 +34,7 @@ enum class ErrorCategory { Validation, // Validation errors (e.g., unresolved functions) NotFound, // Resource not found NotImplemented, // Feature not implemented + UserAbort, // User declined an interactive prompt }; //============================================================================= diff --git a/thirdparty/rexglue-sdk/include/rex/system/function_dispatcher.h b/thirdparty/rexglue-sdk/include/rex/system/function_dispatcher.h index 02fad980..f5d56377 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/function_dispatcher.h +++ b/thirdparty/rexglue-sdk/include/rex/system/function_dispatcher.h @@ -15,7 +15,12 @@ #pragma once +#include +#include +#include #include +#include +#include #include #include @@ -30,8 +35,27 @@ namespace rex::runtime { class ExportResolver; class ThreadState; -class FunctionDispatcher { +/** + * Narrow registration interface used by generated DLLs. + */ +class IModuleRegistrar { public: + /** + * Returns false (and logs) if guest_address is outside every module range. + */ + virtual bool SetFunction(uint32_t guest_address, ::PPCFunc* func) = 0; + + protected: + ~IModuleRegistrar() = default; +}; + +class FunctionDispatcher : public IModuleRegistrar { + public: + /** + * Callback type for module registration functions. + */ + using RegisterFn = void (*)(IModuleRegistrar*); + FunctionDispatcher(memory::Memory* memory, ExportResolver* export_resolver); ~FunctionDispatcher(); @@ -42,33 +66,86 @@ class FunctionDispatcher { uint64_t ExecuteInterrupt(ThreadState* thread_state, uint32_t address, uint64_t args[], size_t arg_count); - // rexglue function table management + // Shared thunk region size per module. + static constexpr uint32_t kThunkReserveSize = 0x10000; // 64KB + + // rexglue function table management (per-module table at IMAGE_BASE + IMAGE_SIZE) + // Set is_entrypoint=true exactly once for the host-loaded entrypoint so + // AllocateThunk(caller_address=0) can route to its pool. bool InitializeFunctionTable(uint32_t code_base, uint32_t code_size, uint32_t image_base, - uint32_t image_size); - void SetFunction(uint32_t guest_address, ::PPCFunc* func); + uint32_t image_size, bool is_entrypoint = false); + bool SetFunction(uint32_t guest_address, ::PPCFunc* func) override; ::PPCFunc* GetFunction(uint32_t guest_address); - bool HasFunctionTable() const { return function_table_initialized_; } - uint32_t AllocateThunk(::PPCFunc* func); + bool HasAnyFunctionTable() const { return !module_tables_.empty(); } + /** + * caller_address must be inside a registered module, or 0 to mean "host- + * initiated, route to the entrypoint pool". + */ + uint32_t AllocateThunk(::PPCFunc* func, uint32_t caller_address); + + /** + * Returns the `code_base` of the module containing `guest_address`, + * or 0 if no module covers that address. + */ + uint32_t FindCallerModuleBase(uint32_t guest_address); + + /** + * Register a module while recording guest addresses written via SetFunction. + * `code_base` must equal the value previously passed to InitializeFunctionTable + * for the same module. + */ + void RegisterModule(const std::string& module_id, uint32_t code_base, RegisterFn register_func); + + /** + * Unregister `module_id`: clears its function-table entries, releases its + * thunk pool, and removes its per-module function table. Returns the + * cleared thunk-pool range `[lo, hi)` for external cache invalidation, or + * nullopt if the module was not registered. + */ + std::optional> UnregisterModule(const std::string& module_id); private: bool Execute(ThreadState* thread_state, uint32_t address); + struct ModuleTableInfo { + uint32_t code_base; + uint32_t code_size; + uint32_t image_base; + uint32_t image_size; + uint32_t next_thunk_address; + uint32_t thunk_limit; + }; + + ModuleTableInfo* FindModuleByAddress(uint32_t guest_address); + memory::Memory* memory_ = nullptr; ExportResolver* export_resolver_ = nullptr; rex::thread::global_critical_region global_critical_region_; - // rexglue function table + // Host-side function lookup. std::unordered_map function_table_; - uint32_t code_base_ = 0; - uint32_t code_size_ = 0; - uint32_t image_base_ = 0; - uint32_t image_size_ = 0; - bool function_table_initialized_ = false; - // Runtime thunk allocation (for XexGetProcedureAddress) - uint32_t next_thunk_address_ = 0; - uint32_t thunk_limit_ = 0; + // Per-module function table metadata. + std::vector module_tables_; + + // code_base of the entrypoint module, or 0 if not yet registered. + uint32_t entrypoint_code_base_ = 0; + + // Module recording for RegisterModule/UnregisterModule. + bool recording_ = false; + std::vector recording_addresses_; + + struct ModuleRegistration { + uint32_t code_base; + std::vector addresses; + }; + + // Recorded state per module, keyed by module_id. + std::unordered_map module_addresses_; + + // Protects dispatcher metadata during module registration and callback dispatch. + mutable std::recursive_mutex dispatch_mutex_; }; } // namespace rex::runtime diff --git a/thirdparty/rexglue-sdk/include/rex/system/guest_path.h b/thirdparty/rexglue-sdk/include/rex/system/guest_path.h new file mode 100644 index 00000000..0bd80d94 --- /dev/null +++ b/thirdparty/rexglue-sdk/include/rex/system/guest_path.h @@ -0,0 +1,30 @@ +/** + * @file rex/system/guest_path.h + * @brief Guest path normalization for Xbox 360 VFS paths + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#pragma once + +#include +#include + +namespace rex::system { + +/** + * Normalize a guest path to the canonical form used as a key for runtime + * lookups: device prefix stripped, separators canonicalized to forward + * slashes, redundant `..` segments resolved, leading slashes removed, + * and ASCII lowercased. Codegen-side canonicalization + * (rex::codegen::CanonicalizeModuleGuestPath) layers a project-scoped prefix + * strip on top of this; both must agree on this base form for runtime + * lookups to find the registered module. + */ +std::string NormalizeGuestPath(std::string_view path); + +} // namespace rex::system diff --git a/thirdparty/rexglue-sdk/include/rex/system/interfaces/graphics.h b/thirdparty/rexglue-sdk/include/rex/system/interfaces/graphics.h index 43348194..26270b01 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/interfaces/graphics.h +++ b/thirdparty/rexglue-sdk/include/rex/system/interfaces/graphics.h @@ -47,9 +47,34 @@ struct GraphicsSwapSubmission { class IGraphicsSystem { public: virtual ~IGraphicsSystem() = default; - virtual X_STATUS Setup(runtime::FunctionDispatcher* function_dispatcher, - KernelState* kernel_state, ui::WindowedAppContext* app_context, - bool with_presentation) = 0; + + // Build the provider + presenter. Safe to call standalone (without a + // Runtime) to stand up a window + ImGui for an installer. Idempotent. + // Must be called before SetupGuestGpu if presentation is desired: some + // backends (e.g. Vulkan) bake swapchain support into the provider, and + // a headless provider from SetupGuestGpu cannot be upgraded in place. + virtual X_STATUS SetupPresentation(ui::WindowedAppContext* app_context) = 0; + + // Wire the GPU into the guest address space: MMIO, command processor, + // vsync worker. Needs the Runtime's dispatcher + kernel state. If + // SetupPresentation has not been called, a headless provider is built. + virtual X_STATUS SetupGuestGpu(runtime::FunctionDispatcher* function_dispatcher, + KernelState* kernel_state) = 0; + + virtual bool has_presentation() const = 0; + + // One-shot convenience for callers that don't care about the split. + X_STATUS Setup(runtime::FunctionDispatcher* function_dispatcher, KernelState* kernel_state, + ui::WindowedAppContext* app_context, bool with_presentation) { + if (with_presentation && !has_presentation()) { + X_STATUS status = SetupPresentation(app_context); + if (XFAILED(status)) { + return status; + } + } + return SetupGuestGpu(function_dispatcher, kernel_state); + } + virtual void Shutdown() = 0; virtual ui::GraphicsProvider* provider() const = 0; virtual ui::Presenter* presenter() const = 0; diff --git a/thirdparty/rexglue-sdk/include/rex/system/kernel_module.h b/thirdparty/rexglue-sdk/include/rex/system/kernel_module.h index dd2f7c6e..f05e0b99 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/kernel_module.h +++ b/thirdparty/rexglue-sdk/include/rex/system/kernel_module.h @@ -1,7 +1,13 @@ #pragma once /** - * ReXGlue runtime - AC6 Recompilation project - * Copyright (c) 2026 Tom Clay. All rights reserved. + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + * + * @modified Tom Clay, 2026 - Adapted for ReXGlue runtime */ #include @@ -21,9 +27,14 @@ class KernelModule : public XModule { const std::string& path() const override { return path_; } const std::string& name() const override { return name_; } - uint32_t GetProcAddressByOrdinal(uint16_t ordinal) override; + uint32_t GetProcAddressByOrdinal(uint16_t ordinal, uint32_t caller_address = 0) override; uint32_t GetProcAddressByName(const std::string_view name) override; + /** + * Erase any cached thunks whose guest address falls in [lo, hi). + */ + void InvalidateThunkCacheInRange(uint32_t lo, uint32_t hi); + protected: rex::runtime::ExportResolver* export_resolver_; @@ -32,8 +43,20 @@ class KernelModule : public XModule { rex::thread::global_critical_region global_critical_region_; - // Cache of ordinal -> thunk guest address (for XexGetProcedureAddress) - std::unordered_map thunk_cache_; + // Cache of (caller_module_base, ordinal) -> thunk guest address. + struct ThunkKey { + uint32_t caller_module_base; + uint16_t ordinal; + bool operator==(const ThunkKey& other) const { + return caller_module_base == other.caller_module_base && ordinal == other.ordinal; + } + }; + struct ThunkKeyHash { + std::size_t operator()(const ThunkKey& k) const noexcept { + return std::hash{}((uint64_t(k.caller_module_base) << 16) | k.ordinal); + } + }; + std::unordered_map thunk_cache_; }; } // namespace rex::system diff --git a/thirdparty/rexglue-sdk/include/rex/system/mmio_handler.h b/thirdparty/rexglue-sdk/include/rex/system/mmio_handler.h index b74f8718..ee52fe17 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/mmio_handler.h +++ b/thirdparty/rexglue-sdk/include/rex/system/mmio_handler.h @@ -51,7 +51,7 @@ class MMIOHandler { const void* host_to_guest_virtual_context, AccessViolationCallback access_violation_callback, void* access_violation_callback_context); - static MMIOHandler* global_handler() { return global_handler_; } + static MMIOHandler* global_handler(); bool RegisterRange(uint32_t virtual_address, uint32_t mask, uint32_t size, void* context, MMIOReadCallback read_callback, MMIOWriteCallback write_callback); diff --git a/thirdparty/rexglue-sdk/include/rex/system/shared_library.h b/thirdparty/rexglue-sdk/include/rex/system/shared_library.h new file mode 100644 index 00000000..eda3234b --- /dev/null +++ b/thirdparty/rexglue-sdk/include/rex/system/shared_library.h @@ -0,0 +1,37 @@ +/** + * @file rex/system/shared_library.h + * @brief Platform-agnostic shared library loader + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#pragma once + +#include + +namespace rex::system { + +class SharedLibrary { + public: + SharedLibrary() = default; + ~SharedLibrary(); + + SharedLibrary(const SharedLibrary&) = delete; + SharedLibrary& operator=(const SharedLibrary&) = delete; + SharedLibrary(SharedLibrary&& other) noexcept; + SharedLibrary& operator=(SharedLibrary&& other) noexcept; + + bool Load(const std::string& name); + void* GetSymbol(const char* name); + void Close(); + bool is_loaded() const { return handle_ != nullptr; } + + private: + void* handle_ = nullptr; +}; + +} // namespace rex::system diff --git a/thirdparty/rexglue-sdk/include/rex/system/user_module.h b/thirdparty/rexglue-sdk/include/rex/system/user_module.h index 992f0523..0374e50f 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/user_module.h +++ b/thirdparty/rexglue-sdk/include/rex/system/user_module.h @@ -60,7 +60,7 @@ class UserModule : public XModule { X_STATUS LoadFromMemory(const void* addr, const size_t length); X_STATUS Unload(); - uint32_t GetProcAddressByOrdinal(uint16_t ordinal) override; + uint32_t GetProcAddressByOrdinal(uint16_t ordinal, uint32_t caller_address = 0) override; uint32_t GetProcAddressByName(const std::string_view name) override; X_STATUS GetSection(const std::string_view name, uint32_t* out_section_data, uint32_t* out_section_size) override; diff --git a/thirdparty/rexglue-sdk/include/rex/system/xmemory.h b/thirdparty/rexglue-sdk/include/rex/system/xmemory.h index dc9fa8ba..3c3969a1 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/xmemory.h +++ b/thirdparty/rexglue-sdk/include/rex/system/xmemory.h @@ -1,7 +1,13 @@ #pragma once /** - * ReXGlue runtime - AC6 Recompilation project - * Copyright (c) 2026 Tom Clay. All rights reserved. + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + * + * @modified Tom Clay, 2026 - Adapted for ReXGlue runtime */ #include @@ -16,8 +22,35 @@ #include #include +namespace rex::stream { +class ByteStream; +} // namespace rex::stream + +namespace rex::memory::detail { + +/// Compensates for Windows 64KB allocation granularity on the 0xE0 physical heap. +/// The backing file maps the 0xE0 heap at a 0x1000-byte offset, but MapViewOfFileEx +/// rounds down to 64KB boundaries. Linux mmap handles 4KB offsets natively. +constexpr u32 PhysicalHostOffset([[maybe_unused]] u32 guest_addr) noexcept { +#if REX_PLATFORM_WIN32 + return (guest_addr >= 0xE0000000u) ? 0x1000u : 0u; +#else + return 0u; +#endif +} + +} // namespace rex::memory::detail + namespace rex::memory { +/// Lightweight guest-to-host pointer translation using the memory base. +/// For hooks and kernel code operating with the base pointer directly. +/// Same raw arithmetic as recompiled code; no Memory* or heap lookup needed. +template +inline T GuestPtr(u8* base, u32 guest_address) noexcept { + return reinterpret_cast(base + guest_address + detail::PhysicalHostOffset(guest_address)); +} + class Memory; enum SystemHeapFlag : uint32_t { @@ -182,6 +215,8 @@ class BaseHeap { // range. rex::memory::PageAccess QueryRangeAccess(uint32_t low_address, uint32_t high_address); + bool Save(stream::ByteStream* stream); + bool Restore(stream::ByteStream* stream); void Reset(); @@ -466,32 +501,21 @@ class Memory { // Dumps a map of all allocated memory to the log. void DumpMap(); + bool Save(stream::ByteStream* stream); + bool Restore(stream::ByteStream* stream); //========================================================================== // Recompiled Code Function Table API //========================================================================== - // These methods support static recompilation by providing a function - // dispatch table stored in guest memory at IMAGE_BASE + IMAGE_SIZE. - // The table is indexed by (guest_addr - code_base) * 2, allowing 8-byte - // pointers for each 4-byte-aligned guest address. - - // Initialize the function table region for recompiled code dispatch. - // Must be called before SetFunction/GetFunction. - // Returns false if memory allocation fails. + // Per-module function dispatch table at IMAGE_BASE + IMAGE_SIZE, indexed by + // (guest_addr - code_base) * 2. Internally locked: callers may invoke from + // any thread. bool InitializeFunctionTable(uint32_t code_base, uint32_t code_size, uint32_t image_base, uint32_t image_size); - - // Register a host function for a guest address. - // The function will be called when recompiled code does an indirect call - // to this address via PPC_LOOKUP_FUNC/PPC_CALL_INDIRECT_FUNC. - void SetFunction(uint32_t guest_address, PPCFunc* host_function); - - // Get the registered host function for a guest address. - // Returns nullptr if no function is registered. - PPCFunc* GetFunction(uint32_t guest_address) const; - - // Check if the function table has been initialized. - bool HasFunctionTable() const { return function_table_base_ != 0; } + bool DestroyFunctionTable(uint32_t code_base); + // Returns false if guest_address is outside every registered module range. + bool SetFunction(uint32_t guest_address, PPCFunc* host_function); + bool HasAnyFunctionTable() const; private: int MapViews(uint8_t* mapping_base); @@ -511,11 +535,14 @@ class Memory { uint8_t* virtual_membase_ = nullptr; uint8_t* physical_membase_ = nullptr; - // Recompiled code function table configuration - uint32_t function_table_base_ = 0; // Guest address of function table (IMAGE_BASE + IMAGE_SIZE) - uint32_t function_code_base_ = 0; // CODE_BASE for offset calculation - uint32_t function_code_size_ = 0; // CODE_SIZE for bounds checking - uint32_t function_thunk_reserve_ = 0; // Extra space reserved for runtime thunks + struct FunctionTableEntry { + uint32_t table_base; + uint32_t code_base; + uint32_t code_size; + uint32_t thunk_reserve; + }; + std::vector function_tables_; + mutable std::mutex function_tables_mutex_; rex::memory::FileMappingHandle mapping_ = rex::memory::kFileMappingHandleInvalid; uint8_t* mapping_base_ = nullptr; diff --git a/thirdparty/rexglue-sdk/include/rex/system/xmodule.h b/thirdparty/rexglue-sdk/include/rex/system/xmodule.h index 28ec7327..3d47a4a8 100644 --- a/thirdparty/rexglue-sdk/include/rex/system/xmodule.h +++ b/thirdparty/rexglue-sdk/include/rex/system/xmodule.h @@ -64,7 +64,7 @@ class XModule : public XObject { rex::runtime::Module* processor_module() const { return processor_module_; } uint32_t hmodule_ptr() const { return hmodule_ptr_; } - virtual uint32_t GetProcAddressByOrdinal(uint16_t ordinal) = 0; + virtual uint32_t GetProcAddressByOrdinal(uint16_t ordinal, uint32_t caller_address = 0) = 0; virtual uint32_t GetProcAddressByName(const std::string_view name) = 0; virtual X_STATUS GetSection(const std::string_view name, uint32_t* out_section_data, uint32_t* out_section_size); diff --git a/thirdparty/rexglue-sdk/include/rex/ui/imgui_drawer.h b/thirdparty/rexglue-sdk/include/rex/ui/imgui_drawer.h index ff323fba..c72b6283 100644 --- a/thirdparty/rexglue-sdk/include/rex/ui/imgui_drawer.h +++ b/thirdparty/rexglue-sdk/include/rex/ui/imgui_drawer.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include struct ImDrawData; +struct ImFontAtlas; struct ImGuiContext; struct ImGuiIO; enum ImGuiKey : int; @@ -36,7 +38,8 @@ class Window; class ImGuiDrawer : public WindowInputListener, public UIDrawer { public: - ImGuiDrawer(Window* window, size_t z_order); + using FontSetupCallback = std::function; + ImGuiDrawer(Window* window, size_t z_order, FontSetupCallback font_setup = nullptr); ~ImGuiDrawer(); ImGuiIO& GetIO(); @@ -85,6 +88,7 @@ class ImGuiDrawer : public WindowInputListener, public UIDrawer { Window* window_; size_t z_order_; + FontSetupCallback font_setup_; ImGuiContext* internal_state_ = nullptr; diff --git a/thirdparty/rexglue-sdk/resources/templates/codegen/_indirect_call.inja b/thirdparty/rexglue-sdk/resources/templates/codegen/_indirect_call.inja new file mode 100644 index 00000000..d4b1830b --- /dev/null +++ b/thirdparty/rexglue-sdk/resources/templates/codegen/_indirect_call.inja @@ -0,0 +1,38 @@ +//============================================================================= +// Indirect Call Dispatch +// +// REX_LOOKUP_FUNC indexes into the per-module dispatch table at +// IMAGE_BASE + IMAGE_SIZE. REX_CALL_INDIRECT_FUNC bounds-checks the target +// against the caller's [REX_CODE_BASE, REX_CODE_BASE + REX_CODE_SIZE + +// REX_THUNK_RESERVE_SIZE) range; out-of-bounds targets and in-bounds +// unregistered slots fall back to rex::runtime::ResolveIndirectFunction, +// which queries the global FunctionDispatcher across every loaded module. +//============================================================================= + +#ifdef REXGLUE_ENABLE_PROFILING +#include +#define REX_PROFILE_INDIRECT_DISPATCH() PROFILE_FUNCTION_DISPATCHED() +#else +#define REX_PROFILE_INDIRECT_DISPATCH() +#endif + +#define REX_LOOKUP_FUNC(x, y) \ + (*(PPCFunc**)(x + REX_IMAGE_BASE + REX_IMAGE_SIZE + (u64(u32(y) - REX_CODE_BASE) * 2))) + +#define REX_CALL_INDIRECT_FUNC(x) \ + do { \ + REX_PROFILE_INDIRECT_DISPATCH(); \ + uint32_t rex_indirect_target_ = (uint32_t)(x); \ + PPCFunc* rex_indirect_func_; \ + if ((uint32_t)(rex_indirect_target_ - REX_CODE_BASE) < \ + REX_CODE_SIZE + REX_THUNK_RESERVE_SIZE) [[likely]] { \ + rex_indirect_func_ = REX_LOOKUP_FUNC(base, rex_indirect_target_); \ + } else { \ + rex_indirect_func_ = nullptr; \ + } \ + if (!rex_indirect_func_) [[unlikely]] { \ + ctx.last_indirect_target = rex_indirect_target_; \ + rex_indirect_func_ = rex::runtime::ResolveIndirectFunction(rex_indirect_target_); \ + } \ + rex_indirect_func_(ctx, base); \ + } while (0) diff --git a/thirdparty/rexglue-sdk/resources/templates/codegen/dll_targets_cmake.inja b/thirdparty/rexglue-sdk/resources/templates/codegen/dll_targets_cmake.inja new file mode 100644 index 00000000..04865948 --- /dev/null +++ b/thirdparty/rexglue-sdk/resources/templates/codegen/dll_targets_cmake.inja @@ -0,0 +1,32 @@ +# Auto-generated by rexglue codegen - DO NOT EDIT +# DLL module shared library targets + +{% for mod in dll_modules %} +# DLL module: {{ mod.target_name }} +if(EXISTS "{{ cmake_var("CMAKE_CURRENT_SOURCE_DIR") }}/{{ mod.output_dir }}/sources.cmake") + set(_REXGLUE_RESTORE_GENERATED_SOURCES FALSE) + if(DEFINED GENERATED_SOURCES) + set(_REXGLUE_PREVIOUS_GENERATED_SOURCES {{ cmake_var("GENERATED_SOURCES") }}) + set(_REXGLUE_RESTORE_GENERATED_SOURCES TRUE) + endif() + include({{ mod.output_dir }}/sources.cmake) + set(DLL_SOURCES_{{ mod.target_name }} {{ cmake_var("GENERATED_SOURCES") }}) + if(_REXGLUE_RESTORE_GENERATED_SOURCES) + set(GENERATED_SOURCES {{ cmake_var("_REXGLUE_PREVIOUS_GENERATED_SOURCES") }}) + else() + unset(GENERATED_SOURCES) + endif() + unset(_REXGLUE_PREVIOUS_GENERATED_SOURCES) + unset(_REXGLUE_RESTORE_GENERATED_SOURCES) + add_library({{ mod.lib_name }} SHARED {{ cmake_var("DLL_SOURCES_" + mod.target_name) }}) + target_include_directories({{ mod.lib_name }} PRIVATE + {{ cmake_var("CMAKE_CURRENT_SOURCE_DIR") }} + {{ cmake_var("CMAKE_CURRENT_SOURCE_DIR") }}/{{ mod.output_dir }} + ) + target_link_libraries({{ mod.lib_name }} PRIVATE rex::runtime) + set_target_properties({{ mod.lib_name }} PROPERTIES + CXX_VISIBILITY_PRESET hidden + ) + rexglue_configure_module_target({{ mod.lib_name }} HOST {{ cmake_var("REXGLUE_HOST_TARGET") }}) +endif() +{% endfor %} diff --git a/thirdparty/rexglue-sdk/resources/templates/codegen/module_registry_cpp.inja b/thirdparty/rexglue-sdk/resources/templates/codegen/module_registry_cpp.inja new file mode 100644 index 00000000..e52471ae --- /dev/null +++ b/thirdparty/rexglue-sdk/resources/templates/codegen/module_registry_cpp.inja @@ -0,0 +1,14 @@ +//============================================================================= +// ReXGlue Generated - {{ project }} Module Registry +//============================================================================= + +#include + +void RegisterRecompiledModules(rex::system::KernelState* kernel_state) { +{% for mod in dll_modules %} + kernel_state->RegisterRecompiledModule( + "{{ mod.pe_name }}", + "{{ mod.guest_path }}", + "{{ mod.shared_lib_name }}"); +{% endfor %} +} diff --git a/thirdparty/rexglue-sdk/resources/templates/codegen/register_cpp.inja b/thirdparty/rexglue-sdk/resources/templates/codegen/register_cpp.inja new file mode 100644 index 00000000..a864dc1d --- /dev/null +++ b/thirdparty/rexglue-sdk/resources/templates/codegen/register_cpp.inja @@ -0,0 +1,21 @@ +//============================================================================= +// ReXGlue Generated - {{ project }} Function Registration +//============================================================================= + +#include "{{ project }}_init.h" +#include + +{% if is_dll %} +#ifdef _WIN32 +#define REX_MODULE_EXPORT __declspec(dllexport) +#else +#define REX_MODULE_EXPORT __attribute__((visibility("default"))) +#endif + +extern "C" REX_MODULE_EXPORT +void ReXModule_Register(rex::runtime::IModuleRegistrar* registrar) { +{% else %} +void {{ project }}_RegisterFunctions(rex::runtime::IModuleRegistrar* registrar) { +{% endif %} +{% for fn in functions %}{% if not fn.below_code_base or fn.is_import %} registrar->SetFunction({{ fn.address }}, {{ fn.name }}); +{% endif %}{% endfor %}} diff --git a/thirdparty/rexglue-sdk/resources/templates/codegen/sources_cmake.inja b/thirdparty/rexglue-sdk/resources/templates/codegen/sources_cmake.inja index 9e1b2073..34b2860d 100644 --- a/thirdparty/rexglue-sdk/resources/templates/codegen/sources_cmake.inja +++ b/thirdparty/rexglue-sdk/resources/templates/codegen/sources_cmake.inja @@ -7,7 +7,9 @@ set(GENERATED_SOURCES {{ cmake_var("CMAKE_CURRENT_LIST_DIR") }}/{{ project }}_config.cpp {{ cmake_var("CMAKE_CURRENT_LIST_DIR") }}/{{ project }}_init.cpp -{% for file in recomp_files %} {{ cmake_var("CMAKE_CURRENT_LIST_DIR") }}/{{ file }} + {{ cmake_var("CMAKE_CURRENT_LIST_DIR") }}/{{ project }}_register.cpp +{% if has_dll_modules and not is_dll %} {{ cmake_var("CMAKE_CURRENT_LIST_DIR") }}/module_registry.cpp +{% endif %}{% for file in recomp_files %} {{ cmake_var("CMAKE_CURRENT_LIST_DIR") }}/{{ file }} {% endfor %}) {% if has_icon %} # Windows application icon resource extracted from the Xbox 360 executable. diff --git a/thirdparty/rexglue-sdk/src/codegen/CMakeLists.txt b/thirdparty/rexglue-sdk/src/codegen/CMakeLists.txt index e34fa2fe..4faa4337 100644 --- a/thirdparty/rexglue-sdk/src/codegen/CMakeLists.txt +++ b/thirdparty/rexglue-sdk/src/codegen/CMakeLists.txt @@ -24,6 +24,7 @@ set(CODEGEN_CORE_SOURCES phase_merge.cpp phase_validate.cpp template_registry.cpp + manifest.cpp ) # PPC architecture sources (merged from rexarch) @@ -85,4 +86,3 @@ target_link_libraries(rexcodegen xxHash::xxhash simde ) - diff --git a/thirdparty/rexglue-sdk/src/codegen/analyze.cpp b/thirdparty/rexglue-sdk/src/codegen/analyze.cpp index ebe92b44..c5f3d142 100644 --- a/thirdparty/rexglue-sdk/src/codegen/analyze.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/analyze.cpp @@ -23,27 +23,33 @@ namespace rex::codegen { -Result Analyze(CodegenContext& ctx) { - REXCODEGEN_INFO("Analyze: starting analysis..."); +Result Analyze(CodegenContext& ctx, ProgressReporter* reporter) { + REXCODEGEN_TRACE("Analyze: starting analysis..."); ctx.initDecoded(); - REXCODEGEN_INFO("Analyze: decoded {} instructions across {} code regions", - ctx.decoded().instructionCount(), ctx.decoded().codeRegions().size()); + REXCODEGEN_TRACE("Analyze: decoded {} instructions across {} code regions", + ctx.decoded().instructionCount(), ctx.decoded().codeRegions().size()); // 1. Register entry points (imports, helpers, config, pdata) - auto regResult = phases::Register(ctx); + if (reporter) + reporter->phaseChanged("Register"); + auto regResult = phases::Register(ctx, reporter); if (!regResult) { return regResult; } // 2. Scan binary into code/data regions - auto scanResult = phases::Scan(ctx); + if (reporter) + reporter->phaseChanged("Scan"); + auto scanResult = phases::Scan(ctx, reporter); if (!scanResult) { return scanResult; } // 3. Discover function blocks iteratively (includes vtable scan) - auto discoverResult = phases::Discover(ctx); + if (reporter) + reporter->phaseChanged("Discover"); + auto discoverResult = phases::Discover(ctx, reporter); if (!discoverResult) { return discoverResult; } @@ -53,25 +59,31 @@ Result Analyze(CodegenContext& ctx) { // functionPointerScan(ctx); // 4. Gap fill uncovered regions + discover blocks for gap-filled functions + cleanup - auto gapFillResult = phases::GapFill(ctx); + if (reporter) + reporter->phaseChanged("GapFill"); + auto gapFillResult = phases::GapFill(ctx, reporter); if (!gapFillResult) { return gapFillResult; } // 5. Merge: resolve jumps and seal functions - auto mergeResult = phases::Merge(ctx); + if (reporter) + reporter->phaseChanged("Merge"); + auto mergeResult = phases::Merge(ctx, reporter); if (!mergeResult) { return mergeResult; } // 6. Validate - auto validateResult = phases::Validate(ctx); + if (reporter) + reporter->phaseChanged("Validate"); + auto validateResult = phases::Validate(ctx, reporter); if (!validateResult) { return validateResult; } - REXCODEGEN_INFO("Analyze: complete - {} functions ready for code generation", - ctx.graph.functionCount()); + REXCODEGEN_TRACE("Analyze: complete - {} functions ready for code generation", + ctx.graph.functionCount()); return Ok(); } diff --git a/thirdparty/rexglue-sdk/src/codegen/builders/vector.cpp b/thirdparty/rexglue-sdk/src/codegen/builders/vector.cpp index 641b8df6..27e83296 100644 --- a/thirdparty/rexglue-sdk/src/codegen/builders/vector.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/builders/vector.cpp @@ -1292,15 +1292,46 @@ bool build_vpkd3d128(BuilderContext& ctx) { case 5: // float16_4 { - // Pack 4 elements into 64 bits (4 x 16-bit floats) - // Guest element 0 goes to highest 16-bit position, element 3 to lowest - // Output u16 index = (3-i) + 2*shift for element i - if (ctx.insn.operands[3] != 2 || ctx.insn.operands[4] > 2) - REXCODEGEN_WARN("Unexpected float16_4 pack instruction at {:X}", ctx.base); + // NOTE: These combinations come from game traces (heuristic handling), not the official spec. + // The spec only defines the encoding, not the exact semantics of y (mask) and z (shift) here. + // Anyone reading this later should know it may need extending if new combos turn up in other + // games. Combinations observed so far: mask=2, shift=0 → write u16[3..0], zero u16[4..7] + // mask=2, shift=2 → write u16[7..4], zero u16[3..0] (first pass) + // mask=3, shift=0 → write u16[3..0] without zeroing (second pass, preserves the upper half) + uint32_t mask = ctx.insn.operands[3]; + uint32_t shift = ctx.insn.operands[4]; + + // Guard fix: The shift bound was too loose (shift=3 would cause dstIdx to reach 9, an OOB + // store). We also explicitly reject shift == 1 since the clear block below only handles 0 + // and 2. Furthermore, we explicitly warn on unexpected mask values (e.g., 0, 1) to avoid + // silent fallthroughs and silently generating miscompiled code. + if ((shift != 0 && shift != 2) || (mask != 2 && mask != 3)) { + REXCODEGEN_WARN("Unexpected float16_4 pack instruction at {:X} (mask={}, shift={})", + ctx.base, mask, shift); + // Emits a debug trap in the generated code to catch this at runtime. + ctx.println("\t__builtin_debugtrap();"); + return true; + } + + // mask=2: before writing, clear the half that will NOT be written. + // Shift is guaranteed to be 0 or 2 at this point due to the guard above. + if (mask == 2) { + // Optimization: Emit a single u64 write instead of two u32 writes. + // shift=0 → clears upper half u64[1] + // shift=2 → clears lower half u64[0] + size_t clearU64Start = (shift == 0) ? 1 : 0; + ctx.println("\t{}.u64[{}] = 0;", ctx.v(ctx.insn.operands[0]), clearU64Start); + } + + // Invariant: dstIdx must stay under 8 (valid u16 lanes are 0..7). + // Capping the shift to 0 or 2 in the guard check above ensures this is safe: + // it restricts the max dstIdx to (3 - 0) + (2 * 2) = 7. + // Do not widen the shift bounds without adjusting this logic. for (size_t i = 0; i < 4; i++) { - size_t srcIdx = 3 - i; // Guest element i is at host array index 3-i - size_t dstIdx = (3 - i) + (2 * ctx.insn.operands[4]); // Output also reversed + size_t srcIdx = 3 - i; + size_t dstIdx = (3 - i) + (2 * shift); + ctx.println("\t{}.u32 = ({}.u32[{}]&0x7FFFFFFF);", ctx.temp(), ctx.v(ctx.insn.operands[1]), srcIdx); ctx.println( diff --git a/thirdparty/rexglue-sdk/src/codegen/codegen.cpp b/thirdparty/rexglue-sdk/src/codegen/codegen.cpp index 210c715e..0fcf6358 100644 --- a/thirdparty/rexglue-sdk/src/codegen/codegen.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/codegen.cpp @@ -77,14 +77,22 @@ Result CodegenPipeline::Create(const std::filesystem::path& con } Result CodegenPipeline::Run(bool force) { - // Phase 1: Analyze (builds and validates function graph) + auto result = RunAnalyze(); + if (!result) + return result; + return RunWrite(force); +} + +Result CodegenPipeline::RunAnalyze() { auto analyzeResult = Analyze(*ctx_); if (!analyzeResult) { REXLOG_ERROR("Analysis failed: {}", analyzeResult.error().message); return analyzeResult; } + return Ok(); +} - // Phase 2: Generate C++ output +Result CodegenPipeline::RunWrite(bool force) { CodegenWriter writer(*ctx_, runtime_.get()); if (!writer.write(force)) return Err(ErrorCategory::Validation, "Code generation failed."); diff --git a/thirdparty/rexglue-sdk/src/codegen/codegen_flags.h b/thirdparty/rexglue-sdk/src/codegen/codegen_flags.h index 648d57eb..27b80db0 100644 --- a/thirdparty/rexglue-sdk/src/codegen/codegen_flags.h +++ b/thirdparty/rexglue-sdk/src/codegen/codegen_flags.h @@ -33,7 +33,3 @@ REXCVAR_DECLARE(uint32_t, max_seh_scope_entries); REXCVAR_DECLARE(uint32_t, backward_scan_limit); REXCVAR_DECLARE(uint32_t, max_jump_table_entries); REXCVAR_DECLARE(uint32_t, max_blocks_per_function); - -// Codegen (moved from main.cpp) -REXCVAR_DECLARE(bool, force); -REXCVAR_DECLARE(bool, enable_exception_handlers); diff --git a/thirdparty/rexglue-sdk/src/codegen/config.cpp b/thirdparty/rexglue-sdk/src/codegen/config.cpp index 09687fa3..47ccd268 100644 --- a/thirdparty/rexglue-sdk/src/codegen/config.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/config.cpp @@ -122,6 +122,9 @@ void ApplyToml(const toml::table& toml, RecompilerConfig& cfg, const std::string hasBool("non_volatile_as_local"), "non_volatile_as_local"); MergeBool(cfg.generateExceptionHandlers, toml["generate_exception_handlers"].value_or(false), hasBool("generate_exception_handlers"), "generate_exception_handlers"); + if (hasBool("is_dll")) { + cfg.isDll = toml["is_dll"].value_or(false); + } // Integer scalars (only override if present) if (auto v = toml["longjmp_address"].value()) { @@ -385,20 +388,18 @@ void ApplyToml(const toml::table& toml, RecompilerConfig& cfg, const std::string } } -// --------------------------------------------------------------------------- -// Recursive include loader -// --------------------------------------------------------------------------- +bool ApplyTableWithIncludes(const toml::table& tbl, const std::filesystem::path& base_dir, + RecompilerConfig& cfg, std::set& visited, uint32_t depth, + const std::string& description); bool LoadRecursive(const std::filesystem::path& filePath, RecompilerConfig& cfg, std::set& visited, uint32_t depth) { - // Depth check if (depth > kMaxIncludeDepth) { REXCODEGEN_ERROR("[config] include depth exceeds maximum ({}) at: {}", kMaxIncludeDepth, filePath.string()); return false; } - // Canonical path for cycle detection std::error_code ec; auto canonical = std::filesystem::canonical(filePath, ec); if (ec) { @@ -406,15 +407,12 @@ bool LoadRecursive(const std::filesystem::path& filePath, RecompilerConfig& cfg, return false; } std::string canonicalStr = canonical.string(); - - // Circular include detection if (visited.contains(canonicalStr)) { REXCODEGEN_ERROR("[config] circular include detected: {}", canonicalStr); return false; } visited.insert(canonicalStr); - // Parse this file toml::table toml; try { toml = toml::parse_file(canonicalStr); @@ -422,20 +420,22 @@ bool LoadRecursive(const std::filesystem::path& filePath, RecompilerConfig& cfg, REXCODEGEN_ERROR("Failed to parse config file '{}': {}", canonicalStr, e.what()); return false; } - REXCODEGEN_DEBUG("[config] loaded: {}", filePath.filename().string()); + return ApplyTableWithIncludes(toml, canonical.parent_path(), cfg, visited, depth, + filePath.filename().string()); +} - // Process includes first (depth-first), before applying this file's values. - // This means this file's values override anything set by included files. - auto parentDir = canonical.parent_path(); - - if (auto* includesArray = toml["includes"].as_array()) { +bool ApplyTableWithIncludes(const toml::table& tbl, const std::filesystem::path& base_dir, + RecompilerConfig& cfg, std::set& visited, uint32_t depth, + const std::string& description) { + // Process includes first (depth-first), so this table's own values win. + if (auto* includesArray = tbl["includes"].as_array()) { for (const auto& elem : *includesArray) { if (auto includePath = elem.value()) { - auto resolved = parentDir / *includePath; + auto resolved = base_dir / *includePath; if (!std::filesystem::exists(resolved)) { REXCODEGEN_ERROR("[config] included file not found: {} (resolved from {})", *includePath, - canonicalStr); + description); return false; } if (!LoadRecursive(resolved, cfg, visited, depth + 1)) { @@ -444,10 +444,7 @@ bool LoadRecursive(const std::filesystem::path& filePath, RecompilerConfig& cfg, } } } - - // Apply this file's config (after includes, so this file wins) - ApplyToml(toml, cfg, filePath.filename().string()); - + ApplyToml(tbl, cfg, description); return true; } @@ -457,48 +454,62 @@ bool LoadRecursive(const std::filesystem::path& filePath, RecompilerConfig& cfg, // Public API // --------------------------------------------------------------------------- -bool RecompilerConfig::Load(const std::string_view& configFilePath) { - std::set visited; - std::filesystem::path path(configFilePath); +namespace { - if (!LoadRecursive(path, *this, visited, 0)) { - return false; - } - - // Post-load summary logging - if (!functions.empty()) { +bool FinalizeConfig(RecompilerConfig& cfg) { + if (!cfg.functions.empty()) { size_t chunks_count = 0; - for (const auto& [addr, cfg] : functions) { - if (cfg.isChunk()) + for (const auto& [addr, fc] : cfg.functions) { + if (fc.isChunk()) chunks_count++; } - REXCODEGEN_INFO("Loaded {} function configs ({} standalone, {} chunks)", functions.size(), - functions.size() - chunks_count, chunks_count); + REXCODEGEN_TRACE("Loaded {} function configs ({} standalone, {} chunks)", cfg.functions.size(), + cfg.functions.size() - chunks_count, chunks_count); + } + if (!cfg.exceptionHandlerFuncHints.empty()) { + std::sort(cfg.exceptionHandlerFuncHints.begin(), cfg.exceptionHandlerFuncHints.end()); + cfg.exceptionHandlerFuncHints.erase( + std::unique(cfg.exceptionHandlerFuncHints.begin(), cfg.exceptionHandlerFuncHints.end()), + cfg.exceptionHandlerFuncHints.end()); } - // Deduplicate exceptionHandlerFuncHints (push_back from multiple files) - if (!exceptionHandlerFuncHints.empty()) { - std::sort(exceptionHandlerFuncHints.begin(), exceptionHandlerFuncHints.end()); - exceptionHandlerFuncHints.erase( - std::unique(exceptionHandlerFuncHints.begin(), exceptionHandlerFuncHints.end()), - exceptionHandlerFuncHints.end()); - } - - // Required field check - if (filePath.empty()) { + bool ok = true; + if (cfg.filePath.empty()) { REXCODEGEN_ERROR("Missing required field: file_path"); + ok = false; } - // Validate assembled config - auto result = Validate(); + auto result = cfg.Validate(); for (const auto& warning : result.warnings) { REXCODEGEN_WARN("[config] {}", warning); } for (const auto& error : result.errors) { REXCODEGEN_ERROR("[config] {}", error); } + if (!result.valid) { + ok = false; + } + return ok; +} - return true; +} // namespace + +bool RecompilerConfig::Load(const std::string_view& configFilePath) { + std::set visited; + std::filesystem::path path(configFilePath); + if (!LoadRecursive(path, *this, visited, 0)) { + return false; + } + return FinalizeConfig(*this); +} + +bool RecompilerConfig::LoadFromTable(const toml::table& tbl, + const std::filesystem::path& base_dir) { + std::set visited; + if (!ApplyTableWithIncludes(tbl, base_dir, *this, visited, 0, "")) { + return false; + } + return FinalizeConfig(*this); } RecompilerConfig::ValidationResult RecompilerConfig::Validate() const { diff --git a/thirdparty/rexglue-sdk/src/codegen/function_graph.cpp b/thirdparty/rexglue-sdk/src/codegen/function_graph.cpp index fb15acdd..2c784784 100644 --- a/thirdparty/rexglue-sdk/src/codegen/function_graph.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/function_graph.cpp @@ -566,8 +566,8 @@ std::string FunctionNode::emitCpp(const EmitContext& ctx) const { for (auto label : activeJt->targets) { labels.emplace(label); } - REXCODEGEN_INFO("Late-detected jump table at 0x{:08X} with {} entries", blockBase, - activeJt->targets.size()); + REXCODEGEN_TRACE("Late-detected jump table at 0x{:08X} with {} entries", blockBase, + activeJt->targets.size()); } } } @@ -1065,10 +1065,10 @@ size_t FunctionGraph::sealAllReady() { sealed++; } else { couldNotSeal++; - REXCODEGEN_WARN("FunctionGraph::sealAllReady: 0x{:08X} ({}) cannot seal ({} unresolved)", - base, node->name(), node->unresolvedJumps().size()); + REXCODEGEN_DEBUG("FunctionGraph::sealAllReady: 0x{:08X} ({}) cannot seal ({} unresolved)", + base, node->name(), node->unresolvedJumps().size()); for (const auto& jump : node->unresolvedJumps()) { - REXCODEGEN_WARN(" 0x{:08X} -> 0x{:08X}", jump.site, jump.target); + REXCODEGEN_DEBUG(" 0x{:08X} -> 0x{:08X}", jump.site, jump.target); } } } @@ -1111,7 +1111,7 @@ void FunctionGraph::sealAll() { throw std::runtime_error(msg); } - REXCODEGEN_INFO("FunctionGraph::sealAll: all {} functions sealed", functions_.size()); + REXCODEGEN_TRACE("FunctionGraph::sealAll: all {} functions sealed", functions_.size()); } //============================================================================= diff --git a/thirdparty/rexglue-sdk/src/codegen/manifest.cpp b/thirdparty/rexglue-sdk/src/codegen/manifest.cpp new file mode 100644 index 00000000..8aca4d6b --- /dev/null +++ b/thirdparty/rexglue-sdk/src/codegen/manifest.cpp @@ -0,0 +1,268 @@ +/** + * @file codegen/manifest.cpp + * @brief Manifest TOML parser for multi-binary projects + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace rex::codegen { + +std::string CanonicalizeModuleGuestPath(std::string_view path, std::string_view project_name) { + std::string guest_path = rex::system::NormalizeGuestPath(path); + + if (!project_name.empty()) { + std::string lower_project(project_name); + std::transform(lower_project.begin(), lower_project.end(), lower_project.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + const std::string project_assets_prefix = lower_project + "/assets/"; + if (guest_path.rfind(project_assets_prefix, 0) == 0) { + guest_path.erase(0, project_assets_prefix.size()); + } + } + + return guest_path; +} + +namespace { + +bool IsValidProjectName(std::string_view name) { + if (name.empty()) + return false; + if (!(std::isalpha(static_cast(name[0])) || name[0] == '_')) { + return false; + } + return std::all_of(name.begin(), name.end(), + [](unsigned char c) { return std::isalnum(c) || c == '_'; }); +} + +/** + * Pull file_path / out_directory_path / project_name onto the recompiler + * config so downstream consumers can rely on them. Other fields (codegen + * flags, sub-tables, includes) come from RecompilerConfig::LoadFromTable. + */ +bool LoadBinaryConfig(const toml::table& tbl, const std::filesystem::path& base_dir, + std::string_view project_name, BinaryConfig& out) { + out.recompiler = RecompilerConfig{}; + out.recompiler.projectName = std::string(project_name); + if (!out.recompiler.LoadFromTable(tbl, base_dir)) { + return false; + } + return true; +} + +} // namespace + +std::optional ManifestConfig::Load(const std::filesystem::path& path) { + toml::table tbl; + try { + tbl = toml::parse_file(path.string()); + } catch (const toml::parse_error& err) { + REXLOG_ERROR("Failed to parse manifest {}: {}", path.string(), err.what()); + return std::nullopt; + } + + ManifestConfig manifest; + manifest.manifestDir = path.parent_path(); + + auto* project = tbl["project"].as_table(); + if (!project) { + REXLOG_ERROR("Manifest missing [project] section: {}", path.string()); + return std::nullopt; + } + manifest.projectName = (*project)["name"].value_or(""); + if (manifest.projectName.empty()) { + REXLOG_ERROR("Manifest missing [project].name: {}", path.string()); + return std::nullopt; + } + if (!IsValidProjectName(manifest.projectName)) { + REXLOG_ERROR( + "Manifest [project].name '{}' is not a valid identifier " + "(letters, digits, underscore; must not start with a digit): {}", + manifest.projectName, path.string()); + return std::nullopt; + } + if (auto stamp = (*project)["sdk_version"].value(); stamp && !stamp->empty()) { + manifest.sdkVersion = *stamp; + } + if (auto root = (*project)["game_root"].value(); root && !root->empty()) { + manifest.gameRoot = *root; + } + + auto* entrypoint = tbl["entrypoint"].as_table(); + if (!entrypoint) { + REXLOG_ERROR("Manifest missing [entrypoint] section: {}", path.string()); + return std::nullopt; + } + if (!LoadBinaryConfig(*entrypoint, manifest.manifestDir, manifest.projectName, + manifest.entrypoint)) { + return std::nullopt; + } + + if (auto modules = tbl["modules"].as_array()) { + size_t index = 0; + for (const auto& mod : *modules) { + auto* modTbl = mod.as_table(); + if (!modTbl) { + REXLOG_ERROR("Manifest [[modules]] entry #{} is not a table", index); + return std::nullopt; + } + BinaryConfig binary; + if (!LoadBinaryConfig(*modTbl, manifest.manifestDir, manifest.projectName, binary)) { + return std::nullopt; + } + auto guest_path = (*modTbl)["guest_path"].value_or(""); + if (guest_path.empty()) { + REXLOG_ERROR("Manifest [[modules]] entry #{} missing guest_path", index); + return std::nullopt; + } + binary.guestPath = CanonicalizeModuleGuestPath(guest_path, manifest.projectName); + for (const auto& existing : manifest.modules) { + if (existing.guestPath == binary.guestPath) { + REXLOG_ERROR("Manifest [[modules]] duplicate guest_path '{}' (entry #{})", + binary.guestPath, index); + return std::nullopt; + } + } + manifest.modules.push_back(std::move(binary)); + ++index; + } + } + + return manifest; +} + +bool ManifestConfig::IsManifest(const std::filesystem::path& path) { + try { + auto tbl = toml::parse_file(path.string()); + return tbl.contains("project"); + } catch (const toml::parse_error&) { + return false; + } +} + +namespace { + +std::optional ParseSectionHeader(std::string_view line) { + auto first = line.find_first_not_of(" \t"); + if (first == std::string_view::npos) + return std::nullopt; + if (line[first] != '[' || (first + 1 < line.size() && line[first + 1] == '[')) + return std::nullopt; + auto end = line.find(']', first + 1); + if (end == std::string_view::npos) + return std::nullopt; + return std::string(line.substr(first + 1, end - first - 1)); +} + +bool LineSetsKey(std::string_view line, std::string_view key) { + auto first = line.find_first_not_of(" \t"); + if (first == std::string_view::npos) + return false; + if (line.compare(first, key.size(), key) != 0) + return false; + auto after = first + key.size(); + while (after < line.size() && (line[after] == ' ' || line[after] == '\t')) + ++after; + return after < line.size() && line[after] == '='; +} + +} // namespace + +bool ManifestConfig::WriteSdkVersionStamp(const std::filesystem::path& path, + std::string_view version) { + std::ifstream in(path); + if (!in) { + REXLOG_ERROR("Failed to open manifest for stamping: {}", path.string()); + return false; + } + std::vector lines; + std::string buf; + while (std::getline(in, buf)) { + lines.push_back(std::move(buf)); + buf.clear(); + } + in.close(); + + const std::string stamp_line = "sdk_version = \"" + std::string(version) + "\""; + + std::optional project_header_idx; + std::optional stamp_idx; + bool in_project = false; + for (size_t i = 0; i < lines.size(); ++i) { + auto sec = ParseSectionHeader(lines[i]); + if (sec) { + in_project = (*sec == "project"); + if (in_project) { + project_header_idx = i; + stamp_idx.reset(); + } + continue; + } + if (in_project && !stamp_idx && LineSetsKey(lines[i], "sdk_version")) { + stamp_idx = i; + } + } + + if (stamp_idx) { + lines[*stamp_idx] = stamp_line; + } else if (project_header_idx) { + lines.insert(lines.begin() + *project_header_idx + 1, stamp_line); + } else { + if (!lines.empty() && !lines.back().empty()) + lines.emplace_back(); + lines.emplace_back("[project]"); + lines.push_back(stamp_line); + } + + auto tmp_path = path; + tmp_path += ".tmp"; + { + std::ofstream out(tmp_path, std::ios::binary); + if (!out) { + REXLOG_ERROR("Failed to open manifest tmp for writing: {}", tmp_path.string()); + return false; + } + for (size_t i = 0; i < lines.size(); ++i) { + out << lines[i]; + if (i + 1 < lines.size()) + out << '\n'; + } + out << '\n'; + if (!out.good()) { + REXLOG_ERROR("Failed while writing manifest tmp: {}", tmp_path.string()); + std::error_code ignore; + std::filesystem::remove(tmp_path, ignore); + return false; + } + } + + std::error_code ec; + std::filesystem::rename(tmp_path, path, ec); + if (ec) { + REXLOG_ERROR("Failed to rename manifest tmp into place: {}", ec.message()); + std::error_code ignore; + std::filesystem::remove(tmp_path, ignore); + return false; + } + return true; +} + +} // namespace rex::codegen diff --git a/thirdparty/rexglue-sdk/src/codegen/phase_discover.cpp b/thirdparty/rexglue-sdk/src/codegen/phase_discover.cpp index 9a9fa938..67e2477d 100644 --- a/thirdparty/rexglue-sdk/src/codegen/phase_discover.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/phase_discover.cpp @@ -191,7 +191,7 @@ void discoverFunction(CodegenContext& ctx, uint32_t funcAddr, } void discoverAllFunctions(CodegenContext& ctx) { - REXCODEGEN_INFO("Analyze: starting iterative discovery..."); + REXCODEGEN_TRACE("Analyze: starting iterative discovery..."); auto& graph = ctx.graph; auto& binary = ctx.binary(); @@ -219,7 +219,7 @@ void discoverAllFunctions(CodegenContext& ctx) { } } - REXCODEGEN_INFO("Analyze: {} functions after call graph expansion", graph.functionCount()); + REXCODEGEN_TRACE("Analyze: {} functions after call graph expansion", graph.functionCount()); // VTable scanning { @@ -242,8 +242,8 @@ void discoverAllFunctions(CodegenContext& ctx) { } } - REXCODEGEN_INFO("Analyze: VTable scan found {} vtables, {} new functions", vtables.size(), - newFunctions); + REXCODEGEN_TRACE("Analyze: VTable scan found {} vtables, {} new functions", vtables.size(), + newFunctions); // Continue discovery for vtable functions if (newFunctions > 0) { @@ -264,7 +264,7 @@ void discoverAllFunctions(CodegenContext& ctx) { } } - REXCODEGEN_INFO("Analyze: {} total functions after vtable scan", graph.functionCount()); + REXCODEGEN_TRACE("Analyze: {} total functions after vtable scan", graph.functionCount()); } //============================================================================= @@ -399,7 +399,7 @@ void functionPointerScan(CodegenContext& ctx) { } } - REXCODEGEN_INFO("functionPointerScan: found {} new function pointer targets", foundCount); + REXCODEGEN_TRACE("functionPointerScan: found {} new function pointer targets", foundCount); } } // anonymous namespace @@ -421,7 +421,8 @@ size_t discoverPendingFunctions(CodegenContext& ctx, namespace phases { -VoidResult Discover(CodegenContext& ctx) { +VoidResult Discover(CodegenContext& ctx, ProgressReporter* reporter) { + (void)reporter; discoverAllFunctions(ctx); return Ok(); } diff --git a/thirdparty/rexglue-sdk/src/codegen/phase_gapfill.cpp b/thirdparty/rexglue-sdk/src/codegen/phase_gapfill.cpp index 31877d58..410b43bb 100644 --- a/thirdparty/rexglue-sdk/src/codegen/phase_gapfill.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/phase_gapfill.cpp @@ -249,7 +249,8 @@ void cleanupAbsorbedGapFills(CodegenContext& ctx) { namespace phases { -VoidResult GapFill(CodegenContext& ctx) { +VoidResult GapFill(CodegenContext& ctx, ProgressReporter* reporter) { + (void)reporter; size_t lastCount = 0; size_t iteration = 0; diff --git a/thirdparty/rexglue-sdk/src/codegen/phase_merge.cpp b/thirdparty/rexglue-sdk/src/codegen/phase_merge.cpp index 4cd755da..16071c28 100644 --- a/thirdparty/rexglue-sdk/src/codegen/phase_merge.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/phase_merge.cpp @@ -28,7 +28,7 @@ namespace { // Merge to resolve jumps then seal functions //============================================================================= void mergeAndSeal(CodegenContext& ctx) { - REXCODEGEN_INFO("Analyze: resolving jumps and sealing functions..."); + REXCODEGEN_TRACE("Analyze: resolving jumps and sealing functions..."); auto& graph = ctx.graph; auto& binary = ctx.binary(); @@ -71,11 +71,11 @@ void mergeAndSeal(CodegenContext& ctx) { size_t totalSealed = graph.sealAllReady(); size_t stillPending = graph.pendingCount(); - REXCODEGEN_INFO("Analyze: {} iterations, resolved={}, sealed={}/{}", iteration, totalResolved, - totalSealed, graph.functionCount()); + REXCODEGEN_TRACE("Analyze: {} iterations, resolved={}, sealed={}/{}", iteration, totalResolved, + totalSealed, graph.functionCount()); if (stillPending > 0) { - REXCODEGEN_WARN("Analyze: {} functions still PENDING with unresolved jumps", stillPending); + REXCODEGEN_DEBUG("Analyze: {} functions still PENDING with unresolved jumps", stillPending); } } @@ -83,7 +83,8 @@ void mergeAndSeal(CodegenContext& ctx) { namespace phases { -VoidResult Merge(CodegenContext& ctx) { +VoidResult Merge(CodegenContext& ctx, ProgressReporter* reporter) { + (void)reporter; mergeAndSeal(ctx); return Ok(); } diff --git a/thirdparty/rexglue-sdk/src/codegen/phase_register.cpp b/thirdparty/rexglue-sdk/src/codegen/phase_register.cpp index d86e6d9a..64878536 100644 --- a/thirdparty/rexglue-sdk/src/codegen/phase_register.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/phase_register.cpp @@ -418,7 +418,7 @@ void detectSaveRestoreHelpers(const BinaryView& binary, AnalysisState& state) { //============================================================================= VoidResult registerEntryPoints(CodegenContext& ctx) { - REXCODEGEN_INFO("Analyze: registering entry points..."); + REXCODEGEN_TRACE("Analyze: registering entry points..."); auto& graph = ctx.graph; auto& config = ctx.Config(); @@ -502,8 +502,9 @@ VoidResult registerEntryPoints(CodegenContext& ctx) { importCount++; } - REXCODEGEN_INFO("Analyze: loaded {} imports ({} resolved, {} unresolved, {} variables skipped)", - importCount, resolvedCount, unresolvedCount, variableCount); + REXCODEGEN_TRACE( + "Analyze: loaded {} imports ({} resolved, {} unresolved, {} variables skipped)", + importCount, resolvedCount, unresolvedCount, variableCount); } // Register save/restore helpers @@ -579,7 +580,7 @@ VoidResult registerEntryPoints(CodegenContext& ctx) { uint32_t offsetInSection = pdataAddr - pdataSection->baseAddress; const uint8_t* pdataData = pdataSection->data + offsetInSection; - REXCODEGEN_INFO("Analyze: PDATA base=0x{:08X}, size={}", pdataAddr, pdataSize); + REXCODEGEN_TRACE("Analyze: PDATA base=0x{:08X}, size={}", pdataAddr, pdataSize); size_t count = pdataSize / sizeof(IMAGE_CE_RUNTIME_FUNCTION); auto* entries = reinterpret_cast(pdataData); @@ -669,7 +670,7 @@ VoidResult registerEntryPoints(CodegenContext& ctx) { pdataAdded++; } - REXCODEGEN_INFO("Analyze: added {} functions from PDATA", pdataAdded); + REXCODEGEN_TRACE("Analyze: added {} functions from PDATA", pdataAdded); // Queue EH-discovered functions size_t ehFuncsQueued = 0; @@ -684,7 +685,7 @@ VoidResult registerEntryPoints(CodegenContext& ctx) { } if (ehFuncsQueued > 0) { - REXCODEGEN_INFO("Analyze: queued {} functions from exception handling", ehFuncsQueued); + REXCODEGEN_TRACE("Analyze: queued {} functions from exception handling", ehFuncsQueued); } return Ok(); @@ -694,7 +695,8 @@ VoidResult registerEntryPoints(CodegenContext& ctx) { namespace phases { -VoidResult Register(CodegenContext& ctx) { +VoidResult Register(CodegenContext& ctx, ProgressReporter* reporter) { + (void)reporter; return registerEntryPoints(ctx); } diff --git a/thirdparty/rexglue-sdk/src/codegen/phase_scan.cpp b/thirdparty/rexglue-sdk/src/codegen/phase_scan.cpp index e1e9e78f..ebfa4bdb 100644 --- a/thirdparty/rexglue-sdk/src/codegen/phase_scan.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/phase_scan.cpp @@ -82,7 +82,7 @@ std::vector segmentSection(const SectionView& section, } void scanBinary(CodegenContext& ctx) { - REXCODEGEN_INFO("Analyze: scanning binary..."); + REXCODEGEN_TRACE("Analyze: scanning binary..."); auto& binary = ctx.binary(); auto& config = ctx.Config(); @@ -116,7 +116,7 @@ void scanBinary(CodegenContext& ctx) { scan.codeRegions.insert(scan.codeRegions.end(), regions.begin(), regions.end()); } - REXCODEGEN_INFO("Analyze: segmented into {} code regions", scan.codeRegions.size()); + REXCODEGEN_TRACE("Analyze: segmented into {} code regions", scan.codeRegions.size()); // Detect data regions for (const auto& section : binary.sections()) { @@ -159,15 +159,16 @@ void scanBinary(CodegenContext& ctx) { } } - REXCODEGEN_INFO("Analyze: {} code regions, {} data regions", scan.codeRegions.size(), - scan.dataRegions.size()); + REXCODEGEN_TRACE("Analyze: {} code regions, {} data regions", scan.codeRegions.size(), + scan.dataRegions.size()); } } // anonymous namespace namespace phases { -VoidResult Scan(CodegenContext& ctx) { +VoidResult Scan(CodegenContext& ctx, ProgressReporter* reporter) { + (void)reporter; scanBinary(ctx); return Ok(); } diff --git a/thirdparty/rexglue-sdk/src/codegen/phase_validate.cpp b/thirdparty/rexglue-sdk/src/codegen/phase_validate.cpp index 68de1118..f6691c05 100644 --- a/thirdparty/rexglue-sdk/src/codegen/phase_validate.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/phase_validate.cpp @@ -43,7 +43,7 @@ const CallEdge* findCallEdgeAt(const FunctionNode* node, uint32_t site) { } VoidResult validateGraph(CodegenContext& ctx) { - REXCODEGEN_INFO("Analyze: validating call graph..."); + REXCODEGEN_TRACE("Analyze: validating call graph..."); auto& graph = ctx.graph; auto& binary = ctx.binary(); @@ -140,8 +140,8 @@ VoidResult validateGraph(CodegenContext& ctx) { } } - REXCODEGEN_INFO("Analyze: checked {} branches in {} functions, verified {} edges", callsChecked, - functionsChecked, edgesVerified); + REXCODEGEN_TRACE("Analyze: checked {} branches in {} functions, verified {} edges", callsChecked, + functionsChecked, edgesVerified); if (errors.HasErrors()) { REXCODEGEN_ERROR("Analyze: found {} errors", errors.Count()); @@ -151,7 +151,7 @@ VoidResult validateGraph(CodegenContext& ctx) { errors.Count(AnalysisErrors::Category::UnresolvedCall))); } - REXCODEGEN_INFO("Analyze: all calls resolve"); + REXCODEGEN_TRACE("Analyze: all calls resolve"); return Ok(); } @@ -159,7 +159,8 @@ VoidResult validateGraph(CodegenContext& ctx) { namespace phases { -VoidResult Validate(CodegenContext& ctx) { +VoidResult Validate(CodegenContext& ctx, ProgressReporter* reporter) { + (void)reporter; return validateGraph(ctx); } diff --git a/thirdparty/rexglue-sdk/src/codegen/ppc/disasm.cpp b/thirdparty/rexglue-sdk/src/codegen/ppc/disasm.cpp index c8748257..c7fe10bf 100644 --- a/thirdparty/rexglue-sdk/src/codegen/ppc/disasm.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/ppc/disasm.cpp @@ -13,13 +13,23 @@ namespace rex::codegen::ppc { -thread_local DisassemblerEngine gBigEndianDisassembler{BFD_ENDIAN_BIG, "cell 64"}; +// Binutils PPC_OPCODE_* dialect bits (see thirdparty/disasm/ppc-dis.c) +constexpr uintptr_t kXenonDialect = 0x1 // PPC + | 0x4 // 64 + | 0x4000 // POWER4 + | 0x8000000 // CELL + | 0x200 // ALTIVEC + | 0x1000000 // VMX_128 + | 0x10000; // CLASSIC + +thread_local DisassemblerEngine gBigEndianDisassembler{BFD_ENDIAN_BIG, nullptr}; DisassemblerEngine::DisassemblerEngine(bfd_endian endian, const char* options) { INIT_DISASSEMBLE_INFO(info, stdout, fprintf); info.arch = bfd_arch_powerpc; info.endian = endian; info.disassembler_options = options; + info.private_data = reinterpret_cast(kXenonDialect); } int DisassemblerEngine::Disassemble(const void* code, size_t size, uint64_t base, ppc_insn& out) { diff --git a/thirdparty/rexglue-sdk/src/codegen/project_recompiler.cpp b/thirdparty/rexglue-sdk/src/codegen/project_recompiler.cpp new file mode 100644 index 00000000..e6c410aa --- /dev/null +++ b/thirdparty/rexglue-sdk/src/codegen/project_recompiler.cpp @@ -0,0 +1,435 @@ +/** + * @file codegen/project_recompiler.cpp + * @brief Project-level recompiler driving manifest-based multi-binary codegen + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "codegen_logging.h" +#include "template_registry_internal.h" + +namespace rex::codegen { + +namespace { + +std::string DeriveTargetNameFromFilePath(const std::string& file_path) { + std::filesystem::path p(file_path); + std::string name = p.stem().string(); + std::replace(name.begin(), name.end(), '.', '_'); + std::replace(name.begin(), name.end(), ' ', '_'); + return name; +} + +} // namespace + +ProjectRecompiler::ProjectRecompiler(ManifestConfig manifest) : manifest_(std::move(manifest)) {} + +Result ProjectRecompiler::Run(const ProjectRecompilerOptions& opts) { + namespace fs = std::filesystem; + + deletedFiles_.clear(); + writtenFiles_.clear(); + + if (manifest_.modules.empty()) { + REXCODEGEN_TRACE("Recompiling '{}' (entrypoint)", manifest_.projectName); + } else { + REXCODEGEN_TRACE("Recompiling '{}' (entrypoint + {} DLL{})", manifest_.projectName, + manifest_.modules.size(), manifest_.modules.size() == 1 ? "" : "s"); + } + + struct ModuleEntry { + std::string targetName; + std::string guestPath; + bool isDll; + RecompilerConfig config; + }; + + std::vector allModules; + allModules.push_back({DeriveTargetNameFromFilePath(manifest_.entrypoint.recompiler.filePath), "", + false, std::move(manifest_.entrypoint.recompiler)}); + for (auto& mod : manifest_.modules) { + allModules.push_back({DeriveTargetNameFromFilePath(mod.recompiler.filePath), mod.guestPath, + true, std::move(mod.recompiler)}); + } + + if (!opts.targets.empty()) { + std::vector unknown; + for (const auto& t : opts.targets) { + bool match = std::any_of(allModules.begin() + 1, allModules.end(), + [&t](const ModuleEntry& m) { return m.targetName == t; }); + if (!match) { + unknown.push_back(t); + } + } + if (!unknown.empty()) { + std::string known; + for (size_t i = 1; i < allModules.size(); ++i) { + if (i > 1) + known += ", "; + known += allModules[i].targetName; + } + std::string list; + for (size_t i = 0; i < unknown.size(); ++i) { + if (i > 0) + list += ", "; + list += unknown[i]; + } + return Err(ErrorCategory::Config, + fmt::format("Unknown --target value(s): {}. Known DLL targets: {}", list, + known.empty() ? "(none)" : known)); + } + } + + std::vector targeted; + targeted.push_back(std::move(allModules[0])); + for (size_t i = 1; i < allModules.size(); ++i) { + if (opts.targets.empty() || std::find(opts.targets.begin(), opts.targets.end(), + allModules[i].targetName) != opts.targets.end()) { + targeted.push_back(std::move(allModules[i])); + } + } + + // Two binaries sharing an out_directory_path would clobber each other's + // sources.cmake on emit (the writer's cleanup sweep is unprefixed for + // that file). + std::unordered_map outDirOwner; + for (const auto& m : targeted) { + const auto& outDir = m.config.outDirectoryPath; + if (outDir.empty()) + continue; + auto [it, inserted] = outDirOwner.emplace(outDir, m.targetName); + if (!inserted) { + return Err( + ErrorCategory::Validation, + fmt::format("out_directory_path '{}' is shared by '{}' and '{}'; each binary " + "needs its own output directory", + outDir, it->second, m.targetName)); + } + } + + const auto& entryConfig = targeted[0].config; + auto configDir = manifest_.manifestDir; + fs::path entryXexPath = configDir / entryConfig.filePath; + if (!entryConfig.patchedFilePath.empty()) { + auto patched = configDir / entryConfig.patchedFilePath; + if (fs::exists(patched)) + entryXexPath = patched; + } + if (!fs::exists(entryXexPath)) { + return Err(ErrorCategory::IO, + fmt::format("Entrypoint XEX not found: {}", entryXexPath.string())); + } + entryXexPath = fs::canonical(entryXexPath); + + // gameRoot anchors VFS root and DLL guest_path derivation. Honor the + // manifest override if set; otherwise default to the entrypoint's parent. + fs::path gameRoot; + if (manifest_.gameRoot && !manifest_.gameRoot->empty()) { + fs::path resolved = configDir / *manifest_.gameRoot; + if (!fs::exists(resolved) || !fs::is_directory(resolved)) { + return Err(ErrorCategory::Validation, + fmt::format("[project].game_root '{}' does not resolve to a directory", + resolved.string())); + } + gameRoot = fs::canonical(resolved); + } else { + gameRoot = fs::canonical(entryXexPath.parent_path()); + } + + fs::path entryRel = fs::relative(entryXexPath, gameRoot); + if (entryRel.empty() || *entryRel.begin() == "..") { + return Err(ErrorCategory::Validation, + fmt::format("Entrypoint XEX '{}' resolves outside game root '{}'", + entryXexPath.string(), gameRoot.string())); + } + + auto runtime = std::make_unique(gameRoot.string()); + auto rtStatus = runtime->Setup(rex::RuntimeConfig{ + .kernel_init = rex::kernel::InitializeKernel, + .tool_mode = true, + }); + if (rtStatus != X_STATUS_SUCCESS) { + return Err(ErrorCategory::IO, + fmt::format("Failed to initialize Runtime: {:#x}", rtStatus)); + } + + std::string entryRelStr = entryRel.string(); + std::replace(entryRelStr.begin(), entryRelStr.end(), '/', '\\'); + auto entryVfsPath = "game:\\" + entryRelStr; + rtStatus = runtime->LoadXexImage(entryVfsPath); + if (rtStatus != X_STATUS_SUCCESS) { + return Err(ErrorCategory::IO, + fmt::format("Failed to load entrypoint XEX: {:#x}", rtStatus)); + } + + std::vector> dllModules; + for (size_t i = 1; i < targeted.size(); ++i) { + const auto& dllConfig = targeted[i].config; + fs::path dllXexPath = configDir / dllConfig.filePath; + if (!dllConfig.patchedFilePath.empty()) { + auto patched = configDir / dllConfig.patchedFilePath; + if (fs::exists(patched)) + dllXexPath = patched; + } + if (!fs::exists(dllXexPath)) { + return Err(ErrorCategory::IO, + fmt::format("DLL XEX not found: {}", dllXexPath.string())); + } + dllXexPath = fs::canonical(dllXexPath); + + auto relPath = fs::relative(dllXexPath, gameRoot); + if (relPath.empty() || *relPath.begin() == "..") { + return Err(ErrorCategory::Validation, + fmt::format("DLL '{}' resolves outside game root '{}': {}", + targeted[i].targetName, gameRoot.string(), dllXexPath.string())); + } + std::string relStr = relPath.string(); + std::replace(relStr.begin(), relStr.end(), '/', '\\'); + auto dllVfsPath = "game:\\" + relStr; + auto userMod = runtime->kernel_state()->LoadUserModule(dllVfsPath, false); + if (!userMod) { + return Err(ErrorCategory::IO, + fmt::format("Failed to load DLL module: {}", targeted[i].targetName)); + } + REXCODEGEN_TRACE("Loaded DLL module '{}' at base 0x{:08X}", targeted[i].targetName, + userMod->xex_module()->base_address()); + dllModules.push_back(std::move(userMod)); + } + + struct ContextEntry { + CodegenContext ctx; + const ModuleEntry* module; + std::string display_name; + }; + std::vector contexts; + + auto make_display_name = [](const std::string& filePath) { + return std::filesystem::path(filePath).filename().string(); + }; + + auto* resolver = runtime->export_resolver(); + + { + auto execMod = runtime->kernel_state()->GetExecutableModule(); + auto bv = BinaryView::fromModule(*execMod->xex_module()); + + auto entry_display = make_display_name(targeted[0].config.filePath); + RecompilerConfig cfg = std::move(targeted[0].config); + if (opts.enableExceptionHandlers) + cfg.generateExceptionHandlers = true; + + auto ctx = CodegenContext::Create(std::move(bv), std::move(cfg)); + ctx.setResolver(resolver); + ctx.setConfigDir(configDir); + ctx.analysisState().format = "xex"; + ctx.analysisState().loadAddress = ctx.binary().baseAddress(); + ctx.analysisState().entryPoint = ctx.binary().entryPoint(); + ctx.analysisState().imageSize = ctx.binary().imageSize(); + ctx.setHasDllModules(!manifest_.modules.empty()); + if (ctx.Config().isDll.has_value()) + ctx.setDllModule(*ctx.Config().isDll); + + contexts.push_back({std::move(ctx), &targeted[0], std::move(entry_display)}); + } + + for (size_t i = 0; i < dllModules.size(); ++i) { + auto& userMod = dllModules[i]; + auto bv = BinaryView::fromModule(*userMod->xex_module()); + + auto dll_display = make_display_name(targeted[i + 1].config.filePath); + RecompilerConfig cfg = std::move(targeted[i + 1].config); + if (opts.enableExceptionHandlers) + cfg.generateExceptionHandlers = true; + + auto ctx = CodegenContext::Create(std::move(bv), std::move(cfg)); + ctx.setResolver(resolver); + ctx.setConfigDir(configDir); + ctx.analysisState().format = "xex"; + ctx.analysisState().loadAddress = ctx.binary().baseAddress(); + ctx.analysisState().entryPoint = ctx.binary().entryPoint(); + ctx.analysisState().imageSize = ctx.binary().imageSize(); + ctx.setDllModule(ctx.Config().isDll.value_or(true)); + ctx.setHasDllModules(true); + + contexts.push_back({std::move(ctx), &targeted[i + 1], std::move(dll_display)}); + } + + std::vector module_started_at(contexts.size()); + for (size_t i = 0; i < contexts.size(); ++i) { + auto& entry = contexts[i]; + if (opts.reporter) { + opts.reporter->moduleStarted(entry.display_name, i, contexts.size()); + } + module_started_at[i] = std::chrono::steady_clock::now(); + REXCODEGEN_TRACE("Analyzing '{}'...", entry.module->targetName); + auto result = Analyze(entry.ctx, opts.reporter); + if (!result) { + if (opts.force && result.error().category == ErrorCategory::Validation) { + REXLOG_WARN("Analysis errors for '{}' (continuing due to --force): {}", + entry.module->targetName, result.error().message); + } else { + REXLOG_ERROR("Analysis failed for '{}'", entry.module->targetName); + return result; + } + } + } + + for (size_t i = 0; i < contexts.size(); ++i) { + uint32_t a_base = contexts[i].ctx.binary().baseAddress(); + uint32_t a_end = a_base + contexts[i].ctx.binary().imageSize(); + for (size_t j = i + 1; j < contexts.size(); ++j) { + uint32_t b_base = contexts[j].ctx.binary().baseAddress(); + uint32_t b_end = b_base + contexts[j].ctx.binary().imageSize(); + if (a_base < b_end && b_base < a_end) { + return Err(ErrorCategory::Validation, + fmt::format("Module '{}' [{:08X}, {:08X}) overlaps '{}' [{:08X}, {:08X})", + contexts[i].module->targetName, a_base, a_end, + contexts[j].module->targetName, b_base, b_end)); + } + } + } + + deletedFiles_.clear(); + writtenFiles_.clear(); + for (size_t i = 0; i < contexts.size(); ++i) { + auto& entry = contexts[i]; + if (opts.reporter) { + opts.reporter->moduleStarted(entry.display_name, i, contexts.size()); + opts.reporter->phaseChanged("Write"); + } + REXCODEGEN_TRACE("Writing output for '{}'...", entry.module->targetName); + CodegenWriter writer(entry.ctx, runtime.get()); + if (!writer.write(opts.force)) { + return Err(ErrorCategory::Validation, + fmt::format("Write failed for '{}'", entry.module->targetName)); + } + deletedFiles_.insert(deletedFiles_.end(), writer.deletedFiles().begin(), + writer.deletedFiles().end()); + writtenFiles_.insert(writtenFiles_.end(), writer.writtenFiles().begin(), + writer.writtenFiles().end()); + if (opts.reporter) { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - module_started_at[i]); + opts.reporter->moduleFinished(elapsed); + } + } + + if (manifest_.modules.empty()) { + REXCODEGEN_TRACE("Project recompiler complete"); + return Ok(); + } + + auto outputPath = contexts[0].ctx.configDir() / contexts[0].ctx.Config().outDirectoryPath; + std::error_code mk_ec; + fs::create_directories(outputPath, mk_ec); + if (mk_ec) { + return Err(ErrorCategory::IO, fmt::format("Failed to create output dir {}: {}", + outputPath.string(), mk_ec.message())); + } + + if (opts.reporter) + opts.reporter->projectPhaseStarted("module_registry"); + { + nlohmann::json registryData; + registryData["project"] = manifest_.projectName; + + auto& dllArray = registryData["dll_modules"]; + dllArray = nlohmann::json::array(); + for (size_t i = 1; i < targeted.size(); ++i) { + auto& mod = targeted[i]; + auto guestPath = mod.guestPath; + std::replace(guestPath.begin(), guestPath.end(), '\\', '/'); + nlohmann::json dllEntry; + dllEntry["pe_name"] = fs::path(mod.guestPath).filename().string(); + dllEntry["guest_path"] = guestPath; + dllEntry["shared_lib_name"] = manifest_.projectName + "_" + mod.targetName; + dllArray.push_back(dllEntry); + } + + TemplateRegistry registry; + auto registryContent = renderWithJson(registry, "codegen/module_registry_cpp", registryData); + + auto registryPath = outputPath / "module_registry.cpp"; + std::ofstream f(registryPath); + if (!f) { + return Err(ErrorCategory::IO, fmt::format("Failed to open {}", registryPath.string())); + } + f << registryContent; + if (!f.good()) { + return Err(ErrorCategory::IO, + fmt::format("Failed while writing {}", registryPath.string())); + } + writtenFiles_.push_back(registryPath.filename().string()); + REXCODEGEN_TRACE("Wrote {}", registryPath.string()); + } + if (opts.reporter) + opts.reporter->projectPhaseFinished(); + + if (opts.reporter) + opts.reporter->projectPhaseStarted("dll_targets.cmake"); + { + nlohmann::json dllTargetsData; + auto& dllTargetsArray = dllTargetsData["dll_modules"]; + dllTargetsArray = nlohmann::json::array(); + for (size_t i = 0; i < dllModules.size(); ++i) { + auto& mod = targeted[i + 1]; + auto& dllCtx = contexts[i + 1].ctx; + nlohmann::json entry; + entry["target_name"] = mod.targetName; + entry["lib_name"] = manifest_.projectName + "_" + mod.targetName; + entry["output_dir"] = dllCtx.Config().outDirectoryPath; + dllTargetsArray.push_back(entry); + } + + TemplateRegistry registry; + auto dllCmakeContent = renderWithJson(registry, "codegen/dll_targets_cmake", dllTargetsData); + + auto dllCmakePath = outputPath / "dll_targets.cmake"; + std::ofstream cf(dllCmakePath); + if (!cf) { + return Err(ErrorCategory::IO, fmt::format("Failed to open {}", dllCmakePath.string())); + } + cf << dllCmakeContent; + if (!cf.good()) { + return Err(ErrorCategory::IO, + fmt::format("Failed while writing {}", dllCmakePath.string())); + } + writtenFiles_.push_back(dllCmakePath.filename().string()); + REXCODEGEN_TRACE("Wrote {}", dllCmakePath.string()); + } + if (opts.reporter) + opts.reporter->projectPhaseFinished(); + + REXCODEGEN_TRACE("Project recompiler complete"); + return Ok(); +} + +} // namespace rex::codegen diff --git a/thirdparty/rexglue-sdk/src/codegen/template_registry.cpp b/thirdparty/rexglue-sdk/src/codegen/template_registry.cpp index 88f0b8c4..ee1aa119 100644 --- a/thirdparty/rexglue-sdk/src/codegen/template_registry.cpp +++ b/thirdparty/rexglue-sdk/src/codegen/template_registry.cpp @@ -57,6 +57,23 @@ struct TemplateRegistry::Impl { oss << "0x" << std::hex << std::uppercase << val; return oss.str(); }); + + // Resolve {% include "" %} against the embedded registry. Templates + // are embedded under canonical IDs without the .inja extension; accept + // either form so include directives can use either. + env_.set_search_included_templates_in_files(false); + env_.set_include_callback( + [this](const std::filesystem::path&, const std::string& name) -> inja::Template { + std::string id = name; + if (id.size() > 5 && id.compare(id.size() - 5, 5, ".inja") == 0) { + id.erase(id.size() - 5); + } + auto it = embedded_.find(id); + if (it == embedded_.end()) { + throw TemplateError(name, "include not found in embedded registry"); + } + return env_.parse(std::string(it->second)); + }); } std::string renderImpl(const std::string& id, const nlohmann::json& data) { @@ -130,7 +147,7 @@ void TemplateRegistry::loadOverrides(const std::filesystem::path& dir) { }(); auto tmpl = impl_->env_.parse(content); impl_->overrides_[id] = std::move(tmpl); - REXCODEGEN_INFO("Loaded template override: {} -> {}", id, entry.path().string()); + REXCODEGEN_TRACE("Loaded template override: {} -> {}", id, entry.path().string()); } catch (const std::exception& e) { throw TemplateError(id, e.what(), entry.path().string()); } diff --git a/thirdparty/rexglue-sdk/src/core/CMakeLists.txt b/thirdparty/rexglue-sdk/src/core/CMakeLists.txt index 919491f3..4cb16b48 100644 --- a/thirdparty/rexglue-sdk/src/core/CMakeLists.txt +++ b/thirdparty/rexglue-sdk/src/core/CMakeLists.txt @@ -41,6 +41,7 @@ if(WIN32) seh_win.cpp socket_win.cpp string_win.cpp + system_win.cpp threading_win.cpp ) elseif(UNIX) @@ -58,6 +59,7 @@ elseif(UNIX) seh_posix.cpp socket_posix.cpp string_posix.cpp + system_posix.cpp threading_posix.cpp ) endif() diff --git a/thirdparty/rexglue-sdk/src/core/cvar.cpp b/thirdparty/rexglue-sdk/src/core/cvar.cpp index 314b3888..fa694879 100644 --- a/thirdparty/rexglue-sdk/src/core/cvar.cpp +++ b/thirdparty/rexglue-sdk/src/core/cvar.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,13 @@ bool g_finalized = false; bool g_lifecycle_override = false; std::mutex g_mutex; +// Recursive: FlagRegistrar chain methods re-enter; change callbacks invoked +// from SetFlagByName must not mutate the registry. +std::recursive_mutex& GetRegistryMutex() { + static std::recursive_mutex m; + return m; +} + // Flag registry - use functions to avoid static init order issues std::vector& GetRegistryStorage() { static std::vector registry; @@ -164,17 +172,60 @@ std::vector& GetRegistry() { return GetRegistryStorage(); } -void RegisterFlag(FlagEntry entry) { - auto it = GetRegistryIndex().find(entry.name); - if (it != GetRegistryIndex().end()) { - return; // Already registered +std::optional RegisterFlag(FlagEntry entry) { + std::lock_guard lock(GetRegistryMutex()); + (void)GetCallbackStorage(); + (void)GetPendingRestartStorage(); + auto& index = GetRegistryIndex(); + auto& storage = GetRegistryStorage(); + auto it = index.find(entry.name); + if (it != index.end()) { + REXLOG_ERROR("cvar: duplicate registration of '{}'; second registration ignored", entry.name); + return std::nullopt; } + size_t pos = storage.size(); + index[entry.name] = pos; + storage.push_back(std::move(entry)); + return pos; +} - GetRegistryIndex()[entry.name] = GetRegistryStorage().size(); - GetRegistryStorage().push_back(std::move(entry)); +void UnregisterFlag(std::string_view name) { + std::lock_guard lock(GetRegistryMutex()); + auto& index = GetRegistryIndex(); + auto& storage = GetRegistryStorage(); + std::string key(name); + auto idx_it = index.find(key); + if (idx_it == index.end()) { + return; + } + size_t pos = idx_it->second; + index.erase(idx_it); + storage.erase(storage.begin() + pos); + for (auto& [n, i] : index) { + if (i > pos) { + --i; + } + } + GetCallbackStorage().erase(key); + auto& pending = GetPendingRestartStorage(); + pending.erase(std::remove(pending.begin(), pending.end(), key), pending.end()); +} + +void FlagRegistrar::apply_(std::function fn) { + if (owned_name_.empty()) { + return; + } + std::lock_guard lock(GetRegistryMutex()); + auto& index = GetRegistryIndex(); + auto it = index.find(owned_name_); + if (it == index.end()) { + return; + } + fn(GetRegistryStorage()[it->second]); } bool SetFlagByName(std::string_view name, std::string_view value) { + std::lock_guard lock(GetRegistryMutex()); auto it = GetRegistryIndex().find(std::string(name)); if (it == GetRegistryIndex().end()) { return false; @@ -215,6 +266,7 @@ bool SetFlagByName(std::string_view name, std::string_view value) { } std::string GetFlagByName(std::string_view name) { + std::lock_guard lock(GetRegistryMutex()); auto it = GetRegistryIndex().find(std::string(name)); if (it == GetRegistryIndex().end()) { return ""; @@ -224,6 +276,7 @@ std::string GetFlagByName(std::string_view name) { } std::vector ListFlags() { + std::lock_guard lock(GetRegistryMutex()); std::vector result; result.reserve(GetRegistryStorage().size()); for (const auto& entry : GetRegistryStorage()) { @@ -234,6 +287,7 @@ std::vector ListFlags() { } std::vector ListFlagsByCategory(std::string_view category) { + std::lock_guard lock(GetRegistryMutex()); std::vector result; for (const auto& entry : GetRegistryStorage()) { if (entry.category == category) { @@ -245,6 +299,7 @@ std::vector ListFlagsByCategory(std::string_view category) { } std::vector ListFlagsByLifecycle(Lifecycle lc) { + std::lock_guard lock(GetRegistryMutex()); std::vector result; for (const auto& entry : GetRegistryStorage()) { if (entry.lifecycle == lc) { @@ -256,6 +311,8 @@ std::vector ListFlagsByLifecycle(Lifecycle lc) { } const FlagEntry* GetFlagInfo(std::string_view name) { + // Pointer is invalidated by any subsequent registry call. + std::lock_guard lock(GetRegistryMutex()); auto it = GetRegistryIndex().find(std::string(name)); if (it == GetRegistryIndex().end()) { return nullptr; @@ -263,15 +320,69 @@ const FlagEntry* GetFlagInfo(std::string_view name) { return &GetRegistryStorage()[it->second]; } +template <> +bool Query(std::string_view name) { + std::string v = GetFlagByName(name); + return v == "true" || v == "1" || v == "yes"; +} + +template <> +int32_t Query(std::string_view name) { + std::string v = GetFlagByName(name); + int32_t out = 0; + std::from_chars(v.data(), v.data() + v.size(), out); + return out; +} + +template <> +int64_t Query(std::string_view name) { + std::string v = GetFlagByName(name); + int64_t out = 0; + std::from_chars(v.data(), v.data() + v.size(), out); + return out; +} + +template <> +uint32_t Query(std::string_view name) { + std::string v = GetFlagByName(name); + uint32_t out = 0; + std::from_chars(v.data(), v.data() + v.size(), out); + return out; +} + +template <> +uint64_t Query(std::string_view name) { + std::string v = GetFlagByName(name); + uint64_t out = 0; + std::from_chars(v.data(), v.data() + v.size(), out); + return out; +} + +template <> +double Query(std::string_view name) { + std::string v = GetFlagByName(name); + double out = 0.0; + ParseDouble(v, out); + return out; +} + +template <> +std::string Query(std::string_view name) { + return GetFlagByName(name); +} + std::vector GetPendingRestartFlags() { + std::lock_guard lock(GetRegistryMutex()); return GetPendingRestartStorage(); } void ClearPendingRestartFlags() { + std::lock_guard lock(GetRegistryMutex()); GetPendingRestartStorage().clear(); } void ResetToDefault(std::string_view name) { + std::lock_guard lock(GetRegistryMutex()); auto it = GetRegistryIndex().find(std::string(name)); if (it == GetRegistryIndex().end()) { return; @@ -281,12 +392,14 @@ void ResetToDefault(std::string_view name) { } void ResetAllToDefaults() { + std::lock_guard lock(GetRegistryMutex()); for (const auto& entry : GetRegistryStorage()) { entry.setter(entry.default_value); } } bool HasNonDefaultValue(std::string_view name) { + std::lock_guard lock(GetRegistryMutex()); auto it = GetRegistryIndex().find(std::string(name)); if (it == GetRegistryIndex().end()) { return false; @@ -296,6 +409,7 @@ bool HasNonDefaultValue(std::string_view name) { } std::vector ListModifiedFlags() { + std::lock_guard lock(GetRegistryMutex()); std::vector result; for (const auto& entry : GetRegistryStorage()) { if (entry.getter() != entry.default_value) { @@ -306,6 +420,7 @@ std::vector ListModifiedFlags() { } std::string SerializeToTOML() { + std::lock_guard lock(GetRegistryMutex()); std::string result; for (const auto& entry : GetRegistryStorage()) { if (entry.getter() != entry.default_value) { @@ -320,6 +435,7 @@ std::string SerializeToTOML() { } std::string SerializeToTOML(std::string_view category) { + std::lock_guard lock(GetRegistryMutex()); std::string result; for (const auto& entry : GetRegistryStorage()) { if (entry.category == category && entry.getter() != entry.default_value) { @@ -334,10 +450,12 @@ std::string SerializeToTOML(std::string_view category) { } void RegisterChangeCallback(std::string_view name, ChangeCallback callback) { + std::lock_guard lock(GetRegistryMutex()); GetCallbackStorage()[std::string(name)].push_back(std::move(callback)); } void UnregisterChangeCallbacks(std::string_view name) { + std::lock_guard lock(GetRegistryMutex()); GetCallbackStorage().erase(std::string(name)); } diff --git a/thirdparty/rexglue-sdk/src/core/logging.cpp b/thirdparty/rexglue-sdk/src/core/logging.cpp index 0b97fab7..a91a38c8 100644 --- a/thirdparty/rexglue-sdk/src/core/logging.cpp +++ b/thirdparty/rexglue-sdk/src/core/logging.cpp @@ -442,6 +442,20 @@ void RemoveSink(LogCategoryId category, spdlog::sink_ptr sink) { } } +void ReplaceConsoleSink(spdlog::sink_ptr sink) { + std::lock_guard lock(g_mutex); + for (auto& entry : g_registry) { + if (!entry.logger) + continue; + auto& sinks = entry.logger->sinks(); + if (g_console_sink) + std::erase(sinks, g_console_sink); + if (sink) + sinks.push_back(sink); + } + g_console_sink = sink; +} + void SetConsolePattern(const std::string& pattern) { if (g_console_sink) g_console_sink->set_pattern(pattern); diff --git a/thirdparty/rexglue-sdk/src/core/system_posix.cpp b/thirdparty/rexglue-sdk/src/core/system_posix.cpp new file mode 100644 index 00000000..1655695f --- /dev/null +++ b/thirdparty/rexglue-sdk/src/core/system_posix.cpp @@ -0,0 +1,33 @@ +/** + * @file core/system_posix.cpp + * @brief POSIX implementations of rex/system.h platform helpers. + * + * @copyright Copyright (c) 2026 Tom Clay + * @license BSD 3-Clause License + */ + +#include + +#include + +namespace rex { + +// TODO(tomc): add linux support for showing a native message box +void ShowSimpleMessageBox(SimpleMessageBoxType type, std::string_view message) { + const char* level = "INFO"; + switch (type) { + case SimpleMessageBoxType::Help: + level = "INFO"; + break; + case SimpleMessageBoxType::Warning: + level = "WARNING"; + break; + case SimpleMessageBoxType::Error: + level = "ERROR"; + break; + } + std::fprintf(stderr, "[%s] %.*s\n", level, static_cast(message.size()), message.data()); + std::fflush(stderr); +} + +} // namespace rex diff --git a/thirdparty/rexglue-sdk/src/core/system_win.cpp b/thirdparty/rexglue-sdk/src/core/system_win.cpp new file mode 100644 index 00000000..bb9e1cbd --- /dev/null +++ b/thirdparty/rexglue-sdk/src/core/system_win.cpp @@ -0,0 +1,37 @@ +/** + * @file core/system_win.cpp + * @brief Windows implementations of rex/system.h platform helpers. + * + * @copyright Copyright (c) 2026 Tom Clay + * @license BSD 3-Clause License + */ + +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +#include +#include + +namespace rex { + +void ShowSimpleMessageBox(SimpleMessageBoxType type, std::string_view message) { + UINT flags = MB_OK | MB_TOPMOST | MB_SETFOREGROUND; + const wchar_t* title = L"ReXGlue"; + switch (type) { + case SimpleMessageBoxType::Help: + flags |= MB_ICONINFORMATION; + break; + case SimpleMessageBoxType::Warning: + flags |= MB_ICONWARNING; + break; + case SimpleMessageBoxType::Error: + flags |= MB_ICONERROR; + break; + } + auto wide = rex::string::to_utf16(message); + ::MessageBoxW(nullptr, reinterpret_cast(wide.c_str()), title, flags); +} + +} // namespace rex diff --git a/thirdparty/rexglue-sdk/src/graphics/d3d12/graphics_system.cpp b/thirdparty/rexglue-sdk/src/graphics/d3d12/graphics_system.cpp index e12ecb5c..caa77cf9 100644 --- a/thirdparty/rexglue-sdk/src/graphics/d3d12/graphics_system.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/d3d12/graphics_system.cpp @@ -37,11 +37,10 @@ std::string D3D12GraphicsSystem::name() const { return "Direct3D 12"; } -X_STATUS D3D12GraphicsSystem::Setup(runtime::FunctionDispatcher* function_dispatcher, - system::KernelState* kernel_state, - ui::WindowedAppContext* app_context, bool with_presentation) { +void D3D12GraphicsSystem::CreateProvider(bool /*with_presentation*/) { + // D3D12 doesn't differentiate headless vs. swapchain-capable providers; + // swapchains are created lazily per-window by the presenter. provider_ = rex::ui::d3d12::D3D12Provider::Create(); - return GraphicsSystem::Setup(function_dispatcher, kernel_state, app_context, with_presentation); } std::unique_ptr D3D12GraphicsSystem::CreateCommandProcessor() { diff --git a/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp b/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp index 23a1ea9f..3dfb0dcc 100644 --- a/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/d3d12/pipeline_cache.cpp @@ -2686,12 +2686,12 @@ ID3D12PipelineState* PipelineCache::CreateD3D12Pipeline( const PipelineDescription& description = runtime_description.description; if (runtime_description.pixel_shader != nullptr) { - REXGPU_INFO("Creating graphics pipeline with VS {:016X}, PS {:016X}", - runtime_description.vertex_shader->shader().ucode_data_hash(), - runtime_description.pixel_shader->shader().ucode_data_hash()); + REXGPU_DEBUG("Creating graphics pipeline with VS {:016X}, PS {:016X}", + runtime_description.vertex_shader->shader().ucode_data_hash(), + runtime_description.pixel_shader->shader().ucode_data_hash()); } else { - REXGPU_INFO("Creating graphics pipeline with VS {:016X}", - runtime_description.vertex_shader->shader().ucode_data_hash()); + REXGPU_DEBUG("Creating graphics pipeline with VS {:016X}", + runtime_description.vertex_shader->shader().ucode_data_hash()); } D3D12_GRAPHICS_PIPELINE_STATE_DESC state_desc; diff --git a/thirdparty/rexglue-sdk/src/graphics/graphics_system.cpp b/thirdparty/rexglue-sdk/src/graphics/graphics_system.cpp index 39767903..6d04b74c 100644 --- a/thirdparty/rexglue-sdk/src/graphics/graphics_system.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/graphics_system.cpp @@ -117,32 +117,56 @@ GraphicsSystem::GraphicsSystem() : vsync_worker_running_(false) {} GraphicsSystem::~GraphicsSystem() = default; -X_STATUS GraphicsSystem::Setup(runtime::FunctionDispatcher* function_dispatcher, - system::KernelState* kernel_state, - ui::WindowedAppContext* app_context, bool with_presentation) { +X_STATUS GraphicsSystem::SetupPresentation(ui::WindowedAppContext* app_context) { + if (presenter_) { + return X_STATUS_SUCCESS; + } + + if (!provider_) { + CreateProvider(true); + if (!provider_) { + REXGPU_ERROR("Unable to create graphics provider"); + return X_STATUS_UNSUCCESSFUL; + } + provider_supports_presentation_ = true; + } else if (!provider_supports_presentation_) { + // A prior SetupGuestGpu built a headless provider; backends like Vulkan + // need swapchain support baked in at provider creation time. + REXGPU_ERROR("SetupPresentation called after headless SetupGuestGpu; call order is reversed"); + return X_STATUS_UNSUCCESSFUL; + } + + app_context_ = app_context; + auto loss_cb = [this](bool is_responsible, bool statically_from_ui_thread) { + OnHostGpuLossFromAnyThread(is_responsible); + }; + if (app_context_) { + // Presenter creation must happen on the UI thread. + app_context_->CallInUIThreadSynchronous( + [this, loss_cb]() { presenter_ = provider_->CreatePresenter(loss_cb); }); + } else { + // Offscreen path (e.g. capturing guest output without a window). + presenter_ = provider_->CreatePresenter(loss_cb); + } + + if (!presenter_) { + REXGPU_ERROR("Unable to create presenter"); + return X_STATUS_UNSUCCESSFUL; + } + return X_STATUS_SUCCESS; +} + +X_STATUS GraphicsSystem::SetupGuestGpu(runtime::FunctionDispatcher* function_dispatcher, + system::KernelState* kernel_state) { memory_ = function_dispatcher->memory(); function_dispatcher_ = function_dispatcher; kernel_state_ = kernel_state; - app_context_ = app_context; - // Create presenter if presentation is requested and provider is available - if (with_presentation && provider_) { - // Safe if either the UI thread call or the presenter creation fails. - if (app_context_) { - app_context_->CallInUIThreadSynchronous([this]() { - presenter_ = - provider_->CreatePresenter([this](bool is_responsible, bool statically_from_ui_thread) { - OnHostGpuLossFromAnyThread(is_responsible); - }); - }); - } else { - // May be needed for offscreen use, such as capturing the guest output - // image. - presenter_ = - provider_->CreatePresenter([this](bool is_responsible, bool statically_from_ui_thread) { - OnHostGpuLossFromAnyThread(is_responsible); - }); - } + // Headless path: no one set up presentation, so build a no-presentation + // provider just for the command processor. + if (!provider_) { + CreateProvider(false); + provider_supports_presentation_ = false; } // Create command processor. This will spin up a thread to process all diff --git a/thirdparty/rexglue-sdk/src/graphics/pipeline/render_target/cache.cpp b/thirdparty/rexglue-sdk/src/graphics/pipeline/render_target/cache.cpp index db82ab07..382b0b22 100644 --- a/thirdparty/rexglue-sdk/src/graphics/pipeline/render_target/cache.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/pipeline/render_target/cache.cpp @@ -1193,7 +1193,7 @@ RenderTargetCache::RenderTarget* RenderTargetCache::GetOrCreateRenderTarget(Rend uint32_t width = key.GetWidth(); uint32_t height = GetRenderTargetHeight(key.pitch_tiles_at_32bpp, key.msaa_samples); if (render_target) { - REXGPU_INFO( + REXGPU_DEBUG( "Created a {}x{} {}xMSAA {} render target with guest format {} at " "EDRAM base {}", width, height, uint32_t(1) << uint32_t(key.msaa_samples), diff --git a/thirdparty/rexglue-sdk/src/graphics/vulkan/graphics_system.cpp b/thirdparty/rexglue-sdk/src/graphics/vulkan/graphics_system.cpp index df56b108..df3f98f9 100644 --- a/thirdparty/rexglue-sdk/src/graphics/vulkan/graphics_system.cpp +++ b/thirdparty/rexglue-sdk/src/graphics/vulkan/graphics_system.cpp @@ -24,11 +24,8 @@ std::string VulkanGraphicsSystem::name() const { return "Vulkan"; } -X_STATUS VulkanGraphicsSystem::Setup(runtime::FunctionDispatcher* function_dispatcher, - system::KernelState* kernel_state, - ui::WindowedAppContext* app_context, bool with_presentation) { +void VulkanGraphicsSystem::CreateProvider(bool with_presentation) { provider_ = rex::ui::vulkan::VulkanProvider::Create(true, with_presentation); - return GraphicsSystem::Setup(function_dispatcher, kernel_state, app_context, with_presentation); } std::unique_ptr VulkanGraphicsSystem::CreateCommandProcessor() { diff --git a/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_threading.cpp b/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_threading.cpp index bf571d83..66d3ab22 100644 --- a/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_threading.cpp +++ b/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_threading.cpp @@ -1471,7 +1471,7 @@ ppc_ptr_result_t InterlockedPushEntrySList_entry(ppc_ptr_t plist assert_not_null(entry); alignas(8) X_SLIST_HEADER old_hdr = *plist_ptr; - alignas(8) X_SLIST_HEADER new_hdr = {0}; + alignas(8) X_SLIST_HEADER new_hdr = {}; uint32_t old_head = 0; do { old_hdr = *plist_ptr; diff --git a/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_video.cpp b/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_video.cpp index fcebf923..baacaab4 100644 --- a/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_video.cpp +++ b/thirdparty/rexglue-sdk/src/kernel/xboxkrnl/xboxkrnl_video.cpp @@ -27,23 +27,6 @@ #include #include -REXCVAR_DEFINE_INT32(video_mode_width, 1280, "GPU", "Guest video mode width in pixels") - .range(640, 0x0FFF) - .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); - -REXCVAR_DEFINE_INT32(video_mode_height, 720, "GPU", "Guest video mode height in pixels") - .range(480, 0x0FFF) - .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); - -REXCVAR_DEFINE_STRING(resolution, "", "GPU", - "Common resolution preset for both guest video mode and startup window (for " - "example: 720p, 1080p, 1440p, 4k, 1280x720)") - .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); - -REXCVAR_DEFINE_DOUBLE(video_mode_refresh_rate, 60.0, "GPU", "Guest video mode refresh rate in Hz") - .range(24.0, 240.0) - .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); - namespace { // Display gamma type: 0 - linear, 1 - sRGB (CRT), 2 - BT.709 (HDTV), 3 - power constexpr uint32_t kDisplayGammaType = 2; diff --git a/thirdparty/rexglue-sdk/src/native/audio/render_driver_frame_layout.cpp b/thirdparty/rexglue-sdk/src/native/audio/render_driver_frame_layout.cpp index 6c3fabe3..5f958ead 100644 --- a/thirdparty/rexglue-sdk/src/native/audio/render_driver_frame_layout.cpp +++ b/thirdparty/rexglue-sdk/src/native/audio/render_driver_frame_layout.cpp @@ -4,24 +4,20 @@ #include #include -#include #include #include #include -#include #include -REXCVAR_DEFINE_STRING(audio_render_driver_layout, "auto", "Audio", - "Layout for XAudio render-driver frames: auto, planar, or interleaved") +REXCVAR_DEFINE_STRING(audio_render_driver_layout, "planar", "Audio", + "Layout for XAudio render-driver frames: planar, interleaved, or auto") .allowed({"auto", "planar", "interleaved"}) .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); namespace rex::audio::conversion { namespace { -std::atomic g_detected_layout{0}; - float DecodeSanitizedSample(const float* input, const size_t ch_sample_count, const size_t sample, const size_t channel, const RenderDriverFrameLayout layout) { const size_t index = layout == RenderDriverFrameLayout::kInterleaved @@ -41,7 +37,6 @@ struct LayoutScore { struct LayoutDetectionResult { RenderDriverFrameLayout layout = RenderDriverFrameLayout::kPlanar; - bool cacheable = false; }; LayoutScore ScoreLayout(const float* input, const size_t ch_sample_count, @@ -77,15 +72,14 @@ LayoutDetectionResult DetectFrameLayout(const float* input, const size_t ch_samp constexpr double kDecisionRatio = 0.75; if (planar.continuity * kDecisionRatio < interleaved.continuity) { - return {RenderDriverFrameLayout::kPlanar, true}; + return {RenderDriverFrameLayout::kPlanar}; } if (interleaved.continuity * kDecisionRatio < planar.continuity) { - return {RenderDriverFrameLayout::kInterleaved, true}; + return {RenderDriverFrameLayout::kInterleaved}; } return {planar.continuity <= interleaved.continuity ? RenderDriverFrameLayout::kPlanar - : RenderDriverFrameLayout::kInterleaved, - false}; + : RenderDriverFrameLayout::kInterleaved}; } } // namespace @@ -100,28 +94,7 @@ RenderDriverFrameLayout ResolveRenderDriverFrameLayout(const float* input, return RenderDriverFrameLayout::kInterleaved; } - const int cached_layout = g_detected_layout.load(std::memory_order_acquire); - if (cached_layout == 1) { - return RenderDriverFrameLayout::kPlanar; - } - if (cached_layout == 2) { - return RenderDriverFrameLayout::kInterleaved; - } - - const LayoutDetectionResult detection = DetectFrameLayout(input, ch_sample_count); - if (!detection.cacheable) { - return detection.layout; - } - - const RenderDriverFrameLayout detected_layout = detection.layout; - const int detected_value = - detected_layout == RenderDriverFrameLayout::kInterleaved ? 2 : 1; - const int previous = - g_detected_layout.exchange(detected_value, std::memory_order_acq_rel); - if (previous == 0) { - REXAPU_INFO("Audio render-driver layout auto-detected: {}", ToString(detected_layout)); - } - return detected_layout; + return DetectFrameLayout(input, ch_sample_count).layout; } const char* ToString(const RenderDriverFrameLayout layout) { diff --git a/thirdparty/rexglue-sdk/src/native/filesystem/devices/null_device.cpp b/thirdparty/rexglue-sdk/src/native/filesystem/devices/null_device.cpp index fdeceb29..afdb92e6 100644 --- a/thirdparty/rexglue-sdk/src/native/filesystem/devices/null_device.cpp +++ b/thirdparty/rexglue-sdk/src/native/filesystem/devices/null_device.cpp @@ -20,7 +20,7 @@ NullDevice::NullDevice(const std::string& mount_path, NullDevice::~NullDevice() = default; bool NullDevice::Initialize() { - auto root_entry = new NullEntry(this, nullptr, mount_path_); + auto root_entry = new NullEntry(this, nullptr, ""); root_entry->attributes_ = kFileAttributeDirectory; root_entry_ = std::unique_ptr(root_entry); @@ -37,7 +37,7 @@ void NullDevice::Dump(string::StringBuffer* string_buffer) { } Entry* NullDevice::ResolvePath(const std::string_view path) { - REXFS_INFO("NullDevice::ResolvePath({})", path); + REXFS_DEBUG("NullDevice::ResolvePath({})", path); auto root = root_entry_.get(); if (path.empty()) { diff --git a/thirdparty/rexglue-sdk/src/native/filesystem/devices/stfs_container_device.cpp b/thirdparty/rexglue-sdk/src/native/filesystem/devices/stfs_container_device.cpp index 16420896..70625202 100644 --- a/thirdparty/rexglue-sdk/src/native/filesystem/devices/stfs_container_device.cpp +++ b/thirdparty/rexglue-sdk/src/native/filesystem/devices/stfs_container_device.cpp @@ -185,7 +185,7 @@ Entry* StfsContainerDevice::ResolvePath(const std::string_view path) { // The filesystem will have stripped our prefix off already, so the path will // be in the form: // some\PATH.foo - REXFS_INFO("StfsContainerDevice::ResolvePath({})", path); + REXFS_DEBUG("StfsContainerDevice::ResolvePath({})", path); return root_entry_->ResolvePath(path); } diff --git a/thirdparty/rexglue-sdk/src/native/filesystem/virtual_file_system.cpp b/thirdparty/rexglue-sdk/src/native/filesystem/virtual_file_system.cpp index b8e5e101..958ba457 100644 --- a/thirdparty/rexglue-sdk/src/native/filesystem/virtual_file_system.cpp +++ b/thirdparty/rexglue-sdk/src/native/filesystem/virtual_file_system.cpp @@ -123,12 +123,20 @@ Entry* VirtualFileSystem::ResolvePath(const std::string_view path) { auto* entry = device->ResolvePath(relative_path); if (entry) { - REXFS_INFO("VFS: '{}' -> '{}' -> device '{}' -> host '{}'", path, - had_symlink ? normalized_path : "(no symlink)", device->mount_path(), - entry->absolute_path()); + if (had_symlink) { + REXFS_TRACE("VFS resolved '{}' via symlink '{}' on device '{}' -> '{}'", path, + normalized_path, device->mount_path(), entry->absolute_path()); + } else { + REXFS_TRACE("VFS resolved '{}' on device '{}' -> '{}'", path, device->mount_path(), + entry->absolute_path()); + } } else { - REXFS_WARN("VFS: '{}' -> '{}' -> device '{}' -> [entry not found]", path, - had_symlink ? normalized_path : "(no symlink)", device->mount_path()); + if (had_symlink) { + REXFS_WARN("VFS: entry not found for '{}' (via symlink '{}') on device '{}'", path, + normalized_path, device->mount_path()); + } else { + REXFS_WARN("VFS: entry not found for '{}' on device '{}'", path, device->mount_path()); + } } return entry; diff --git a/thirdparty/rexglue-sdk/src/native/ui/d3d12/d3d12_util.cpp b/thirdparty/rexglue-sdk/src/native/ui/d3d12/d3d12_util.cpp index 1fcd6af6..65ef8791 100644 --- a/thirdparty/rexglue-sdk/src/native/ui/d3d12/d3d12_util.cpp +++ b/thirdparty/rexglue-sdk/src/native/ui/d3d12/d3d12_util.cpp @@ -16,10 +16,6 @@ namespace rex::ui::d3d12 { namespace util { -const D3D12_HEAP_PROPERTIES kHeapPropertiesDefault = {D3D12_HEAP_TYPE_DEFAULT}; -const D3D12_HEAP_PROPERTIES kHeapPropertiesUpload = {D3D12_HEAP_TYPE_UPLOAD}; -const D3D12_HEAP_PROPERTIES kHeapPropertiesReadback = {D3D12_HEAP_TYPE_READBACK}; - ID3D12RootSignature* CreateRootSignature(const D3D12Provider& provider, const D3D12_ROOT_SIGNATURE_DESC& desc) { ID3DBlob* blob; diff --git a/thirdparty/rexglue-sdk/src/native/ui/window.cpp b/thirdparty/rexglue-sdk/src/native/ui/window.cpp index 06e000f7..ed57c787 100644 --- a/thirdparty/rexglue-sdk/src/native/ui/window.cpp +++ b/thirdparty/rexglue-sdk/src/native/ui/window.cpp @@ -32,6 +32,23 @@ REXCVAR_DEFINE_INT32(monitor, 0, "UI/Window", .range(0, 16) .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); +REXCVAR_DEFINE_INT32(video_mode_width, 1280, "GPU", "Guest video mode width in pixels") + .range(640, 0x0FFF) + .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); + +REXCVAR_DEFINE_INT32(video_mode_height, 720, "GPU", "Guest video mode height in pixels") + .range(480, 0x0FFF) + .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); + +REXCVAR_DEFINE_STRING(resolution, "", "GPU", + "Common resolution preset for both guest video mode and startup window (for " + "example: 720p, 1080p, 1440p, 4k, 1280x720)") + .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); + +REXCVAR_DEFINE_DOUBLE(video_mode_refresh_rate, 60.0, "GPU", "Guest video mode refresh rate in Hz") + .range(24.0, 240.0) + .lifecycle(rex::cvar::Lifecycle::kRequiresRestart); + namespace rex { namespace ui { diff --git a/thirdparty/rexglue-sdk/src/native/ui/window_win.cpp b/thirdparty/rexglue-sdk/src/native/ui/window_win.cpp index 50d87d86..14531469 100644 --- a/thirdparty/rexglue-sdk/src/native/ui/window_win.cpp +++ b/thirdparty/rexglue-sdk/src/native/ui/window_win.cpp @@ -1243,7 +1243,7 @@ LRESULT Win32Window::WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lPar TABLET_DISABLE_TOUCHUIFORCEON | TABLET_ENABLE_MULTITOUCHDATA; case WM_MENUCOMMAND: { - MENUINFO menu_info = {0}; + MENUINFO menu_info = {}; menu_info.cbSize = sizeof(menu_info); menu_info.fMask = MIM_MENUDATA; GetMenuInfo(HMENU(lParam), &menu_info); @@ -1331,7 +1331,7 @@ Win32MenuItem::Win32MenuItem(Type type, const std::string& text, const std::stri break; } if (handle_) { - MENUINFO menu_info = {0}; + MENUINFO menu_info = {}; menu_info.cbSize = sizeof(menu_info); menu_info.fMask = MIM_MENUDATA | MIM_STYLE; menu_info.dwMenuData = ULONG_PTR(this); diff --git a/thirdparty/rexglue-sdk/src/system/CMakeLists.txt b/thirdparty/rexglue-sdk/src/system/CMakeLists.txt index e90504e9..3bf813fd 100644 --- a/thirdparty/rexglue-sdk/src/system/CMakeLists.txt +++ b/thirdparty/rexglue-sdk/src/system/CMakeLists.txt @@ -20,6 +20,8 @@ set(REXSYSTEM_SOURCES xsymboliclink.cpp xobject.cpp xmodule.cpp + shared_library.cpp + guest_path.cpp # Utilities util/object_table.cpp @@ -70,6 +72,8 @@ target_link_libraries(rexsystem if(WIN32) target_link_libraries(rexsystem PRIVATE ws2_32) +elseif(UNIX) + target_link_libraries(rexsystem PRIVATE dl) endif() # Propagate graphics backend selection diff --git a/thirdparty/rexglue-sdk/src/system/function_dispatcher.cpp b/thirdparty/rexglue-sdk/src/system/function_dispatcher.cpp index ae4d40b1..2365217d 100644 --- a/thirdparty/rexglue-sdk/src/system/function_dispatcher.cpp +++ b/thirdparty/rexglue-sdk/src/system/function_dispatcher.cpp @@ -19,10 +19,40 @@ #include #include #include +#include #include namespace rex::runtime { +namespace { + +FunctionDispatcher* GetBoundFunctionDispatcher() { + auto* thread_state = ThreadState::Get(); + if (!thread_state || !thread_state->context() || !thread_state->context()->kernel_state) { + return nullptr; + } + return thread_state->context()->kernel_state->function_dispatcher(); +} + +} // namespace + +static void InvalidFunctionTrap(PPCContext& ctx, uint8_t* /*base*/) { + REX_FATAL("Call to invalid or unregistered indirect function (ctr=0x{:08X})", ctx.ctr.u32); +} + +PPCFunc* ResolveIndirectFunction(uint32_t guest_address) { + FunctionDispatcher* dispatcher = GetBoundFunctionDispatcher(); + if (!dispatcher) { + return &InvalidFunctionTrap; + } + + if (PPCFunc* func = dispatcher->GetFunction(guest_address)) { + return func; + } + + return &InvalidFunctionTrap; +} + FunctionDispatcher::FunctionDispatcher(rex::memory::Memory* memory, ExportResolver* export_resolver) : memory_(memory), export_resolver_(export_resolver) {} @@ -31,29 +61,29 @@ FunctionDispatcher::~FunctionDispatcher() = default; bool FunctionDispatcher::Execute(ThreadState* thread_state, uint32_t address) { SCOPE_profile_cpu_f("cpu"); - // rexglue: Look up pre-compiled function - auto fn = GetFunction(address); + PPCFunc* fn = GetFunction(address); if (!fn) { REXCPU_ERROR("Execute({:08X}): function not in function table", address); return false; } auto* ctx = thread_state->context(); + auto* previous_thread_state = ThreadState::Get(); - // Pad out stack a bit, as some games seem to overwrite the caller by about - // 16 to 32b. + // Rebind the active guest thread for cross-module callbacks. + ThreadState::Bind(thread_state); + + // Pad out stack a bit, as some games seem to overwrite the caller by about 16 to 32b. ctx->r1.u64 -= 64 + 112; - // This could be set to anything to give us a unique identifier to track - // re-entrancy/etc. uint64_t previous_lr = ctx->lr; ctx->lr = 0xBCBCBCBC; - // NOTE(tomc): rexglue direct function call fn(*ctx, memory_->virtual_membase()); ctx->lr = previous_lr; ctx->r1.u64 += 64 + 112; + ThreadState::Bind(previous_thread_state); return true; } @@ -64,7 +94,6 @@ uint64_t FunctionDispatcher::Execute(ThreadState* thread_state, uint32_t address auto* ctx = thread_state->context(); - // Set up arguments (rexglue uses named registers) if (arg_count > 0) ctx->r3.u64 = args[0]; if (arg_count > 1) @@ -82,9 +111,8 @@ uint64_t FunctionDispatcher::Execute(ThreadState* thread_state, uint32_t address if (arg_count > 7) ctx->r10.u64 = args[7]; + // FIXME: stack-arg path assumes 32-bit values; 64-bit and float args are wrong. if (arg_count > 8) { - // Rest of the arguments go on the stack. - // FIXME: This assumes arguments are 32 bits! auto stack_arg_base = memory_->TranslateVirtual(static_cast(ctx->r1.u64) + 0x54 - (64 + 112)); for (size_t i = 8; i < arg_count; i++) { @@ -104,14 +132,11 @@ uint64_t FunctionDispatcher::ExecuteInterrupt(ThreadState* thread_state, uint32_ SCOPE_profile_cpu_f("cpu"); // Hold the global lock during interrupt dispatch. - // This will block if any code is in a critical region (has interrupts - // disabled) or if any other interrupt is executing. auto global_lock = global_critical_region_.Acquire(); auto* ctx = thread_state->context(); assert_true(arg_count <= 5); - // Set up arguments (rexglue uses named registers) if (arg_count > 0) ctx->r3.u64 = args[0]; if (arg_count > 1) @@ -123,8 +148,8 @@ uint64_t FunctionDispatcher::ExecuteInterrupt(ThreadState* thread_state, uint32_ if (arg_count > 4) ctx->r7.u64 = args[4]; - // TLS ptr must be zero during interrupts. Some games check this and - // early-exit routines when under interrupts. + // TLS ptr must be zero during interrupts. Some games check this and early-exit + // routines when under interrupts. auto pcr_address = memory_->TranslateVirtual(static_cast(ctx->r13.u64)); uint32_t old_tls_ptr = memory::load_and_swap(pcr_address); memory::store_and_swap(pcr_address, 0); @@ -133,52 +158,110 @@ uint64_t FunctionDispatcher::ExecuteInterrupt(ThreadState* thread_state, uint32_ return 0xDEADBABE; } - // Restores TLS ptr. + // Restore TLS ptr. memory::store_and_swap(pcr_address, old_tls_ptr); return ctx->r3.u64; } // rexglue function table management + bool FunctionDispatcher::InitializeFunctionTable(uint32_t code_base, uint32_t code_size, - uint32_t image_base, uint32_t image_size) { - if (function_table_initialized_) { - REXLOG_WARN("Function table already initialized"); + uint32_t image_base, uint32_t image_size, + bool is_entrypoint) { + std::lock_guard lock(dispatch_mutex_); + + if (is_entrypoint && entrypoint_code_base_ != 0) { + REXLOG_ERROR("InitializeFunctionTable: entrypoint already registered at {:08X}", + entrypoint_code_base_); return false; } - // Initialize the guest memory function table (for PPC_LOOKUP_FUNC in recompiled code) + uint32_t new_table_end = image_base + image_size + (code_size + kThunkReserveSize) * 2; + uint32_t new_code_end = code_base + code_size + kThunkReserveSize; + for (const auto& existing : module_tables_) { + uint32_t existing_table_end = + existing.image_base + existing.image_size + (existing.code_size + kThunkReserveSize) * 2; + uint32_t existing_code_end = existing.code_base + existing.code_size + kThunkReserveSize; + if (image_base < existing_table_end && new_table_end > existing.image_base) { + REXLOG_ERROR("Module image range [{:08X}, {:08X}) overlaps existing [{:08X}, {:08X})", + image_base, new_table_end, existing.image_base, existing_table_end); + return false; + } + if (code_base < existing_code_end && new_code_end > existing.code_base) { + REXLOG_ERROR("Module code range [{:08X}, {:08X}) overlaps existing [{:08X}, {:08X})", + code_base, new_code_end, existing.code_base, existing_code_end); + return false; + } + } + if (!memory_->InitializeFunctionTable(code_base, code_size, image_base, image_size)) { REXLOG_ERROR("Failed to initialize guest memory function table"); return false; } - code_base_ = code_base; - code_size_ = code_size; - image_base_ = image_base; - image_size_ = image_size; - function_table_initialized_ = true; + module_tables_.push_back({ + .code_base = code_base, + .code_size = code_size, + .image_base = image_base, + .image_size = image_size, + .next_thunk_address = code_base + code_size, + .thunk_limit = code_base + code_size + kThunkReserveSize, + }); - // Initialize thunk allocation region (immediately after code section) - next_thunk_address_ = code_base + code_size; - thunk_limit_ = next_thunk_address_ + 0x10000; - REXLOG_INFO( - "FunctionDispatcher function table initialized: code={:08X}-{:08X}, image={:08X}-{:08X}", - code_base, code_base + code_size, image_base, image_base + image_size); + if (is_entrypoint) { + entrypoint_code_base_ = code_base; + } + + REXLOG_INFO("Function table initialized for module: code={:08X}-{:08X}, image={:08X}-{:08X}", + code_base, code_base + code_size, image_base, image_base + image_size); return true; } -void FunctionDispatcher::SetFunction(uint32_t guest_address, ::PPCFunc* func) { - assert_true(function_table_initialized_); +FunctionDispatcher::ModuleTableInfo* FunctionDispatcher::FindModuleByAddress( + uint32_t guest_address) { + for (auto& mod : module_tables_) { + if (guest_address >= mod.code_base && guest_address < mod.thunk_limit) { + return &mod; + } + } + return nullptr; +} + +uint32_t FunctionDispatcher::FindCallerModuleBase(uint32_t guest_address) { + std::lock_guard lock(dispatch_mutex_); + if (auto* mod = FindModuleByAddress(guest_address)) { + return mod->code_base; + } + return 0; +} + +bool FunctionDispatcher::SetFunction(uint32_t guest_address, ::PPCFunc* func) { + std::lock_guard lock(dispatch_mutex_); + assert_true(!module_tables_.empty()); + + if (!FindModuleByAddress(guest_address)) { + REXLOG_ERROR("SetFunction: address {:08X} outside all registered module ranges", guest_address); + return false; + } - // Store in C++ map (for FunctionDispatcher::Execute/GetFunction) function_table_[guest_address] = func; - // Also write to guest memory (for PPC_LOOKUP_FUNC in recompiled code) - memory_->SetFunction(guest_address, func); + if (!memory_->SetFunction(guest_address, func)) { + REXLOG_ERROR("SetFunction: dispatcher / Memory module-table state out of sync at {:08X}", + guest_address); + function_table_.erase(guest_address); + return false; + } + + if (recording_) { + recording_addresses_.push_back(guest_address); + } + return true; } ::PPCFunc* FunctionDispatcher::GetFunction(uint32_t guest_address) { + std::lock_guard lock(dispatch_mutex_); auto it = function_table_.find(guest_address); if (it != function_table_.end()) { return it->second; @@ -186,15 +269,119 @@ void FunctionDispatcher::SetFunction(uint32_t guest_address, ::PPCFunc* func) { return nullptr; } -uint32_t FunctionDispatcher::AllocateThunk(::PPCFunc* func) { - if (next_thunk_address_ >= thunk_limit_) { - REXLOG_ERROR("Thunk address space exhausted"); +uint32_t FunctionDispatcher::AllocateThunk(::PPCFunc* func, uint32_t caller_address) { + std::lock_guard lock(dispatch_mutex_); + auto* mod = FindModuleByAddress(caller_address); + if (!mod) { + if (caller_address != 0) { + REXLOG_ERROR("AllocateThunk: caller_address {:08X} not in any registered module", + caller_address); + return 0; + } + if (entrypoint_code_base_ == 0) { + REXLOG_ERROR("AllocateThunk: caller_address=0 but no entrypoint registered"); + return 0; + } + mod = FindModuleByAddress(entrypoint_code_base_); + if (!mod) { + REXLOG_ERROR("AllocateThunk: entrypoint code_base {:08X} not in module_tables_", + entrypoint_code_base_); + return 0; + } + } + + if (mod->next_thunk_address >= mod->thunk_limit) { + REXLOG_ERROR("Thunk address space exhausted for module at {:08X}", mod->code_base); + return 0; + } + uint32_t addr = mod->next_thunk_address; + mod->next_thunk_address += 4; + if (!SetFunction(addr, func)) { + mod->next_thunk_address -= 4; return 0; } - uint32_t addr = next_thunk_address_; - next_thunk_address_ += 4; // 4-byte aligned - SetFunction(addr, func); return addr; } +void FunctionDispatcher::RegisterModule(const std::string& module_id, uint32_t code_base, + RegisterFn register_func) { + std::lock_guard lock(dispatch_mutex_); + if (recording_) { + REX_FATAL("RegisterModule called while already recording (re-entrancy)"); + return; + } + + if (module_addresses_.find(module_id) != module_addresses_.end()) { + REXLOG_WARN("RegisterModule: '{}' is already registered; cleaning up prior batch", module_id); + UnregisterModule(module_id); + } + + REXLOG_INFO("Registering module: {} (code_base={:08X})", module_id, code_base); + + recording_addresses_.clear(); + recording_ = true; + + struct RecordingGuard { + FunctionDispatcher* self; + ~RecordingGuard() { + self->recording_ = false; + self->recording_addresses_.clear(); + } + } guard{this}; + + register_func(this); + + ModuleRegistration reg; + reg.code_base = code_base; + reg.addresses = std::move(recording_addresses_); + + size_t count = reg.addresses.size(); + module_addresses_[module_id] = std::move(reg); + + REXLOG_INFO("Module '{}' registered {} functions", module_id, count); +} + +std::optional> FunctionDispatcher::UnregisterModule( + const std::string& module_id) { + std::lock_guard lock(dispatch_mutex_); + auto it = module_addresses_.find(module_id); + if (it == module_addresses_.end()) { + REXLOG_WARN("UnregisterModule: module '{}' not found", module_id); + return std::nullopt; + } + + REXLOG_INFO("Unregistering module: {} ({} functions)", module_id, it->second.addresses.size()); + + auto table_it = std::find_if(module_tables_.begin(), module_tables_.end(), + [code_base = it->second.code_base](const ModuleTableInfo& mti) { + return mti.code_base == code_base; + }); + + for (uint32_t addr : it->second.addresses) { + function_table_.erase(addr); + memory_->SetFunction(addr, nullptr); + } + + std::optional> cleared_range; + if (table_it != module_tables_.end()) { + uint32_t pool_start = table_it->code_base + table_it->code_size; + uint32_t pool_end = table_it->next_thunk_address; + for (uint32_t addr = pool_start; addr < pool_end; addr += 4) { + function_table_.erase(addr); + memory_->SetFunction(addr, nullptr); + } + cleared_range = std::make_pair(pool_start, pool_end); + + if (table_it->code_base == entrypoint_code_base_) { + entrypoint_code_base_ = 0; + } + memory_->DestroyFunctionTable(table_it->code_base); + module_tables_.erase(table_it); + } + + module_addresses_.erase(it); + + return cleared_range; +} + } // namespace rex::runtime diff --git a/thirdparty/rexglue-sdk/src/system/guest_path.cpp b/thirdparty/rexglue-sdk/src/system/guest_path.cpp new file mode 100644 index 00000000..25a5d14c --- /dev/null +++ b/thirdparty/rexglue-sdk/src/system/guest_path.cpp @@ -0,0 +1,44 @@ +/** + * @file system/guest_path.cpp + * @brief Guest path normalization for Xbox 360 VFS paths + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include + +#include +#include + +#include + +namespace rex::system { + +std::string NormalizeGuestPath(std::string_view path) { + // Manifest / config consumers expect POSIX-style guest paths: forward + // slashes, no device prefix, lowercase. The runtime VFS tolerates either + // separator, so converting backslashes here keeps writers (TOML) happy. + std::string result = rex::string::utf8_fix_path_separators(path, U'/'); + + auto colon = result.find(':'); + if (colon != std::string::npos && colon + 1 < result.size() && result[colon + 1] == '/') { + result.erase(0, colon + 2); + } + + result = rex::string::utf8_canonicalize_path(result, U'/'); + + while (!result.empty() && result.front() == '/') { + result.erase(result.begin()); + } + + std::transform(result.begin(), result.end(), result.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + return result; +} + +} // namespace rex::system diff --git a/thirdparty/rexglue-sdk/src/system/kernel_module.cpp b/thirdparty/rexglue-sdk/src/system/kernel_module.cpp index ec58e46e..5a004f32 100644 --- a/thirdparty/rexglue-sdk/src/system/kernel_module.cpp +++ b/thirdparty/rexglue-sdk/src/system/kernel_module.cpp @@ -1,13 +1,19 @@ /** - * ReXGlue runtime - AC6 Recompilation project - * Copyright (c) 2026 Tom Clay. All rights reserved. + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + * + * @modified Tom Clay, 2026 - Adapted for ReXGlue runtime */ #include #include #include -#include #include +#include #include namespace rex::system { @@ -27,7 +33,7 @@ KernelModule::KernelModule(KernelState* kernel_state, const std::string_view pat KernelModule::~KernelModule() {} -uint32_t KernelModule::GetProcAddressByOrdinal(uint16_t ordinal) { +uint32_t KernelModule::GetProcAddressByOrdinal(uint16_t ordinal, uint32_t caller_address) { // Look up the export in the resolver auto export_entry = export_resolver_->GetExportByOrdinal(name_, ordinal); if (!export_entry) { @@ -42,8 +48,12 @@ uint32_t KernelModule::GetProcAddressByOrdinal(uint16_t ordinal) { return export_entry->variable_ptr; } - // Check thunk cache first (already allocated) - auto thunk_it = thunk_cache_.find(ordinal); + auto* dispatcher = kernel_state_->function_dispatcher(); + uint32_t caller_module_base = dispatcher->FindCallerModuleBase(caller_address); + ThunkKey key{caller_module_base, ordinal}; + + // Check thunk cache first (already allocated for this caller's module) + auto thunk_it = thunk_cache_.find(key); if (thunk_it != thunk_cache_.end()) { REXSYS_DEBUG("GetProcAddressByOrdinal: {} ({:04X}) in {} -> cached thunk {:08X}", export_entry->name, ordinal, name_, thunk_it->second); @@ -55,10 +65,9 @@ uint32_t KernelModule::GetProcAddressByOrdinal(uint16_t ordinal) { REXSYS_DEBUG("GetProcAddressByOrdinal: searching registry for '{}'", imp_name); PPCFunc* func = rex::FindPPCFuncByName(imp_name.c_str()); if (func) { - auto* dispatcher = kernel_state_->function_dispatcher(); - uint32_t thunk_addr = dispatcher->AllocateThunk(func); + uint32_t thunk_addr = dispatcher->AllocateThunk(func, caller_address); if (thunk_addr) { - thunk_cache_[ordinal] = thunk_addr; + thunk_cache_[key] = thunk_addr; REXSYS_INFO("GetProcAddressByOrdinal: {} ({:04X}) in {} -> thunk at {:08X}", export_entry->name, ordinal, name_, thunk_addr); return thunk_addr; @@ -78,4 +87,14 @@ uint32_t KernelModule::GetProcAddressByName(const std::string_view name) { return 0; } +void KernelModule::InvalidateThunkCacheInRange(uint32_t lo, uint32_t hi) { + for (auto it = thunk_cache_.begin(); it != thunk_cache_.end();) { + if (it->second >= lo && it->second < hi) { + it = thunk_cache_.erase(it); + } else { + ++it; + } + } +} + } // namespace rex::system diff --git a/thirdparty/rexglue-sdk/src/system/mmio_handler.cpp b/thirdparty/rexglue-sdk/src/system/mmio_handler.cpp index 1bab6a0d..a2d5ade7 100644 --- a/thirdparty/rexglue-sdk/src/system/mmio_handler.cpp +++ b/thirdparty/rexglue-sdk/src/system/mmio_handler.cpp @@ -21,6 +21,10 @@ namespace rex::runtime { MMIOHandler* MMIOHandler::global_handler_ = nullptr; +MMIOHandler* MMIOHandler::global_handler() { + return global_handler_; +} + std::unique_ptr MMIOHandler::Install(uint8_t* virtual_membase, uint8_t* physical_membase, uint8_t* membase_end, HostToGuestVirtual host_to_guest_virtual, @@ -38,7 +42,7 @@ std::unique_ptr MMIOHandler::Install(uint8_t* virtual_membase, host_to_guest_virtual_context, access_violation_callback, access_violation_callback_context)); // Install exception handler for memory coherence (SharedMemory write tracking). - // Note: MMIO operations are handled at the recompiler level via PPC_MM_LOAD/STORE + // Note: MMIO operations are handled at the recompiler level via REX_MM_LOAD/STORE // macros that call CheckLoad/CheckStore directly. arch::ExceptionHandler::Install(ExceptionCallbackThunk, handler.get()); diff --git a/thirdparty/rexglue-sdk/src/system/shared_library.cpp b/thirdparty/rexglue-sdk/src/system/shared_library.cpp new file mode 100644 index 00000000..2de192b6 --- /dev/null +++ b/thirdparty/rexglue-sdk/src/system/shared_library.cpp @@ -0,0 +1,121 @@ +/** + * @file system/shared_library.cpp + * @brief Platform-agnostic shared library loader + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include + +#include + +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace rex::system { + +namespace { + +#ifdef _WIN32 +std::string FormatLastError(DWORD err) { + char* buffer = nullptr; + DWORD len = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), reinterpret_cast(&buffer), 0, + nullptr); + std::string msg = (len && buffer) ? std::string(buffer, len) : std::string{}; + if (buffer) { + LocalFree(buffer); + } + while (!msg.empty() && (msg.back() == '\n' || msg.back() == '\r' || msg.back() == ' ')) { + msg.pop_back(); + } + return msg.empty() ? fmt::format("error {}", err) : fmt::format("{} (error {})", msg, err); +} +#endif + +} // namespace + +SharedLibrary::~SharedLibrary() { + Close(); +} + +SharedLibrary::SharedLibrary(SharedLibrary&& other) noexcept : handle_(other.handle_) { + other.handle_ = nullptr; +} + +SharedLibrary& SharedLibrary::operator=(SharedLibrary&& other) noexcept { + if (this != &other) { + Close(); + handle_ = other.handle_; + other.handle_ = nullptr; + } + return *this; +} + +bool SharedLibrary::Load(const std::string& name) { + assert_true(handle_ == nullptr); + if (handle_) { + REXSYS_ERROR("SharedLibrary::Load called over a live handle"); + return false; + } + auto exe_dir = rex::filesystem::GetExecutableFolder(); +#ifdef _WIN32 + std::string full_name = (exe_dir / (name + ".dll")).string(); + handle_ = LoadLibraryA(full_name.c_str()); + if (!handle_) { + DWORD err = GetLastError(); + REXSYS_ERROR("Failed to load shared library '{}': {}", full_name, FormatLastError(err)); + return false; + } +#else + std::string full_name = (exe_dir / ("lib" + name + ".so")).string(); + handle_ = dlopen(full_name.c_str(), RTLD_NOW); + if (!handle_) { + const char* err = dlerror(); + REXSYS_ERROR("Failed to load shared library '{}': {}", full_name, err ? err : "unknown"); + return false; + } +#endif + return true; +} + +void* SharedLibrary::GetSymbol(const char* name) { + if (!handle_) + return nullptr; +#ifdef _WIN32 + return reinterpret_cast(GetProcAddress(static_cast(handle_), name)); +#else + return dlsym(handle_, name); +#endif +} + +void SharedLibrary::Close() { + if (!handle_) { + return; + } +#ifdef _WIN32 + if (!FreeLibrary(static_cast(handle_))) { + REXSYS_ERROR("FreeLibrary failed: {}", FormatLastError(GetLastError())); + } +#else + if (dlclose(handle_) != 0) { + const char* err = dlerror(); + REXSYS_ERROR("dlclose failed: {}", err ? err : "unknown"); + } +#endif + handle_ = nullptr; +} + +} // namespace rex::system diff --git a/thirdparty/rexglue-sdk/src/system/thread_state.cpp b/thirdparty/rexglue-sdk/src/system/thread_state.cpp index ab78f13f..af5e7c91 100644 --- a/thirdparty/rexglue-sdk/src/system/thread_state.cpp +++ b/thirdparty/rexglue-sdk/src/system/thread_state.cpp @@ -52,7 +52,8 @@ ThreadState* ThreadState::Get() { } uint32_t ThreadState::GetThreadID() { - return thread_state_ ? thread_state_->thread_id_ : 0xFFFFFFFF; + auto* thread_state = Get(); + return thread_state ? thread_state->thread_id_ : 0xFFFFFFFF; } } // namespace rex::runtime diff --git a/thirdparty/rexglue-sdk/src/system/user_module.cpp b/thirdparty/rexglue-sdk/src/system/user_module.cpp index 2237c47b..c82f802f 100644 --- a/thirdparty/rexglue-sdk/src/system/user_module.cpp +++ b/thirdparty/rexglue-sdk/src/system/user_module.cpp @@ -15,7 +15,8 @@ #include #include -REXCVAR_DEFINE_BOOL(xex_apply_patches, true, "Kernel", "Apply XEX patches"); +REXCVAR_DEFINE_BOOL(xex_apply_patches, false, "Kernel", + "Search for and apply XEX patches (path + 'p') on module load"); namespace rex::system { @@ -233,8 +234,19 @@ X_STATUS UserModule::Unload() { return X_STATUS_UNSUCCESSFUL; } -uint32_t UserModule::GetProcAddressByOrdinal(uint16_t ordinal) { - return xex_module()->GetProcAddress(ordinal); +uint32_t UserModule::GetProcAddressByOrdinal(uint16_t ordinal, uint32_t caller_address) { + uint32_t guest_addr = xex_module()->GetProcAddress(ordinal); + if (!guest_addr || !caller_address) { + return guest_addr; + } + + auto* dispatcher = kernel_state_->function_dispatcher(); + auto* func = dispatcher->GetFunction(guest_addr); + if (!func) { + return guest_addr; + } + + return dispatcher->AllocateThunk(func, caller_address); } uint32_t UserModule::GetProcAddressByName(std::string_view name) { diff --git a/thirdparty/rexglue-sdk/src/system/util/xdbf_utils.cpp b/thirdparty/rexglue-sdk/src/system/util/xdbf_utils.cpp index dcb2ea08..ccf84543 100644 --- a/thirdparty/rexglue-sdk/src/system/util/xdbf_utils.cpp +++ b/thirdparty/rexglue-sdk/src/system/util/xdbf_utils.cpp @@ -53,7 +53,7 @@ XdbfBlock XdbfWrapper::GetEntry(XdbfSection section, uint64_t id) const { return block; } } - return {0}; + return {}; } std::string XdbfWrapper::GetStringTableEntry(XLanguage language, uint16_t string_id) const { diff --git a/thirdparty/rexglue-sdk/src/system/xam/app_manager.cpp b/thirdparty/rexglue-sdk/src/system/xam/app_manager.cpp index 064a6ad8..79fb865e 100644 --- a/thirdparty/rexglue-sdk/src/system/xam/app_manager.cpp +++ b/thirdparty/rexglue-sdk/src/system/xam/app_manager.cpp @@ -21,20 +21,28 @@ void AppManager::RegisterApp(std::unique_ptr app) { X_HRESULT AppManager::DispatchMessageSync(uint32_t app_id, uint32_t message, uint32_t buffer_ptr, uint32_t buffer_length) { - const auto& it = app_lookup_.find(app_id); - if (it == app_lookup_.end()) { - return X_E_NOTFOUND; + App* app; + { + auto it = app_lookup_.find(app_id); + if (it == app_lookup_.end()) { + return X_E_NOTFOUND; + } + app = it->second; } - return it->second->DispatchMessageSync(message, buffer_ptr, buffer_length); + return app->DispatchMessageSync(message, buffer_ptr, buffer_length); } X_HRESULT AppManager::DispatchMessageAsync(uint32_t app_id, uint32_t message, uint32_t buffer_ptr, uint32_t buffer_length) { - const auto& it = app_lookup_.find(app_id); - if (it == app_lookup_.end()) { - return X_E_NOTFOUND; + App* app; + { + auto it = app_lookup_.find(app_id); + if (it == app_lookup_.end()) { + return X_E_NOTFOUND; + } + app = it->second; } - return it->second->DispatchMessageSync(message, buffer_ptr, buffer_length); + return app->DispatchMessageSync(message, buffer_ptr, buffer_length); } } // namespace xam diff --git a/thirdparty/rexglue-sdk/src/system/xmemory.cpp b/thirdparty/rexglue-sdk/src/system/xmemory.cpp index ba04d6c3..ca7f4e86 100644 --- a/thirdparty/rexglue-sdk/src/system/xmemory.cpp +++ b/thirdparty/rexglue-sdk/src/system/xmemory.cpp @@ -1,6 +1,12 @@ /** - * ReXGlue runtime - AC6 Recompilation project - * Copyright (c) 2026 Tom Clay. All rights reserved. + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + * + * @modified Tom Clay, 2026 - Adapted for ReXGlue runtime */ #include @@ -14,6 +20,8 @@ #include #include #include +#include +#include #include #include #include @@ -120,7 +128,7 @@ Memory::~Memory() { } bool Memory::Initialize() { - file_name_ = fmt::format("rexglue_memory_{}", chrono::Clock::QueryHostTickCount()); + file_name_ = fmt::format("xenia_memory_{}", chrono::Clock::QueryHostTickCount()); // Create main page file-backed mapping. This is all reserved but // uncommitted (so it shouldn't expand page file). @@ -668,6 +676,27 @@ void Memory::DumpMap() { REXSYS_ERROR(""); } +bool Memory::Save(stream::ByteStream* stream) { + REXSYS_DEBUG("Serializing memory..."); + heaps_.v00000000.Save(stream); + heaps_.v40000000.Save(stream); + heaps_.v80000000.Save(stream); + heaps_.v90000000.Save(stream); + heaps_.physical.Save(stream); + + return true; +} + +bool Memory::Restore(stream::ByteStream* stream) { + REXSYS_DEBUG("Restoring memory..."); + heaps_.v00000000.Restore(stream); + heaps_.v40000000.Restore(stream); + heaps_.v80000000.Restore(stream); + heaps_.v90000000.Restore(stream); + heaps_.physical.Restore(stream); + + return true; +} //============================================================================= // Recompiled Code Function Table @@ -675,91 +704,87 @@ void Memory::DumpMap() { bool Memory::InitializeFunctionTable(uint32_t code_base, uint32_t code_size, uint32_t image_base, uint32_t image_size) { - if (function_table_base_ != 0) { - REXSYS_ERROR("Function table already initialized"); - return false; - } - - // The function table lives at IMAGE_BASE + IMAGE_SIZE in guest address space. - // Each 4-byte-aligned guest address gets an 8-byte slot for a host function pointer. - // Table size = (code_size + thunk_reserve) * 2 bytes (since offset = (addr - code_base) * 2). - // The thunk reserve provides space for runtime-allocated thunks (e.g. XexGetProcedureAddress). - constexpr uint32_t kThunkReserveSize = 0x10000; // 64KB = up to 16K thunks - function_table_base_ = image_base + image_size; - function_code_base_ = code_base; - function_code_size_ = code_size; - function_thunk_reserve_ = kThunkReserveSize; - + constexpr uint32_t kThunkReserveSize = runtime::FunctionDispatcher::kThunkReserveSize; + uint32_t table_base = image_base + image_size; uint32_t table_size = (code_size + kThunkReserveSize) * 2; REXSYS_DEBUG( "Initializing function table at {:08X}, size {:08X} for code {:08X}-{:08X} " "(+{:08X} thunk reserve)", - function_table_base_, table_size, code_base, code_base + code_size, kThunkReserveSize); + table_base, table_size, code_base, code_base + code_size, kThunkReserveSize); - // Allocate the function table region in guest memory. - // Use the 64k page heap (v80000000) since that's where XEX code lives. if (!heaps_.v80000000.AllocFixed( - function_table_base_, table_size, 0x10000, + table_base, table_size, 0x10000, memory::kMemoryAllocationReserve | memory::kMemoryAllocationCommit, memory::kMemoryProtectRead | memory::kMemoryProtectWrite)) { - REXSYS_ERROR("Failed to allocate function table at {:08X}", function_table_base_); - function_table_base_ = 0; + REXSYS_ERROR("Failed to allocate function table at {:08X}", table_base); return false; } - // Zero-initialize the table (nullptr for all entries). - Zero(function_table_base_, table_size); + Zero(table_base, table_size); + + std::lock_guard lock(function_tables_mutex_); + function_tables_.push_back({ + .table_base = table_base, + .code_base = code_base, + .code_size = code_size, + .thunk_reserve = kThunkReserveSize, + }); return true; } -void Memory::SetFunction(uint32_t guest_address, PPCFunc* host_function) { - if (function_table_base_ == 0) { - REXSYS_ERROR("SetFunction called before InitializeFunctionTable"); - return; +bool Memory::DestroyFunctionTable(uint32_t code_base) { + uint32_t table_base = 0; + uint32_t table_size = 0; + uint32_t logged_code_base = 0; + uint32_t logged_code_size = 0; + { + std::lock_guard lock(function_tables_mutex_); + auto it = std::find_if( + function_tables_.begin(), function_tables_.end(), + [code_base](const FunctionTableEntry& entry) { return entry.code_base == code_base; }); + if (it == function_tables_.end()) { + return false; + } + table_base = it->table_base; + table_size = (it->code_size + it->thunk_reserve) * 2; + logged_code_base = it->code_base; + logged_code_size = it->code_size; + function_tables_.erase(it); } - // Bounds check - addresses outside code section + thunk reserve are unexpected. - // IAT imports are called directly via __imp__ symbols, not through function table. - // Thunk reserve extends past code section for runtime-allocated thunks. - if (guest_address < function_code_base_ || - guest_address >= function_code_base_ + function_code_size_ + function_thunk_reserve_) { - REXSYS_DEBUG("SetFunction: skipping {:08X} (outside code+thunk range [{:08X}, {:08X}))", - guest_address, function_code_base_, - function_code_base_ + function_code_size_ + function_thunk_reserve_); - return; - } + REXSYS_DEBUG("Destroying function table at {:08X}, size {:08X} for code {:08X}-{:08X}", + table_base, table_size, logged_code_base, logged_code_base + logged_code_size); - // Calculate table offset: (guest_addr - code_base) * 2 - // This gives us the byte offset into the table for this 8-byte slot. - uint64_t offset = (uint64_t(guest_address) - function_code_base_) * 2; - uint32_t table_address = function_table_base_ + uint32_t(offset); - - // Write the host function pointer to the table. - // The table is in guest memory but stores host pointers. - auto* slot = TranslateVirtual(table_address); - *slot = host_function; + heaps_.v80000000.Release(table_base); + return true; } -PPCFunc* Memory::GetFunction(uint32_t guest_address) const { - if (function_table_base_ == 0) { - return nullptr; +bool Memory::SetFunction(uint32_t guest_address, PPCFunc* host_function) { + uint32_t table_address = 0; + { + std::lock_guard lock(function_tables_mutex_); + for (const auto& entry : function_tables_) { + uint32_t range_end = entry.code_base + entry.code_size + entry.thunk_reserve; + if (guest_address >= entry.code_base && guest_address < range_end) { + uint64_t offset = (uint64_t(guest_address) - entry.code_base) * 2; + table_address = entry.table_base + uint32_t(offset); + break; + } + } } - - // Bounds check (includes thunk reserve for runtime-allocated thunks) - if (guest_address < function_code_base_ || - guest_address >= function_code_base_ + function_code_size_ + function_thunk_reserve_) { - return nullptr; + if (!table_address) { + return false; } + auto* slot = TranslateVirtual(table_address); + *slot = host_function; + return true; +} - // Calculate table offset - uint64_t offset = (uint64_t(guest_address) - function_code_base_) * 2; - uint32_t table_address = function_table_base_ + uint32_t(offset); - - // Read the host function pointer from the table. - auto* slot = const_cast(this)->TranslateVirtual(table_address); - return *slot; +bool Memory::HasAnyFunctionTable() const { + std::lock_guard lock(function_tables_mutex_); + return !function_tables_.empty(); } rex::memory::PageAccess ToPageAccess(uint32_t protect) { @@ -908,6 +933,74 @@ uint32_t BaseHeap::GetUnreservedPageCount() { return count; } +bool BaseHeap::Save(stream::ByteStream* stream) { + REXSYS_DEBUG("Heap {:08X}-{:08X}", heap_base_, heap_base_ + (heap_size_ - 1)); + + for (size_t i = 0; i < page_table_.size(); i++) { + auto& page = page_table_[i]; + stream->Write(page.qword); + if (!page.state) { + // Unallocated. + continue; + } + + // TODO(DrChat): write compressed with snappy. + if (page.state & memory::kMemoryAllocationCommit) { + void* addr = TranslateRelative(i << page_size_shift_); + + memory::PageAccess old_access; + memory::Protect(addr, page_size_, memory::PageAccess::kReadWrite, &old_access); + + stream->Write(addr, page_size_); + + memory::Protect(addr, page_size_, old_access, nullptr); + } + } + + return true; +} + +bool BaseHeap::Restore(stream::ByteStream* stream) { + REXSYS_DEBUG("Heap {:08X}-{:08X}", heap_base_, heap_base_ + (heap_size_ - 1)); + + for (size_t i = 0; i < page_table_.size(); i++) { + auto& page = page_table_[i]; + page.qword = stream->Read(); + if (!page.state) { + // Unallocated. + continue; + } + + memory::PageAccess page_access = memory::PageAccess::kNoAccess; + if ((page.current_protect & memory::kMemoryProtectRead) && + (page.current_protect & memory::kMemoryProtectWrite)) { + page_access = memory::PageAccess::kReadWrite; + } else if (page.current_protect & memory::kMemoryProtectRead) { + page_access = memory::PageAccess::kReadOnly; + } + + // Commit the memory if it isn't already. We do not need to reserve any + // memory, as the mapping has already taken care of that. + if (page.state & memory::kMemoryAllocationCommit) { + rex::memory::AllocFixed(TranslateRelative(i << page_size_shift_), page_size_, + memory::AllocationType::kCommit, memory::PageAccess::kReadWrite); + } + + // Now read into memory. We'll set R/W protection first, then set the + // protection back to its previous state. + // TODO(DrChat): read compressed with snappy. + if (page.state & memory::kMemoryAllocationCommit) { + void* addr = TranslateRelative(i << page_size_shift_); + rex::memory::Protect(addr, page_size_, memory::PageAccess::kReadWrite, nullptr); + + stream->Read(addr, page_size_); + + rex::memory::Protect(addr, page_size_, page_access, nullptr); + } + } + + return true; +} void BaseHeap::Reset() { // TODO(DrChat): protect pages. diff --git a/thirdparty/rexglue-sdk/src/system/xthread.cpp b/thirdparty/rexglue-sdk/src/system/xthread.cpp index 272c8096..f64620e4 100644 --- a/thirdparty/rexglue-sdk/src/system/xthread.cpp +++ b/thirdparty/rexglue-sdk/src/system/xthread.cpp @@ -101,16 +101,24 @@ XThread::~XThread() { thread_local XThread* current_xthread_tls_ = nullptr; +namespace { + +XThread* GetBoundCurrentXThread() { + return current_xthread_tls_; +} + +} // namespace + bool XThread::IsInThread() { - return current_xthread_tls_ != nullptr; + return GetBoundCurrentXThread() != nullptr; } bool XThread::IsInThread(XThread* other) { - return current_xthread_tls_ == other; + return GetBoundCurrentXThread() == other; } XThread* XThread::GetCurrentThread() { - XThread* thread = reinterpret_cast(current_xthread_tls_); + XThread* thread = GetBoundCurrentXThread(); if (!thread) { assert_always("Attempting to use kernel stuff from a non-kernel thread"); } diff --git a/thirdparty/rexglue-sdk/src/ui/imgui_drawer.cpp b/thirdparty/rexglue-sdk/src/ui/imgui_drawer.cpp index 1036734b..d9364b55 100644 --- a/thirdparty/rexglue-sdk/src/ui/imgui_drawer.cpp +++ b/thirdparty/rexglue-sdk/src/ui/imgui_drawer.cpp @@ -34,8 +34,8 @@ const char kProggyTinyCompressedDataBase85[10950 + 1] = static_assert(sizeof(ImmediateVertex) == sizeof(ImDrawVert), "Vertex types must match"); -ImGuiDrawer::ImGuiDrawer(rex::ui::Window* window, size_t z_order) - : window_(window), z_order_(z_order) { +ImGuiDrawer::ImGuiDrawer(rex::ui::Window* window, size_t z_order, FontSetupCallback font_setup) + : window_(window), z_order_(z_order), font_setup_(std::move(font_setup)) { Initialize(); } @@ -134,6 +134,10 @@ void ImGuiDrawer::Initialize() { } #endif + if (font_setup_) { + font_setup_(io.Fonts); + } + auto& style = ImGui::GetStyle(); style.ScrollbarRounding = 0; style.WindowRounding = 0; diff --git a/thirdparty/rexglue-sdk/src/ui/overlay/settings_overlay.cpp b/thirdparty/rexglue-sdk/src/ui/overlay/settings_overlay.cpp index 98004a05..85b38218 100644 --- a/thirdparty/rexglue-sdk/src/ui/overlay/settings_overlay.cpp +++ b/thirdparty/rexglue-sdk/src/ui/overlay/settings_overlay.cpp @@ -13,7 +13,6 @@ #include #include #include -REXCVAR_DECLARE(bool, mnk_mode); #include #include @@ -333,7 +332,8 @@ void SettingsDialog::OnDraw(ImGuiIO& /*io*/) { if (is_keybind_category(entry.category)) { // Grey out controller keybinds when MnK mode is disabled - bool mnk_disabled = (entry.category == "Input/Keybinds/Controller" && !REXCVAR_GET(mnk_mode)); + bool mnk_disabled = + (entry.category == "Input/Keybinds/Controller" && !REXCVAR_QUERY(bool, mnk_mode)); if (mnk_disabled) ImGui::BeginDisabled(); diff --git a/thirdparty/rexglue-sdk/tests/unit/codegen/manifest_test.cpp b/thirdparty/rexglue-sdk/tests/unit/codegen/manifest_test.cpp new file mode 100644 index 00000000..0fd737a8 --- /dev/null +++ b/thirdparty/rexglue-sdk/tests/unit/codegen/manifest_test.cpp @@ -0,0 +1,231 @@ +/** + * @file tests/unit/codegen/manifest_test.cpp + * @brief Unit tests for manifest TOML parser + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include +#include + +#include +#include + +namespace fs = std::filesystem; + +namespace { + +struct TempDir { + fs::path path; + TempDir() : path(fs::temp_directory_path() / "manifest_test") { + fs::remove_all(path); + fs::create_directories(path); + } + ~TempDir() { fs::remove_all(path); } + void writeFile(const std::string& name, const std::string& content) const { + std::ofstream f(path / name); + f << content; + } +}; + +} // namespace + +TEST_CASE("Manifest: parse manifest with inline entrypoint", "[codegen][manifest]") { + TempDir tmp; + + tmp.writeFile("manifest.toml", R"( +[project] +name = "mygame" + +[entrypoint] +file_path = "assets/default.xex" +out_directory_path = "generated/default" +includes = [] + )"); + + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + REQUIRE(result.has_value()); + CHECK(result->projectName == "mygame"); + CHECK(result->entrypoint.recompiler.filePath == "assets/default.xex"); + CHECK(result->entrypoint.recompiler.outDirectoryPath == "generated/default"); + CHECK(result->entrypoint.guestPath.empty()); + CHECK(result->modules.empty()); +} + +TEST_CASE("Manifest: IsManifest detection", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", + "[project]\nname = \"test\"\n[entrypoint]\nfile_path = \"x.xex\"\n"); + tmp.writeFile("config.toml", "project_name = \"test\"\nfile_path = \"test.xex\""); + + CHECK(rex::codegen::ManifestConfig::IsManifest(tmp.path / "manifest.toml")); + CHECK_FALSE(rex::codegen::ManifestConfig::IsManifest(tmp.path / "config.toml")); +} + +TEST_CASE("Manifest: parse manifest with inline modules", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", R"( +[project] +name = "mygame" + +[entrypoint] +file_path = "assets/default.xex" +out_directory_path = "generated/default" + +[[modules]] +guest_path = "bin/lib_a.dll" +file_path = "assets/lib_a.dll" +out_directory_path = "generated/lib_a" + +[[modules]] +guest_path = "bin/lib_b.dll" +file_path = "assets/lib_b.dll" +out_directory_path = "generated/lib_b" + )"); + + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + REQUIRE(result.has_value()); + REQUIRE(result->modules.size() == 2u); + CHECK(result->modules[0].guestPath == "bin/lib_a.dll"); + CHECK(result->modules[0].recompiler.filePath == "assets/lib_a.dll"); + CHECK(result->modules[1].guestPath == "bin/lib_b.dll"); + CHECK(result->modules[1].recompiler.filePath == "assets/lib_b.dll"); +} + +TEST_CASE("Manifest: missing project section fails", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", "[entrypoint]\nfile_path = \"x.xex\"\n"); + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + CHECK_FALSE(result.has_value()); +} + +TEST_CASE("Manifest: missing entrypoint section fails", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", "[project]\nname = \"mygame\"\n"); + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + CHECK_FALSE(result.has_value()); +} + +TEST_CASE("Manifest: missing project name fails", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", + "[project]\n[entrypoint]\nfile_path = \"x.xex\"\nout_directory_path = \"o\"\n"); + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + CHECK_FALSE(result.has_value()); +} + +TEST_CASE("Manifest: parses sdk_version when present", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", R"( +[project] +name = "mygame" +sdk_version = "0.7.8.48" + +[entrypoint] +file_path = "assets/default.xex" +out_directory_path = "generated/default" + )"); + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + REQUIRE(result.has_value()); + REQUIRE(result->sdkVersion.has_value()); + CHECK(*result->sdkVersion == "0.7.8.48"); +} + +TEST_CASE("Manifest: sdk_version is nullopt when absent", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", R"( +[project] +name = "mygame" + +[entrypoint] +file_path = "assets/default.xex" +out_directory_path = "generated/default" + )"); + auto result = rex::codegen::ManifestConfig::Load(tmp.path / "manifest.toml"); + REQUIRE(result.has_value()); + CHECK_FALSE(result->sdkVersion.has_value()); +} + +TEST_CASE("Manifest: WriteSdkVersionStamp inserts when missing", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", R"([project] +name = "mygame" + +[entrypoint] +file_path = "assets/default.xex" +out_directory_path = "generated/default" +)"); + + auto path = tmp.path / "manifest.toml"; + CHECK(rex::codegen::ManifestConfig::WriteSdkVersionStamp(path, "0.8.0")); + + auto result = rex::codegen::ManifestConfig::Load(path); + REQUIRE(result.has_value()); + REQUIRE(result->sdkVersion.has_value()); + CHECK(*result->sdkVersion == "0.8.0"); + CHECK(result->projectName == "mygame"); +} + +TEST_CASE("Manifest: WriteSdkVersionStamp overwrites existing", "[codegen][manifest]") { + TempDir tmp; + tmp.writeFile("manifest.toml", R"([project] +name = "mygame" +sdk_version = "0.7.0" + +[entrypoint] +file_path = "assets/default.xex" +out_directory_path = "generated/default" +)"); + + auto path = tmp.path / "manifest.toml"; + CHECK(rex::codegen::ManifestConfig::WriteSdkVersionStamp(path, "0.8.0")); + + auto result = rex::codegen::ManifestConfig::Load(path); + REQUIRE(result.has_value()); + REQUIRE(result->sdkVersion.has_value()); + CHECK(*result->sdkVersion == "0.8.0"); +} + +TEST_CASE("CanonicalizeModuleGuestPath: device prefix", "[codegen][manifest][canonicalize]") { + using rex::codegen::CanonicalizeModuleGuestPath; + CHECK(CanonicalizeModuleGuestPath("game:\\bin\\foo.dll") == "bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("game:/bin/foo.dll") == "bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("d:\\bin\\foo.dll") == "bin/foo.dll"); +} + +TEST_CASE("CanonicalizeModuleGuestPath: case + slashes", "[codegen][canonicalize]") { + using rex::codegen::CanonicalizeModuleGuestPath; + CHECK(CanonicalizeModuleGuestPath("BIN\\Foo.DLL") == "bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("bin\\sub\\Foo.dll") == "bin/sub/foo.dll"); +} + +TEST_CASE("CanonicalizeModuleGuestPath: leading slashes", "[codegen][canonicalize]") { + using rex::codegen::CanonicalizeModuleGuestPath; + CHECK(CanonicalizeModuleGuestPath("/bin/foo.dll") == "bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("//bin/foo.dll") == "bin/foo.dll"); +} + +TEST_CASE("CanonicalizeModuleGuestPath: bare assets prefix preserved without project", + "[codegen][canonicalize]") { + using rex::codegen::CanonicalizeModuleGuestPath; + CHECK(CanonicalizeModuleGuestPath("assets/bin/foo.dll") == "assets/bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("Assets/Bin/Foo.DLL") == "assets/bin/foo.dll"); +} + +TEST_CASE("CanonicalizeModuleGuestPath: project assets prefix stripped", + "[codegen][canonicalize]") { + using rex::codegen::CanonicalizeModuleGuestPath; + CHECK(CanonicalizeModuleGuestPath("mygame/assets/bin/foo.dll", "mygame") == "bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("MyGame/Assets/Bin/Foo.DLL", "mygame") == "bin/foo.dll"); +} + +TEST_CASE("CanonicalizeModuleGuestPath: project mismatch keeps prefix", "[codegen][canonicalize]") { + using rex::codegen::CanonicalizeModuleGuestPath; + CHECK(CanonicalizeModuleGuestPath("othergame/assets/bin/foo.dll", "mygame") == + "othergame/assets/bin/foo.dll"); + CHECK(CanonicalizeModuleGuestPath("assets/foo.dll", "mygame") == "assets/foo.dll"); +} diff --git a/thirdparty/rexglue-sdk/tests/unit/rexglue/migration_scan_test.cpp b/thirdparty/rexglue-sdk/tests/unit/rexglue/migration_scan_test.cpp new file mode 100644 index 00000000..fae15266 --- /dev/null +++ b/thirdparty/rexglue-sdk/tests/unit/rexglue/migration_scan_test.cpp @@ -0,0 +1,516 @@ +/** + * @file tests/unit/rexglue/migration_scan_test.cpp + * @brief Tests for project-tree migration scanners + * + * @copyright Copyright (c) 2026 Tom Clay + * @license BSD 3-Clause License + */ + +#include "rexglue/commands/migration_scan.h" + +#include + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +struct TempProject { + fs::path root; + explicit TempProject(const std::string& tag = "migration_scan_test") + : root(fs::temp_directory_path() / tag) { + fs::remove_all(root); + fs::create_directories(root / "generated"); + } + ~TempProject() { fs::remove_all(root); } + + void writeRexglueCmake(const std::string& content) const { + std::ofstream f(root / "generated" / "rexglue.cmake"); + f << content; + } + + void writeFile(const fs::path& rel, const std::string& content) const { + fs::create_directories((root / rel).parent_path()); + std::ofstream f(root / rel); + f << content; + } +}; + +} // namespace + +// --------------------------------------------------------------------------- +// SDK template drift +// --------------------------------------------------------------------------- + +TEST_CASE("MigrationScan: empty plan when rexglue.cmake matches template", + "[rexglue][migration_scan]") { + TempProject tp; + std::string rendered = rexglue::cli::RenderRexglueCmake("mygame", "0.8.0", "generated/default"); + tp.writeRexglueCmake(rendered); + + auto plan = rexglue::cli::ScanSdkTemplateDrift(tp.root, "mygame", "0.8.0", "generated/default"); + CHECK(plan.empty()); +} + +TEST_CASE("MigrationScan: drift entry is silent (lives inside generated/)", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeRexglueCmake("# obviously stale content\n"); + + auto plan = rexglue::cli::ScanSdkTemplateDrift(tp.root, "mygame", "0.8.0", "generated/default"); + REQUIRE(plan.size() == 1u); + CHECK(plan[0].path.filename() == "rexglue.cmake"); + CHECK(plan[0].silent); + CHECK(plan[0].action == rexglue::cli::OverwriteAction::Write); + CHECK(plan[0].rendered_content.find("rex::runtime") != std::string::npos); + CHECK(plan[0].rendered_content.find("generated/default/sources.cmake") != std::string::npos); +} + +TEST_CASE("MigrationScan: drift entry is silent when the file is missing", + "[rexglue][migration_scan]") { + TempProject tp; // generated/rexglue.cmake never written + + auto plan = rexglue::cli::ScanSdkTemplateDrift(tp.root, "mygame", "0.8.0", "generated/default"); + REQUIRE(plan.size() == 1u); + CHECK(plan[0].silent); + CHECK(plan[0].path == tp.root / "generated" / "rexglue.cmake"); +} + +// --------------------------------------------------------------------------- +// CMake reference rewrites +// --------------------------------------------------------------------------- + +TEST_CASE("MigrationScan: ScanCmakeReferences rewrites legacy config refs", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("CMakeLists.txt", + "add_custom_target(mygame_codegen\n" + " COMMAND rexglue codegen ${CMAKE_CURRENT_SOURCE_DIR}/mygame_config.toml\n" + ")\n"); + tp.writeFile("cmake/extra.cmake", "# nothing to do here\n"); + + auto entries = + rexglue::cli::ScanCmakeReferences(tp.root, "mygame_config.toml", "mygame_manifest.toml"); + REQUIRE(entries.size() == 1u); + CHECK(entries[0].path == tp.root / "CMakeLists.txt"); + CHECK_FALSE(entries[0].silent); + CHECK(entries[0].rendered_content.find("mygame_manifest.toml") != std::string::npos); + CHECK(entries[0].rendered_content.find("mygame_config.toml") == std::string::npos); +} + +TEST_CASE("MigrationScan: ScanCmakeReferences leaves embedded substrings alone", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("CMakeLists.txt", + "# legacy file: mygame_config.toml.bak retained for reference\n" + "add_custom_target(mygame_codegen\n" + " COMMAND rexglue codegen mygame_config.toml\n" + ")\n"); + auto entries = + rexglue::cli::ScanCmakeReferences(tp.root, "mygame_config.toml", "mygame_manifest.toml"); + REQUIRE(entries.size() == 1u); + CHECK(entries[0].rendered_content.find("mygame_config.toml.bak") != std::string::npos); + CHECK(entries[0].rendered_content.find("rexglue codegen mygame_manifest.toml") != + std::string::npos); +} + +TEST_CASE("MigrationScan: ScanCmakeReferences skips files inside generated/", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("generated/rexglue.cmake", + "add_custom_target(mygame_codegen\n" + " COMMAND rexglue codegen mygame_config.toml\n" + ")\n"); + + auto entries = + rexglue::cli::ScanCmakeReferences(tp.root, "mygame_config.toml", "mygame_manifest.toml"); + CHECK(entries.empty()); +} + +TEST_CASE("MigrationScan: ScanCmakeReferences ignores irrelevant files", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("README.md", "mygame_config.toml is the legacy config\n"); + tp.writeFile("src/main.cpp", "// references mygame_config.toml in a comment\n"); + + auto entries = + rexglue::cli::ScanCmakeReferences(tp.root, "mygame_config.toml", "mygame_manifest.toml"); + CHECK(entries.empty()); +} + +// --------------------------------------------------------------------------- +// Source #include rewrites +// --------------------------------------------------------------------------- + +TEST_CASE("MigrationScan: ScanSourceIncludeRewrites renames _config.h to _init.h", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("src/main.cpp", + "#include \"generated/mygame_config.h\"\n" + "int main() { return 0; }\n"); + + auto entries = rexglue::cli::ScanSourceIncludeRewrites(tp.root, "mygame"); + REQUIRE(entries.size() == 1u); + CHECK(entries[0].path == tp.root / "src" / "main.cpp"); + CHECK_FALSE(entries[0].silent); + CHECK(entries[0].action == rexglue::cli::OverwriteAction::Write); + CHECK(entries[0].rendered_content.find("generated/mygame_init.h") != std::string::npos); + CHECK(entries[0].rendered_content.find("mygame_config.h") == std::string::npos); + CHECK(entries[0].rendered_content.find("int main()") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanSourceIncludeRewrites drops duplicate when _init.h already present", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("src/main.cpp", + "#include \"generated/mygame_config.h\"\n" + "#include \"generated/mygame_init.h\"\n" + "int main() { return 0; }\n"); + + auto entries = rexglue::cli::ScanSourceIncludeRewrites(tp.root, "mygame"); + REQUIRE(entries.size() == 1u); + CHECK(entries[0].rendered_content.find("mygame_config.h") == std::string::npos); + std::size_t init_count = 0; + for (std::size_t pos = 0; + (pos = entries[0].rendered_content.find("mygame_init.h", pos)) != std::string::npos; + pos += 1) { + ++init_count; + } + CHECK(init_count == 1u); + CHECK(entries[0].rendered_content.find("int main()") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanSourceIncludeRewrites no-op when no _config.h reference", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("src/main.cpp", + "#include \"generated/mygame_init.h\"\n" + "int main() { return 0; }\n"); + + auto entries = rexglue::cli::ScanSourceIncludeRewrites(tp.root, "mygame"); + CHECK(entries.empty()); +} + +TEST_CASE("MigrationScan: ScanSourceIncludeRewrites skips files inside generated/", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("generated/mygame_register.cpp", "#include \"mygame_config.h\"\n"); + + auto entries = rexglue::cli::ScanSourceIncludeRewrites(tp.root, "mygame"); + CHECK(entries.empty()); +} + +TEST_CASE("MigrationScan: ScanSourceIncludeRewrites only matches the project's own header", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("src/main.cpp", "#include \"otherproj_config.h\"\n"); + + auto entries = rexglue::cli::ScanSourceIncludeRewrites(tp.root, "mygame"); + CHECK(entries.empty()); +} + +TEST_CASE("MigrationScan: ScanSourceIncludeRewrites is case-insensitive on basename", + "[rexglue][migration_scan]") { + TempProject tp; + tp.writeFile("src/main.cpp", "#include \"MyGame_Config.H\"\n"); + + auto entries = rexglue::cli::ScanSourceIncludeRewrites(tp.root, "mygame"); + REQUIRE(entries.size() == 1u); + CHECK(entries[0].rendered_content.find("mygame_init.h") != std::string::npos); +} + +// --------------------------------------------------------------------------- +// Stale include warnings +// --------------------------------------------------------------------------- + +TEST_CASE("MigrationScan: ScanStaleIncludes matches quoted #include with stale basename", + "[rexglue][migration_scan]") { + TempProject tp("migration_stale_includes"); + tp.writeFile("main.cpp", + R"(#include "generated/foo_config.h" +int main() { return 0; } +)"); + std::unordered_set removed{"foo_config.h"}; + + auto results = rexglue::cli::ScanStaleIncludes(tp.root, removed); + REQUIRE(results.size() == 1u); + CHECK(results[0].file == tp.root / "main.cpp"); + CHECK(results[0].line_number == 1u); + CHECK(results[0].detail.find("generated/foo_config.h") != std::string::npos); + CHECK(results[0].hint.find("no longer emitted") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanStaleIncludes ignores unrelated #include directives", + "[rexglue][migration_scan]") { + TempProject tp("migration_stale_includes_unrelated"); + tp.writeFile("a.cpp", + R"(#include +#include "bar.h" +)"); + std::unordered_set removed{"foo_config.h"}; + + auto results = rexglue::cli::ScanStaleIncludes(tp.root, removed); + CHECK(results.empty()); +} + +TEST_CASE("MigrationScan: ScanStaleIncludes walks subdirectories and supports common extensions", + "[rexglue][migration_scan]") { + TempProject tp("migration_stale_includes_walk"); + tp.writeFile("a.cpp", "#include \"foo_config.h\"\n"); + tp.writeFile("sub/b.h", "#include \"foo_config.h\"\n"); + tp.writeFile("sub/c.hpp", "#include \n"); + tp.writeFile("sub/d.inl", "#include \"foo_config.h\"\n"); + tp.writeFile("sub/e.txt", "#include \"foo_config.h\"\n"); + std::unordered_set removed{"foo_config.h"}; + + auto results = rexglue::cli::ScanStaleIncludes(tp.root, removed); + CHECK(results.size() == 4u); +} + +TEST_CASE("MigrationScan: ScanStaleIncludes returns empty when src dir does not exist", + "[rexglue][migration_scan]") { + fs::path nonexistent = fs::temp_directory_path() / "migration_stale_includes_missing"; + fs::remove_all(nonexistent); + std::unordered_set removed{"foo.h"}; + auto results = rexglue::cli::ScanStaleIncludes(nonexistent, removed); + CHECK(results.empty()); +} + +TEST_CASE("MigrationScan: ScanStaleIncludes matches case-insensitively on basename", + "[rexglue][migration_scan]") { + TempProject tp("migration_stale_includes_case"); + tp.writeFile("a.cpp", "#include \"Foo_Config.H\"\n"); + std::unordered_set removed{"foo_config.h"}; + + auto results = rexglue::cli::ScanStaleIncludes(tp.root, removed); + REQUIRE(results.size() == 1u); + CHECK(results[0].detail.find("Foo_Config.H") != std::string::npos); +} + +// --------------------------------------------------------------------------- +// Legacy identifier scanner +// --------------------------------------------------------------------------- + +TEST_CASE("MigrationScan: ScanLegacyIdentifiers rewrites whole-token PPC_FUNC to REX_FUNC", + "[rexglue][migration_scan]") { + TempProject tp("migration_legacy_idents"); + tp.writeFile("src/foo.cpp", + "#include \n" + "PPC_FUNC(sub_1234) {\n" + " PPC_LOAD_U32(ctx.r3.u32);\n" + "}\n"); + + auto findings = rexglue::cli::ScanLegacyIdentifiers(tp.root); + REQUIRE(findings.rewrites.size() == 1u); + CHECK(findings.warnings.empty()); + const auto& entry = findings.rewrites[0]; + CHECK(entry.path == tp.root / "src" / "foo.cpp"); + CHECK_FALSE(entry.silent); + CHECK(entry.rendered_content.find("REX_FUNC(sub_1234)") != std::string::npos); + CHECK(entry.rendered_content.find("REX_LOAD_U32") != std::string::npos); + CHECK(entry.rendered_content.find("PPC_FUNC") == std::string::npos); + CHECK(entry.rendered_content.find("PPC_LOAD_U32") == std::string::npos); +} + +TEST_CASE("MigrationScan: ScanLegacyIdentifiers leaves non-matching prefixes alone", + "[rexglue][migration_scan]") { + TempProject tp("migration_legacy_idents_prefix"); + // Real binutils tokens that happen to start with PPC_ but are NOT in the + // breaking-change rule list - they must not be rewritten. + tp.writeFile("src/disasm.cpp", + "#include \n" + "uint32_t op = PPC_OP(insn);\n" + "if (op == PPC_INST_BL) { /* ... */ }\n"); + + auto findings = rexglue::cli::ScanLegacyIdentifiers(tp.root); + CHECK(findings.rewrites.empty()); + CHECK(findings.warnings.empty()); +} + +TEST_CASE("MigrationScan: ScanLegacyIdentifiers warns on tokens with no replacement", + "[rexglue][migration_scan]") { + TempProject tp("migration_legacy_idents_warn"); + tp.writeFile("src/decls.h", "PPC_EXTERN_FUNC(some_helper);\n"); + + auto findings = rexglue::cli::ScanLegacyIdentifiers(tp.root); + CHECK(findings.rewrites.empty()); + REQUIRE(findings.warnings.size() == 1u); + CHECK(findings.warnings[0].file == tp.root / "src" / "decls.h"); + CHECK(findings.warnings[0].line_number == 1u); + CHECK(findings.warnings[0].detail.find("PPC_EXTERN_FUNC") != std::string::npos); + CHECK(findings.warnings[0].hint.find("extern REX_FUNC") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanLegacyIdentifiers respects identifier boundaries", + "[rexglue][migration_scan]") { + TempProject tp("migration_legacy_idents_boundary"); + // PPC_FUNC_PROLOGUE is a known token; PPC_FUNC_THINGAMAJIG is not. The token + // matcher must prefer the longest run of identifier characters and only + // rewrite when the full identifier is in the rule table. + tp.writeFile("src/bar.cpp", + "PPC_FUNC_PROLOGUE();\n" + "PPC_FUNC_THINGAMAJIG();\n"); + + auto findings = rexglue::cli::ScanLegacyIdentifiers(tp.root); + REQUIRE(findings.rewrites.size() == 1u); + CHECK(findings.rewrites[0].rendered_content.find("REX_FUNC_PROLOGUE();") != std::string::npos); + CHECK(findings.rewrites[0].rendered_content.find("PPC_FUNC_THINGAMAJIG();") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanLegacyIdentifiers skips files inside generated/", + "[rexglue][migration_scan]") { + TempProject tp("migration_legacy_idents_gen"); + tp.writeFile("generated/foo.cpp", "PPC_FUNC(sub_1) {}\n"); + + auto findings = rexglue::cli::ScanLegacyIdentifiers(tp.root); + CHECK(findings.rewrites.empty()); + CHECK(findings.warnings.empty()); +} + +TEST_CASE("MigrationScan: ScanLegacyIdentifiers accepts a custom rule list", + "[rexglue][migration_scan]") { + TempProject tp("migration_legacy_idents_custom"); + tp.writeFile("src/custom.cpp", "OLD_THING x = NEW_THING;\nUSE_ONCE();\n"); + + std::array rules = {{ + {"OLD_THING", "NEW_THING_V2", "renamed: OLD_THING -> NEW_THING_V2"}, + {"USE_ONCE", "", "removed; manual fix required"}, + }}; + auto findings = rexglue::cli::ScanLegacyIdentifiers(tp.root, rules); + REQUIRE(findings.rewrites.size() == 1u); + CHECK(findings.rewrites[0].rendered_content.find("NEW_THING_V2 x = NEW_THING;") != + std::string::npos); + REQUIRE(findings.warnings.size() == 1u); + CHECK(findings.warnings[0].detail.find("USE_ONCE") != std::string::npos); +} + +// --------------------------------------------------------------------------- +// Call-site pattern scanner +// --------------------------------------------------------------------------- + +TEST_CASE("MigrationScan: ScanCallSitePatterns flags removed game_directory positional", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_gamedir"); + tp.writeFile("src/app.cpp", + "#include \n" + "void f() {\n" + " if (GetArgument(\"game_directory\").has_value()) {}\n" + "}\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + REQUIRE(warnings.size() == 1u); + CHECK(warnings[0].file == tp.root / "src" / "app.cpp"); + CHECK(warnings[0].line_number == 3u); + CHECK(warnings[0].detail.find("game_directory") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns leaves unrelated GetArgument calls alone", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_gamedir_other"); + tp.writeFile("src/app.cpp", " GetArgument(\"verbose\");\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + CHECK(warnings.empty()); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns flags one-arg AllocateThunk", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_thunk_one"); + tp.writeFile("src/d.cpp", + "auto* d = dispatcher();\n" + "uint32_t a = d->AllocateThunk(&Helper);\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + REQUIRE(warnings.size() == 1u); + CHECK(warnings[0].line_number == 2u); + CHECK(warnings[0].detail.find("AllocateThunk") != std::string::npos); + CHECK(warnings[0].hint.find("caller_address") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns ignores two-arg AllocateThunk", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_thunk_two"); + tp.writeFile("src/d.cpp", "uint32_t a = d->AllocateThunk(s_trampolines[i], ctx.lr);\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + CHECK(warnings.empty()); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns handles single arg with nested parens", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_thunk_nested"); + tp.writeFile("src/d.cpp", "auto a = d->AllocateThunk(MakeHelper(env));\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + REQUIRE(warnings.size() == 1u); + CHECK(warnings[0].detail.find("AllocateThunk") != std::string::npos); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns ignores zero-arg AllocateThunk", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_thunk_zero"); + tp.writeFile("src/d.cpp", "auto a = d->AllocateThunk();\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + CHECK(warnings.empty()); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns skips files inside generated/", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_gen"); + tp.writeFile("generated/foo.cpp", " d->AllocateThunk(&Fn);\n"); + + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root); + CHECK(warnings.empty()); +} + +TEST_CASE("MigrationScan: ScanCallSitePatterns accepts a custom rule list", + "[rexglue][migration_scan]") { + TempProject tp("migration_callsite_custom"); + tp.writeFile("src/x.cpp", " legacy_api(42);\n"); + + std::array rules = {{ + {"legacy_api(", "uses removed legacy_api()", "switch to modern_api() (drop-in)", nullptr}, + }}; + auto warnings = rexglue::cli::ScanCallSitePatterns(tp.root, rules); + REQUIRE(warnings.size() == 1u); + CHECK(warnings[0].detail.find("legacy_api") != std::string::npos); + CHECK(warnings[0].hint.find("modern_api") != std::string::npos); +} + +TEST_CASE("MigrationScan: DefaultCallSiteRules covers known manual fixes", + "[rexglue][migration_scan]") { + auto rules = rexglue::cli::DefaultCallSiteRules(); + REQUIRE(!rules.empty()); + bool saw_gamedir = false; + bool saw_thunk = false; + for (const auto& r : rules) { + if (r.pattern.find("game_directory") != std::string_view::npos) + saw_gamedir = true; + if (r.pattern == "AllocateThunk(") + saw_thunk = true; + } + CHECK(saw_gamedir); + CHECK(saw_thunk); +} + +TEST_CASE("MigrationScan: DefaultBreakingChangeRules covers PPC_ legacy macros", + "[rexglue][migration_scan]") { + auto rules = rexglue::cli::DefaultBreakingChangeRules(); + REQUIRE(!rules.empty()); + bool saw_ppc_func = false; + bool saw_ppc_round_nearest = false; + for (const auto& r : rules) { + if (r.legacy_token == "PPC_FUNC") + saw_ppc_func = (r.replacement == "REX_FUNC"); + if (r.legacy_token == "PPC_ROUND_NEAREST") + saw_ppc_round_nearest = (r.replacement == "rex::ppc::kRoundNearest"); + } + CHECK(saw_ppc_func); + CHECK(saw_ppc_round_nearest); +} diff --git a/thirdparty/rexglue-sdk/tests/unit/rexglue/ui_test.cpp b/thirdparty/rexglue-sdk/tests/unit/rexglue/ui_test.cpp new file mode 100644 index 00000000..7346dbc0 --- /dev/null +++ b/thirdparty/rexglue-sdk/tests/unit/rexglue/ui_test.cpp @@ -0,0 +1,175 @@ +/** + * @file tests/unit/rexglue/ui_test.cpp + * @brief Unit tests for the rexglue presentation layer + * + * @copyright Copyright (c) 2026 Tom Clay + * @license BSD 3-Clause License + */ + +#include "rexglue/ui/progress.h" +#include "rexglue/ui/ui.h" + +#include +#include +#include +#include +#include + +#include +#include + +namespace { + +// Test helper: build a logger that writes through the sink under test. +struct LoggerHarness { + std::ostringstream out; + std::shared_ptr sink; + std::shared_ptr logger; + + explicit LoggerHarness(bool tty = false, bool color = false) + : sink(std::make_shared(out, tty, color)), + logger(std::make_shared("test", sink)) { + logger->set_level(spdlog::level::trace); + } +}; + +// For block-writer tests we need ui:: free functions to find a global sink. +struct UiBlockHarness { + std::ostringstream out; + UiBlockHarness() { + rexglue::ui::Shutdown(); + rexglue::ui::detail::SetGlobalSinkForTesting( + std::make_unique(out, /*tty=*/false, + /*color=*/false)); + } + ~UiBlockHarness() { rexglue::ui::Shutdown(); } +}; + +} // namespace + +TEST_CASE("PresentationSink routes spdlog messages to the injected stream", "[ui][sink]") { + LoggerHarness h; + h.logger->info("hello world"); + h.logger->flush(); + REQUIRE(h.out.str().find("hello world") != std::string::npos); +} + +TEST_CASE("ui::Banner writes title followed by blank line", "[ui][block]") { + UiBlockHarness h; + rexglue::ui::Banner("ReXGlue 0.8.0.26"); + REQUIRE(h.out.str() == "ReXGlue 0.8.0.26\n\n"); +} + +TEST_CASE("ui::KeyValueBlock aligns keys and emits header", "[ui][block]") { + UiBlockHarness h; + std::array rows = {{ + {"Manifest", "/abs/manifest.toml"}, + {"Project", "reblue (entrypoint + 2 DLLs)"}, + }}; + rexglue::ui::KeyValueBlock("", rows); + auto s = h.out.str(); + REQUIRE(s.find(" Manifest: /abs/manifest.toml\n") != std::string::npos); + REQUIRE(s.find(" Project: reblue (entrypoint + 2 DLLs)\n") != std::string::npos); +} + +TEST_CASE("ui::PlanTable formats rows with action label and reason", "[ui][block]") { + UiBlockHarness h; + std::array rows = {{ + {"write ", "/a/b.toml", "upgrade format"}, + {"delete", "/a/c.toml", "absorbed"}, + }}; + rexglue::ui::PlanTable("Migration: 2 file(s) will be rewritten.", rows); + auto s = h.out.str(); + REQUIRE(s.find("Migration: 2 file(s) will be rewritten.\n") != std::string::npos); + REQUIRE(s.find(" [write ] /a/b.toml") != std::string::npos); + REQUIRE(s.find("upgrade format") != std::string::npos); + REQUIRE(s.find(" [delete] /a/c.toml") != std::string::npos); + REQUIRE(s.find("absorbed") != std::string::npos); +} + +TEST_CASE("ui::ManualReviewList nests detail and optional hint", "[ui][block]") { + UiBlockHarness h; + std::array rows = {{ + {"f.cpp:329", "single-arg call", "pass ctx.lr instead"}, + {"g.cpp:43", "removed positional argument", ""}, + }}; + rexglue::ui::ManualReviewList("Migration: 2 site(s) need manual review:", rows); + auto s = h.out.str(); + REQUIRE(s.find("Migration: 2 site(s) need manual review:\n") != std::string::npos); + REQUIRE(s.find(" f.cpp:329 single-arg call\n") != std::string::npos); + REQUIRE(s.find(" pass ctx.lr instead\n") != std::string::npos); + REQUIRE(s.find(" g.cpp:43 removed positional argument\n") != std::string::npos); + REQUIRE(s.find(" \n") == std::string::npos); +} + +TEST_CASE("ui::Confirm returns false on non-TTY without consuming input", "[ui][confirm]") { + UiBlockHarness h; + std::istringstream input("y\n"); + bool result = rexglue::ui::detail::ConfirmWithStream("Apply?", input, /*tty=*/false); + REQUIRE(result == false); + REQUIRE(input.tellg() == 0); + REQUIRE(h.out.str().find("Apply? [y/N]: ") != std::string::npos); +} + +TEST_CASE("ui::Confirm returns true for 'y' on TTY", "[ui][confirm]") { + UiBlockHarness h; + std::istringstream input("y\n"); + REQUIRE(rexglue::ui::detail::ConfirmWithStream("Apply?", input, + /*tty=*/true) == true); +} + +TEST_CASE("ui::Confirm returns true for 'YES' (case-insensitive)", "[ui][confirm]") { + UiBlockHarness h; + std::istringstream input("YES\n"); + REQUIRE(rexglue::ui::detail::ConfirmWithStream("Apply?", input, + /*tty=*/true) == true); +} + +TEST_CASE("ui::Confirm returns false for 'n', empty, and EOF", "[ui][confirm]") { + for (const std::string& answer : {std::string{"n\n"}, std::string{"\n"}, std::string{}}) { + UiBlockHarness h; + std::istringstream input(answer); + REQUIRE(rexglue::ui::detail::ConfirmWithStream("Apply?", input, + /*tty=*/true) == false); + } +} + +TEST_CASE("ProgressView non-TTY emits one line per event in order", "[ui][progress]") { + UiBlockHarness h; + { + rexglue::ui::ProgressView pv("Recompiling reblue (entrypoint + 1 DLL)"); + pv.moduleStarted("reblue", 0, 2); + pv.phaseChanged("Register"); + pv.phaseChanged("Discover"); + pv.moduleFinished(std::chrono::milliseconds{1234}); + pv.moduleStarted("bdengine", 1, 2); + pv.phaseChanged("Register"); + pv.moduleFinished(std::chrono::milliseconds{2345}); + } + auto s = h.out.str(); + REQUIRE(s.find("Recompiling reblue (entrypoint + 1 DLL)\n") == 0); + auto pos1 = s.find(" start reblue"); + auto pos2 = s.find(" phase reblue: Register"); + auto pos3 = s.find(" phase reblue: Discover"); + auto pos4 = s.find(" done reblue (1.2s)"); + auto pos5 = s.find(" start bdengine"); + auto pos6 = s.find(" done bdengine (2.3s)"); + REQUIRE(pos1 != std::string::npos); + REQUIRE(pos1 < pos2); + REQUIRE(pos2 < pos3); + REQUIRE(pos3 < pos4); + REQUIRE(pos4 < pos5); + REQUIRE(pos5 < pos6); +} + +TEST_CASE("ui::DoneSummary writes single Done line", "[ui][summary]") { + UiBlockHarness h; + rexglue::ui::DoneSummary(std::chrono::milliseconds{8400}); + REQUIRE(h.out.str() == "Done in 8.4s.\n"); +} + +TEST_CASE("ui::FailureSummary writes Failed line with reason and duration", "[ui][summary]") { + UiBlockHarness h; + rexglue::ui::FailureSummary("manifest not found", std::chrono::milliseconds{2500}); + REQUIRE(h.out.str() == "Failed: manifest not found (after 2.5s)\n"); +} diff --git a/thirdparty/rexglue-sdk/tests/unit/system/function_dispatcher_test.cpp b/thirdparty/rexglue-sdk/tests/unit/system/function_dispatcher_test.cpp new file mode 100644 index 00000000..967cccff --- /dev/null +++ b/thirdparty/rexglue-sdk/tests/unit/system/function_dispatcher_test.cpp @@ -0,0 +1,165 @@ +/** + * @file tests/unit/system/function_dispatcher_test.cpp + * @brief Unit tests for caller-aware thunk allocation and unregister cleanup + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include + +#include +#include +#include +#include +#include + +namespace { + +rex::memory::Memory& GetTestMemory() { + static rex::memory::Memory memory; + static bool initialized = false; + if (!initialized) { + rex::InitLogging(); + REQUIRE(memory.Initialize()); + initialized = true; + } + return memory; +} + +void DummyFn(PPCContext&, uint8_t*) {} + +} // namespace + +TEST_CASE("FunctionDispatcher: caller_address routes thunk to caller's module pool", + "[runtime][dispatcher]") { + auto& memory = GetTestMemory(); + rex::runtime::ExportResolver resolver; + rex::runtime::FunctionDispatcher dispatcher(&memory, &resolver); + + constexpr uint32_t kModA = 0x82000000u; + constexpr uint32_t kCodeSize = 0x10000u; + constexpr uint32_t kImageSize = 0x100000u; + + REQUIRE(dispatcher.InitializeFunctionTable(kModA, kCodeSize, kModA, kImageSize)); + + constexpr uint32_t kModB = 0x83000000u; + REQUIRE(dispatcher.InitializeFunctionTable(kModB, kCodeSize, kModB, kImageSize)); + + uint32_t thunk_a = dispatcher.AllocateThunk(&DummyFn, kModA + 0x100); + uint32_t thunk_b = dispatcher.AllocateThunk(&DummyFn, kModB + 0x100); + + CHECK(thunk_a >= kModA + kCodeSize); + CHECK(thunk_a < kModA + kCodeSize + rex::runtime::FunctionDispatcher::kThunkReserveSize); + CHECK(thunk_b >= kModB + kCodeSize); + CHECK(thunk_b < kModB + kCodeSize + rex::runtime::FunctionDispatcher::kThunkReserveSize); + + CHECK(dispatcher.GetFunction(thunk_a) == &DummyFn); + CHECK(dispatcher.GetFunction(thunk_b) == &DummyFn); +} + +TEST_CASE("FunctionDispatcher: AllocateThunk(0) uses the entrypoint pool only when explicit", + "[runtime][dispatcher]") { + // caller_address=0 is reserved for host-initiated allocations that have no + // guest caller (the entrypoint wiring its own __imp__* exports during + // setup). It must land in the entrypoint module's pool. + auto& memory = GetTestMemory(); + rex::runtime::ExportResolver resolver; + rex::runtime::FunctionDispatcher dispatcher(&memory, &resolver); + + constexpr uint32_t kModA = 0x84000000u; + constexpr uint32_t kCodeSize = 0x10000u; + constexpr uint32_t kImageSize = 0x100000u; + + REQUIRE(dispatcher.InitializeFunctionTable(kModA, kCodeSize, kModA, kImageSize, + /*is_entrypoint=*/true)); + + uint32_t thunk = dispatcher.AllocateThunk(&DummyFn, 0); + CHECK(thunk >= kModA + kCodeSize); + CHECK(thunk < kModA + kCodeSize + rex::runtime::FunctionDispatcher::kThunkReserveSize); +} + +TEST_CASE("FunctionDispatcher: AllocateThunk(0) rejects when no entrypoint registered", + "[runtime][dispatcher]") { + auto& memory = GetTestMemory(); + rex::runtime::ExportResolver resolver; + rex::runtime::FunctionDispatcher dispatcher(&memory, &resolver); + + constexpr uint32_t kModA = 0x88000000u; + constexpr uint32_t kCodeSize = 0x10000u; + constexpr uint32_t kImageSize = 0x100000u; + + REQUIRE(dispatcher.InitializeFunctionTable(kModA, kCodeSize, kModA, kImageSize)); + + CHECK(dispatcher.AllocateThunk(&DummyFn, 0) == 0); +} + +TEST_CASE("FunctionDispatcher: AllocateThunk rejects caller_address outside any module", + "[runtime][dispatcher]") { + auto& memory = GetTestMemory(); + rex::runtime::ExportResolver resolver; + rex::runtime::FunctionDispatcher dispatcher(&memory, &resolver); + + constexpr uint32_t kModA = 0x86000000u; + constexpr uint32_t kCodeSize = 0x10000u; + constexpr uint32_t kImageSize = 0x100000u; + + REQUIRE(dispatcher.InitializeFunctionTable(kModA, kCodeSize, kModA, kImageSize)); + + // A non-zero caller_address that doesn't fall inside any module is a bug + // in the caller: the right answer is to refuse, not to silently route the + // thunk to the entrypoint pool. + uint32_t thunk = dispatcher.AllocateThunk(&DummyFn, 0xDEADBEEFu); + CHECK(thunk == 0); +} + +namespace { +constexpr uint32_t kRegisterModBase = 0x85000000u; +void RegisterOne(rex::runtime::IModuleRegistrar* registrar) { + registrar->SetFunction(kRegisterModBase + 0x10, &DummyFn); +} +} // namespace + +TEST_CASE("FunctionDispatcher: UnregisterModule clears pool and slots", "[runtime][dispatcher]") { + auto& memory = GetTestMemory(); + rex::runtime::ExportResolver resolver; + rex::runtime::FunctionDispatcher dispatcher(&memory, &resolver); + + constexpr uint32_t kCodeSize = 0x10000u; + constexpr uint32_t kImageSize = 0x100000u; + + REQUIRE(dispatcher.InitializeFunctionTable(kRegisterModBase, kCodeSize, kRegisterModBase, + kImageSize)); + + dispatcher.RegisterModule("modA", kRegisterModBase, &RegisterOne); + + uint32_t thunk = dispatcher.AllocateThunk(&DummyFn, kRegisterModBase + 0x100); + REQUIRE(thunk != 0); + REQUIRE(dispatcher.GetFunction(kRegisterModBase + 0x10) == &DummyFn); + REQUIRE(dispatcher.GetFunction(thunk) == &DummyFn); + + auto cleared = dispatcher.UnregisterModule("modA"); + REQUIRE(cleared.has_value()); + CHECK(cleared->first == kRegisterModBase + kCodeSize); + CHECK(cleared->second == thunk + 4); + CHECK(dispatcher.GetFunction(kRegisterModBase + 0x10) == nullptr); + CHECK(dispatcher.GetFunction(thunk) == nullptr); + + // Re-init must succeed: UnregisterModule destroys the per-module table so the + // same code range can be reloaded without tripping the overlap check. + REQUIRE(dispatcher.InitializeFunctionTable(kRegisterModBase, kCodeSize, kRegisterModBase, + kImageSize)); + uint32_t thunk_after = dispatcher.AllocateThunk(&DummyFn, kRegisterModBase + 0x100); + CHECK(thunk_after == thunk); +} + +TEST_CASE("FunctionDispatcher: UnregisterModule on unknown id returns nullopt", + "[runtime][dispatcher]") { + auto& memory = GetTestMemory(); + rex::runtime::ExportResolver resolver; + rex::runtime::FunctionDispatcher dispatcher(&memory, &resolver); + CHECK_FALSE(dispatcher.UnregisterModule("nope").has_value()); +} diff --git a/thirdparty/rexglue-sdk/tests/unit/system/guest_path_test.cpp b/thirdparty/rexglue-sdk/tests/unit/system/guest_path_test.cpp new file mode 100644 index 00000000..cd6d9e2c --- /dev/null +++ b/thirdparty/rexglue-sdk/tests/unit/system/guest_path_test.cpp @@ -0,0 +1,33 @@ +/** + * @file tests/unit/system/guest_path_test.cpp + * @brief Unit tests for guest path normalization + * + * @copyright Copyright (c) 2026 Tom Clay + * All rights reserved. + * + * @license BSD 3-Clause License + * See LICENSE file in the project root for full license text. + */ + +#include +#include + +using rex::system::NormalizeGuestPath; + +TEST_CASE("Guest path normalization: strip device prefix", "[system][path]") { + CHECK(NormalizeGuestPath("game:\\bin\\somelib.dll") == "bin/somelib.dll"); + CHECK(NormalizeGuestPath("GAME:\\BIN\\SomeLib.DLL") == "bin/somelib.dll"); + CHECK(NormalizeGuestPath("d:\\content\\modules\\test.dll") == "content/modules/test.dll"); +} + +TEST_CASE("Guest path normalization: already normalized", "[system][path]") { + CHECK(NormalizeGuestPath("bin/somelib.dll") == "bin/somelib.dll"); +} + +TEST_CASE("Guest path normalization: backslash to forward slash", "[system][path]") { + CHECK(NormalizeGuestPath("data\\libs\\foo.dll") == "data/libs/foo.dll"); +} + +TEST_CASE("Guest path normalization: case folding", "[system][path]") { + CHECK(NormalizeGuestPath("BIN/SomeLib.DLL") == "bin/somelib.dll"); +} diff --git a/thirdparty/rexglue-sdk/thirdparty/disruptorplus/include/disruptorplus/multi_threaded_claim_strategy.hpp b/thirdparty/rexglue-sdk/thirdparty/disruptorplus/include/disruptorplus/multi_threaded_claim_strategy.hpp index 6f2e15cb..ae540eaf 100644 --- a/thirdparty/rexglue-sdk/thirdparty/disruptorplus/include/disruptorplus/multi_threaded_claim_strategy.hpp +++ b/thirdparty/rexglue-sdk/thirdparty/disruptorplus/include/disruptorplus/multi_threaded_claim_strategy.hpp @@ -312,7 +312,7 @@ namespace disruptorplus return false; } } - reducedCount = std::min(count, static_cast(diff + 1)); + reducedCount = std::min(count, static_cast(diff + 1)); } while (!m_nextClaimable.compare_exchange_weak( sequence, static_cast(sequence + reducedCount), diff --git a/tools/ac6_mode1_codec.py b/tools/ac6_mode1_codec.py new file mode 100644 index 00000000..a3b748e9 --- /dev/null +++ b/tools/ac6_mode1_codec.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import struct +import zlib +from functools import lru_cache + + +# Number of distinct pads. The keygen masks the index with 0xFF, so there are +# exactly 256 possible pads; entry indices beyond 255 reuse them mod 256. +_PAD_COUNT = 256 + +# We need pi fractional words up to 2*(255)+2 = 512, plus slack. +_PI_WORDS_NEEDED = 2 * (_PAD_COUNT - 1) + 3 + + +@lru_cache(maxsize=1) +def _pi_fractional_words(nwords: int) -> tuple[int, ...]: + """First `nwords` base-2^32 words of frac(pi), most-significant word first. + + Pure-integer Machin formula: pi = 16*arctan(1/5) - 4*arctan(1/239). + """ + bits = 32 * nwords + 64 + + def arctan_inv(x: int) -> int: + total = 0 + term = (1 << bits) // x + x2 = x * x + k = 0 + sign = 1 + while term // (2 * k + 1): + total += sign * (term // (2 * k + 1)) + term //= x2 + k += 1 + sign = -sign + return total + + pi_scaled = 16 * arctan_inv(5) - 4 * arctan_inv(239) + frac = pi_scaled - (3 << bits) # drop the integer part "3" + return tuple( + (frac >> (bits - 32 * (i + 1))) & 0xFFFFFFFF for i in range(nwords) + ) + + +@lru_cache(maxsize=_PAD_COUNT) +def pad_for_index(index: int) -> bytes: + """Return the 8-byte descramble pad for a DATA.TBL entry index.""" + words = _pi_fractional_words(_PI_WORDS_NEEDED) + w0 = 2 * (index & 0xFF) + 1 + return struct.pack(">II", words[w0], words[w0 + 1]) + + +def descramble(data: bytes, index: int) -> bytes: + """Undo the XOR scrambling for a compressed entry at the given table index.""" + pad = pad_for_index(index) + return bytes(b ^ pad[i & 7] for i, b in enumerate(data)) + + +def decompress_entry(data: bytes, index: int, expected_size: int | None = None) -> bytes: + """Descramble + raw-inflate a compressed PAC entry. + + Args: + data: the on-disk (scrambled, compressed) entry bytes from DATA00/01.PAC. + index: the entry's DATA.TBL row index (drives the descramble pad). + expected_size: optional decompressed size from DATA.TBL for validation. + + Returns the decompressed payload (typically an FHM container). + Raises zlib.error on a decode failure or ValueError on a size mismatch. + """ + raw = descramble(data, index) + out = zlib.decompress(raw, wbits=-15) + if expected_size is not None and len(out) != expected_size: + raise ValueError( + f"decompressed size mismatch: got {len(out)}, expected {expected_size}" + ) + return out + + +if __name__ == "__main__": + # Smoke test: print the first few pads so the table can be eyeballed. + for i in range(4): + print(f"pad[{i}] = {pad_for_index(i).hex()}") diff --git a/tools/extract_ac6_pac.py b/tools/extract_ac6_pac.py index 3ecd69d5..919b671d 100644 --- a/tools/extract_ac6_pac.py +++ b/tools/extract_ac6_pac.py @@ -6,8 +6,13 @@ import hashlib import json import os import struct +import sys +import zlib from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import ac6_mode1_codec + HEADER_SIZE = 8 ENTRY_SIZE = 16 @@ -56,7 +61,8 @@ def sha256_path(path: Path) -> str: return h.hexdigest() -def extract_entries(asset_root: Path, output_root: Path, entries: list[dict], include_compressed: bool) -> dict: +def extract_entries(asset_root: Path, output_root: Path, entries: list[dict], + include_compressed: bool, decompress: bool) -> dict: pac_bytes = { "DATA00.PAC": asset_root.joinpath("DATA00.PAC").read_bytes(), "DATA01.PAC": asset_root.joinpath("DATA01.PAC").read_bytes(), @@ -70,6 +76,8 @@ def extract_entries(asset_root: Path, output_root: Path, entries: list[dict], in manifest_entries = [] extracted_count = 0 skipped_count = 0 + decompressed_count = 0 + decode_failures = 0 for entry in entries: if entry["storage_kind"] == "compressed" and not include_compressed: @@ -87,20 +95,41 @@ def extract_entries(asset_root: Path, output_root: Path, entries: list[dict], in ) blob = pac_bytes[pac_name][start:end] + is_compressed = entry["storage_kind"] == "compressed" + + # Offline decode of compressed (mode-1) entries: pi-keygen descramble + + # raw DEFLATE. Raw entries are written as-is. + payload = blob + record = {**entry} + if is_compressed and decompress: + try: + payload = ac6_mode1_codec.decompress_entry( + blob, entry["index"], expected_size=entry["decompressed_size"] + ) + record["decompressed"] = True + decompressed_count += 1 + except (zlib.error, ValueError) as exc: + record["decompressed"] = False + record["decode_error"] = str(exc) + decode_failures += 1 + subdir = files_dir / pac_name.replace(".PAC", "") / entry["storage_kind"] subdir.mkdir(parents=True, exist_ok=True) - out_path = subdir / f"{entry['index']:04d}.bin" - out_path.write_bytes(blob) + suffix = ".bin" + if is_compressed and decompress and record.get("decompressed"): + suffix = ".decompressed.bin" + out_path = subdir / f"{entry['index']:04d}{suffix}" + out_path.write_bytes(payload) - manifest_entries.append( + record.update( { - **entry, "extracted": True, "path": str(out_path.relative_to(output_root)).replace("\\", "/"), - "sha256": hashlib.sha256(blob).hexdigest(), - "head_hex": blob[:32].hex(), + "sha256": hashlib.sha256(payload).hexdigest(), + "head_hex": payload[:32].hex(), } ) + manifest_entries.append(record) extracted_count += 1 manifest = { @@ -109,7 +138,10 @@ def extract_entries(asset_root: Path, output_root: Path, entries: list[dict], in "entry_count": len(entries), "extracted_count": extracted_count, "skipped_count": skipped_count, + "decompressed_count": decompressed_count, + "decode_failures": decode_failures, "include_compressed": include_compressed, + "decompress": decompress, "archives": { name: { "size": size, @@ -138,12 +170,22 @@ def main() -> int: action="store_true", help="Extract only entries marked raw in DATA.TBL and skip compressed entries", ) + parser.add_argument( + "--decompress", + action="store_true", + help="Decode compressed (mode-1) entries offline (pi-keygen descramble + raw DEFLATE) " + "instead of writing the raw scrambled blob", + ) args = parser.parse_args() asset_root = args.asset_root.resolve() output_root = args.output.resolve() entries = parse_tbl(asset_root / "DATA.TBL") - manifest = extract_entries(asset_root, output_root, entries, include_compressed=not args.raw_only) + manifest = extract_entries( + asset_root, output_root, entries, + include_compressed=not args.raw_only, + decompress=args.decompress, + ) print( json.dumps( @@ -151,6 +193,8 @@ def main() -> int: "entry_count": manifest["entry_count"], "extracted_count": manifest["extracted_count"], "skipped_count": manifest["skipped_count"], + "decompressed_count": manifest["decompressed_count"], + "decode_failures": manifest["decode_failures"], "output_root": manifest["output_root"], }, indent=2, diff --git a/tools/launch_ac6_with_pac_dump.ps1 b/tools/launch_ac6_with_pac_dump.ps1 deleted file mode 100644 index cd4085c5..00000000 --- a/tools/launch_ac6_with_pac_dump.ps1 +++ /dev/null @@ -1,44 +0,0 @@ -[CmdletBinding()] -param( - # Enable the PAC stream-worker dispatch probe. When set, every distinct - # work-item virtual that rex_sub_82343E18 dispatches gets logged once - # via `[AC6 PAC WORKER] new dispatch target=...`. Cross-reference these - # against compressed-entry writes to identify the mode-1 decoder. - [switch]$TraceWorkItems, - - # Enable per-NtReadFile guest stack traces on PAC reads. Each call into - # NtReadFile / NtReadFileScatter for a PAC path logs `stack=[...]` with - # the full guest back-chain. Used to pin the decoder when it sits above - # the read-issuing function on the reader thread's call chain. - [switch]$TraceStacks -) - -$ErrorActionPreference = 'Stop' - -$repoRoot = Split-Path -Parent $PSScriptRoot -$exePath = Join-Path $repoRoot 'out\build\win-amd64-relwithdebinfo\ac6recomp.exe' - -if (-not (Test-Path -LiteralPath $exePath)) { - throw "ac6recomp.exe not found at $exePath" -} - -$env:AC6_DUMP_PAC_DECODED = '1' -Write-Host "AC6_DUMP_PAC_DECODED=1" - -if ($TraceWorkItems) { - $env:AC6_TRACE_PAC_WORK_ITEMS = '1' - Write-Host "AC6_TRACE_PAC_WORK_ITEMS=1" -} else { - Remove-Item Env:AC6_TRACE_PAC_WORK_ITEMS -ErrorAction SilentlyContinue -} - -if ($TraceStacks) { - $env:AC6_TRACE_PAC_STACKS = '1' - Write-Host "AC6_TRACE_PAC_STACKS=1" -} else { - Remove-Item Env:AC6_TRACE_PAC_STACKS -ErrorAction SilentlyContinue -} - -Write-Host "Launching $exePath" - -Start-Process -FilePath $exePath -WorkingDirectory (Split-Path -Parent $exePath) diff --git a/tools/pac_probe_lzx.cpp b/tools/pac_probe_lzx.cpp deleted file mode 100644 index ec37a359..00000000 --- a/tools/pac_probe_lzx.cpp +++ /dev/null @@ -1,333 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "mspack/lzx.h" -#include "mspack/mspack.h" -} - -namespace { - -constexpr uint32_t kEntrySize = 16; -constexpr uint32_t kHeaderSize = 8; -constexpr int kInputBufferSize = 1 << 15; - -struct TblEntry { - uint32_t group; - uint32_t offset; - uint32_t compressed_size; - uint32_t decompressed_size; -}; - -struct ProbeInput { - std::vector data; - size_t pos = 0; -}; - -struct ProbeOutput { - std::vector data; -}; - -struct ProbeFile { - mspack_file base{}; - ProbeInput* input = nullptr; - ProbeOutput* output = nullptr; -}; - -struct ProbeResult { - int window_bits = 0; - int reset_interval = 0; - int status = MSPACK_ERR_ARGS; - std::vector output; -}; - -uint32_t ReadBE32(const uint8_t* bytes) { - return (uint32_t(bytes[0]) << 24) | (uint32_t(bytes[1]) << 16) | - (uint32_t(bytes[2]) << 8) | uint32_t(bytes[3]); -} - -std::string HexPrefix(const std::vector& data, size_t count) { - std::ostringstream out; - out << std::hex << std::setfill('0'); - const size_t limit = std::min(count, data.size()); - for (size_t i = 0; i < limit; ++i) { - out << std::setw(2) << unsigned(data[i]); - } - return out.str(); -} - -std::string AsciiPrefix(const std::vector& data, size_t count) { - std::string out; - const size_t limit = std::min(count, data.size()); - out.reserve(limit); - for (size_t i = 0; i < limit; ++i) { - const unsigned char c = data[i]; - out.push_back(std::isprint(c) ? char(c) : '.'); - } - return out; -} - -std::optional> ReadFile(const std::string& path) { - std::ifstream file(path, std::ios::binary); - if (!file) { - return std::nullopt; - } - file.seekg(0, std::ios::end); - const auto size = file.tellg(); - if (size < 0) { - return std::nullopt; - } - file.seekg(0, std::ios::beg); - std::vector data(static_cast(size)); - if (!data.empty()) { - file.read(reinterpret_cast(data.data()), static_cast(data.size())); - if (!file) { - return std::nullopt; - } - } - return data; -} - -std::optional> ParseTbl(const std::string& path) { - const auto bytes = ReadFile(path); - if (!bytes || bytes->size() < kHeaderSize) { - return std::nullopt; - } - - const uint32_t count = ReadBE32(bytes->data()); - const uint32_t pack_count = ReadBE32(bytes->data() + 4); - (void)pack_count; - if (bytes->size() != kHeaderSize + (size_t(count) * kEntrySize)) { - return std::nullopt; - } - - std::vector entries; - entries.reserve(count); - for (uint32_t i = 0; i < count; ++i) { - const uint8_t* p = bytes->data() + kHeaderSize + (i * kEntrySize); - entries.push_back(TblEntry{ - ReadBE32(p + 0), - ReadBE32(p + 4), - ReadBE32(p + 8), - ReadBE32(p + 12), - }); - } - return entries; -} - -int ProbeRead(mspack_file* file, void* buffer, int bytes) { - auto* handle = reinterpret_cast(file); - if (!handle || !handle->input || bytes < 0) { - return -1; - } - const size_t remaining = handle->input->data.size() - handle->input->pos; - const size_t to_read = std::min(remaining, static_cast(bytes)); - if (to_read > 0) { - std::memcpy(buffer, handle->input->data.data() + handle->input->pos, to_read); - handle->input->pos += to_read; - } - return static_cast(to_read); -} - -int ProbeWrite(mspack_file* file, void* buffer, int bytes) { - auto* handle = reinterpret_cast(file); - if (!handle || !handle->output || bytes < 0) { - return -1; - } - const auto* src = reinterpret_cast(buffer); - handle->output->data.insert(handle->output->data.end(), src, src + bytes); - return bytes; -} - -int ProbeSeek(mspack_file* file, off_t offset, int mode) { - auto* handle = reinterpret_cast(file); - if (!handle || !handle->input) { - return -1; - } - - size_t base = 0; - switch (mode) { - case MSPACK_SYS_SEEK_START: - base = 0; - break; - case MSPACK_SYS_SEEK_CUR: - base = handle->input->pos; - break; - case MSPACK_SYS_SEEK_END: - base = handle->input->data.size(); - break; - default: - return -1; - } - - if (offset < 0 && static_cast(-offset) > base) { - return -1; - } - - const size_t next = offset >= 0 ? base + static_cast(offset) - : base - static_cast(-offset); - if (next > handle->input->data.size()) { - return -1; - } - handle->input->pos = next; - return 0; -} - -off_t ProbeTell(mspack_file* file) { - auto* handle = reinterpret_cast(file); - if (!handle || !handle->input) { - return off_t(-1); - } - return static_cast(handle->input->pos); -} - -void ProbeMessage(mspack_file*, const char* format, ...) { - std::va_list args; - va_start(args, format); - std::vfprintf(stderr, format, args); - std::fputc('\n', stderr); - va_end(args); -} - -void* ProbeAlloc(mspack_system*, size_t bytes) { - return std::malloc(bytes); -} - -void ProbeFree(void* ptr) { - std::free(ptr); -} - -void ProbeCopy(void* src, void* dest, size_t bytes) { - std::memcpy(dest, src, bytes); -} - -ProbeResult TryLzx(const std::vector& compressed, uint32_t expected_size, - int window_bits, int reset_interval) { - ProbeInput input{compressed, 0}; - ProbeOutput output; - ProbeFile in_file{}; - ProbeFile out_file{}; - in_file.input = &input; - out_file.output = &output; - - mspack_system system{}; - system.open = nullptr; - system.close = nullptr; - system.read = &ProbeRead; - system.write = &ProbeWrite; - system.seek = &ProbeSeek; - system.tell = &ProbeTell; - system.message = &ProbeMessage; - system.alloc = &ProbeAlloc; - system.free = &ProbeFree; - system.copy = &ProbeCopy; - system.null_ptr = nullptr; - - ProbeResult result; - result.window_bits = window_bits; - result.reset_interval = reset_interval; - - lzxd_stream* lzx = lzxd_init(&system, &in_file.base, &out_file.base, window_bits, - reset_interval, kInputBufferSize, expected_size, 0); - if (!lzx) { - result.status = MSPACK_ERR_NOMEMORY; - return result; - } - - result.status = lzxd_decompress(lzx, expected_size); - result.output = std::move(output.data); - lzxd_free(lzx); - return result; -} - -std::string PacPathForGroup(const std::string& asset_root, uint32_t group) { - const bool is_data01 = (group & 0x01000000u) != 0; - return asset_root + "\\" + (is_data01 ? "DATA01.PAC" : "DATA00.PAC"); -} - -void Usage() { - std::cerr << "usage: pac_probe_lzx \n"; -} - -} // namespace - -int main(int argc, char** argv) { - if (argc != 3) { - Usage(); - return 1; - } - - const std::string asset_root = argv[1]; - const int entry_index = std::atoi(argv[2]); - if (entry_index < 0) { - std::cerr << "invalid entry index\n"; - return 1; - } - - const auto entries = ParseTbl(asset_root + "\\DATA.TBL"); - if (!entries) { - std::cerr << "failed to parse DATA.TBL\n"; - return 1; - } - if (static_cast(entry_index) >= entries->size()) { - std::cerr << "entry index out of range\n"; - return 1; - } - - const TblEntry& entry = (*entries)[entry_index]; - const auto pac_bytes = ReadFile(PacPathForGroup(asset_root, entry.group)); - if (!pac_bytes) { - std::cerr << "failed to read PAC file\n"; - return 1; - } - if (size_t(entry.offset) + size_t(entry.compressed_size) > pac_bytes->size()) { - std::cerr << "entry is out of bounds for PAC file\n"; - return 1; - } - - const std::vector compressed( - pac_bytes->begin() + entry.offset, - pac_bytes->begin() + entry.offset + entry.compressed_size); - - std::cout << "entry=" << entry_index << " group=0x" << std::hex << entry.group - << " offset=0x" << entry.offset << " csize=0x" << entry.compressed_size - << " usize=0x" << entry.decompressed_size << std::dec << "\n"; - std::cout << "compressed_head_hex=" << HexPrefix(compressed, 32) << "\n"; - - std::array reset_candidates{0, 1, 2, 4, 8, 16, 32}; - bool found = false; - for (int window_bits = 15; window_bits <= 21; ++window_bits) { - for (int reset_interval : reset_candidates) { - ProbeResult result = TryLzx(compressed, entry.decompressed_size, window_bits, reset_interval); - if (result.status == MSPACK_ERR_OK && result.output.size() == entry.decompressed_size) { - found = true; - std::cout << "OK window_bits=" << window_bits - << " reset_interval=" << reset_interval - << " out_head_hex=" << HexPrefix(result.output, 32) - << " out_head_ascii=" << AsciiPrefix(result.output, 32) << "\n"; - } else { - std::cout << "FAIL window_bits=" << window_bits - << " reset_interval=" << reset_interval - << " status=" << result.status - << " produced=" << result.output.size() << "\n"; - } - } - } - - return found ? 0 : 2; -} diff --git a/tools/unpack_ac6_fhm.py b/tools/unpack_ac6_fhm.py new file mode 100644 index 00000000..011b8459 --- /dev/null +++ b/tools/unpack_ac6_fhm.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import struct +from pathlib import Path + + +MAGIC_EXT = { + "FHM ": ".fhm", "NTXR": ".ntxr", "NDXR": ".ndxr", "NSXR": ".nsxr", + "MDLP": ".mdlp", "PLAD": ".plad", "MATE": ".mate", "NFIC": ".nfic", + "NFH\x00": ".nfh", "CAPT": ".capt", "Scen": ".scen", "ACE6": ".ace6", + "RIFF": ".wav", "SWG\x00": ".swg", +} + + +def parse_fhm(blob: bytes): + """Return list of (index, offset, child_bytes) or None if not an FHM.""" + if len(blob) < 0x1C or blob[:4] != b"FHM ": + return None + count = struct.unpack_from(">I", blob, 0x10)[0] + if count == 0 or count > 100000: + return [] + offs_base = 0x14 + size_base = offs_base + count * 4 + if size_base + count * 4 > len(blob): + return [] + offsets = [struct.unpack_from(">I", blob, offs_base + i * 4)[0] for i in range(count)] + sizes = [struct.unpack_from(">I", blob, size_base + i * 4)[0] for i in range(count)] + out = [] + for i, (off, sz) in enumerate(zip(offsets, sizes)): + if off == 0 or off >= len(blob): + continue + end = off + sz + if end > len(blob) or end <= off: + nxt = offsets[i + 1] if i + 1 < count else len(blob) + end = min(nxt, len(blob)) + if end > off: + out.append((i, off, blob[off:end])) + return out + + +def magic_of(blob: bytes) -> str: + return blob[:4].decode("latin-1") if len(blob) >= 4 else "" + + +def ext_for(blob: bytes) -> str: + return MAGIC_EXT.get(magic_of(blob), ".bin") + + +def safe_tag(magic: str) -> str: + return "".join(c if c.isalnum() else "_" for c in magic) or "raw" + + +def unpack(blob: bytes, out_dir: Path, root: Path, depth: int, max_depth: int) -> list[dict]: + children = parse_fhm(blob) + if children is None: + return [] + recs = [] + out_dir.mkdir(parents=True, exist_ok=True) + for idx, off, child in children: + magic = magic_of(child) + name = f"{idx:04d}_{safe_tag(magic)}{ext_for(child)}" + path = out_dir / name + path.write_bytes(child) + rec = {"index": idx, "offset": off, "size": len(child), "magic": magic, + "path": str(path.relative_to(root)).replace("\\", "/")} + if depth < max_depth and child[:4] == b"FHM ": + nested = unpack(child, out_dir / f"{idx:04d}_FHM", root, depth + 1, max_depth) + if nested: + rec["children"] = nested + recs.append(rec) + return recs + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--input", type=Path, default=Path("out") / "ac6_pac_extracted_raw" / "files") + ap.add_argument("--output", type=Path, default=Path("out") / "ac6_fhm_unpacked") + ap.add_argument("--max-depth", type=int, default=8) + args = ap.parse_args() + + inp = args.input.resolve() + out_root = args.output.resolve() + out_root.mkdir(parents=True, exist_ok=True) + + if inp.is_file(): + sources = [inp] + else: + sources = sorted(inp.rglob("*.bin")) + + manifest = [] + fhm_count = leaf_count = 0 + for src in sources: + blob = src.read_bytes() + stem = src.stem.split(".")[0] + if blob[:4] != b"FHM ": + # Top-level non-FHM (raw entry): copy through as a leaf. + dst = out_root / f"{stem}{ext_for(blob)}" + dst.write_bytes(blob) + manifest.append({"source": src.name, "kind": "leaf", "magic": magic_of(blob), + "path": str(dst.relative_to(out_root)).replace("\\", "/")}) + leaf_count += 1 + continue + cdir = out_root / stem + recs = unpack(blob, cdir, out_root, 0, args.max_depth) + manifest.append({"source": src.name, "kind": "fhm", "child_count": len(recs), + "children": recs}) + fhm_count += 1 + + def count_leaves(recs): + n = 0 + for r in recs: + if "children" in r: + n += count_leaves(r["children"]) + else: + n += 1 + return n + + total_leaves = leaf_count + sum( + count_leaves(m["children"]) for m in manifest if m["kind"] == "fhm") + (out_root / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + print(json.dumps({"sources": len(sources), "fhm_containers": fhm_count, + "total_leaves": total_leaves, "output": str(out_root)}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())