From ed8b3ebee13a5909da9910ade730e2dde6e713e3 Mon Sep 17 00:00:00 2001 From: Sinan Date: Sat, 6 Jun 2026 05:15:14 +0200 Subject: [PATCH] Reduce recompiler output memory usage, and added multi-threading to recomp process (#128) * feat(recomp): Reduce recompiler output memory usage Stream output generation, add low-memory config controls, and avoid pathological indirect-jump switch expansion in generated C++. Low-memory mode now avoids retaining per-instruction disassembly strings while still emitting asm comments during output generation. Output workers are bounded/configurable, combined output is streamed, and decoded buffers are released after generation. Also document the new output memory settings. * fix(recomp): added tests for unregistered JR/JALR, updated fallback logic to cover JR/JALR, moved Rabbitizer formatting into R5900Decoder --- README.md | 4 + ps2xRecomp/example_config.toml | 7 + ps2xRecomp/include/ps2recomp/code_generator.h | 3 + ps2xRecomp/include/ps2recomp/r5900_decoder.h | 8 +- ps2xRecomp/include/ps2recomp/types.h | 2 + ps2xRecomp/src/lib/code_generator.cpp | 58 +- ps2xRecomp/src/lib/config_manager.cpp | 20 + ps2xRecomp/src/lib/ps2_recompiler.cpp | 516 +++++++++++++++--- ps2xRecomp/src/lib/r5900_decoder.cpp | 45 +- ps2xTest/src/code_generator_tests.cpp | 122 ++++- 10 files changed, 659 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index a5ef4f4..e164ffb 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ Main fields in `config.toml`: * `general.ghidra_output`: recommended function map CSV exported from Ghidra. * `general.output`: generated C++ output folder. * `general.single_file_output`: one combined cpp or one file per function. +* `general.low_memory_mode`: reduce peak output-generation memory by avoiding retained disassembly strings and forcing serial output generation. Generated instruction comments are still emitted; disassembly text is produced while writing each output file instead of being kept in memory. +* `general.output_worker_threads`: number of output-generation workers (clamped to nproc * 2). A positive value uses exactly that many workers. `0` uses `nproc - 1` when at least two hardware threads are available, otherwise serial output generation. `1` forces serial output generation. * `general.patch_syscalls`: apply configured patches to `SYSCALL` instructions (`false` recommended). * `general.patch_cop0`: apply configured patches to COP0 instructions. * `general.patch_cache`: apply configured patches to CACHE instructions. @@ -118,6 +120,8 @@ ghidra_output = "" output = "output/" single_file_output = true +low_memory_mode = true +output_worker_threads = 0 patch_syscalls = false patch_cop0 = true patch_cache = true diff --git a/ps2xRecomp/example_config.toml b/ps2xRecomp/example_config.toml index 14fbdad..2cfb541 100644 --- a/ps2xRecomp/example_config.toml +++ b/ps2xRecomp/example_config.toml @@ -8,6 +8,13 @@ output = "output/" # Single file output mode (false for one file per function) single_file_output = false +# Lower peak memory by avoiding retained disassembly strings and forcing serial output generation. +low_memory_mode = false + +# Function generation workers. 0 uses nproc - 1 when at least 2 hardware threads are available; 1 disables parallel generation. +# Limited to nproc * 2 to avoid oversubscription. +output_worker_threads = 0 + # Path to runtime header (optional) runtime_header = "include/ps2_runtime.h" diff --git a/ps2xRecomp/include/ps2recomp/code_generator.h b/ps2xRecomp/include/ps2recomp/code_generator.h index e5ce9e1..5fdecf0 100644 --- a/ps2xRecomp/include/ps2recomp/code_generator.h +++ b/ps2xRecomp/include/ps2recomp/code_generator.h @@ -39,6 +39,7 @@ namespace ps2recomp std::unordered_set entryPoints; std::unordered_set externalEntryPoints; std::unordered_set resumeEntryPoints; + std::unordered_set indirectFallbackEntryPoints; std::unordered_map> jumpTableTargets; }; @@ -52,6 +53,7 @@ namespace ps2recomp void setRelocationCallNames(const std::unordered_map &callNames); void setConfiguredJumpTables(const std::vector &jumpTables); void setResumeEntryTargets(const std::unordered_map> &resumeTargetsByOwner); + void setEmitInstructionComments(bool emitInstructionComments); AnalysisResult collectInternalBranchTargets(const Function &function, const std::vector &instructions, @@ -65,6 +67,7 @@ namespace ps2recomp std::unordered_map> m_resumeEntryTargetsByOwner; const std::vector
& m_sections; BootstrapInfo m_bootstrapInfo; + bool m_emitInstructionComments = true; std::string translateInstruction(const Instruction &inst); std::string translateMMIInstruction(const Instruction &inst); diff --git a/ps2xRecomp/include/ps2recomp/r5900_decoder.h b/ps2xRecomp/include/ps2recomp/r5900_decoder.h index 1b3d178..2b7bc80 100644 --- a/ps2xRecomp/include/ps2recomp/r5900_decoder.h +++ b/ps2xRecomp/include/ps2recomp/r5900_decoder.h @@ -4,6 +4,7 @@ #include "ps2recomp/types.h" #include "ps2recomp/instructions.h" #include +#include namespace ps2recomp { @@ -14,7 +15,10 @@ namespace ps2recomp R5900Decoder(); ~R5900Decoder(); - Instruction decodeInstruction(uint32_t address, uint32_t rawInstruction) const; + static std::string disassembleInstruction(uint32_t address, uint32_t rawInstruction); + static std::string disassembleInstruction(const Instruction &inst); + + Instruction decodeInstruction(uint32_t address, uint32_t rawInstruction, bool includeDisassembly = true) const; bool isBranchInstruction(const Instruction &inst) const; bool isJumpInstruction(const Instruction &inst) const; @@ -52,4 +56,4 @@ namespace ps2recomp } // namespace ps2recomp -#endif // PS2RECOMP_R5900_DECODER_H \ No newline at end of file +#endif // PS2RECOMP_R5900_DECODER_H diff --git a/ps2xRecomp/include/ps2recomp/types.h b/ps2xRecomp/include/ps2recomp/types.h index 4b667e6..7e03280 100644 --- a/ps2xRecomp/include/ps2recomp/types.h +++ b/ps2xRecomp/include/ps2recomp/types.h @@ -173,6 +173,8 @@ namespace ps2recomp std::string outputPath; std::string ghidraMapPath; bool singleFileOutput = false; + bool lowMemoryMode = false; + uint32_t outputWorkerThreads = 0; bool patchSyscalls = false; bool patchCop0 = true; bool patchCache = true; diff --git a/ps2xRecomp/src/lib/code_generator.cpp b/ps2xRecomp/src/lib/code_generator.cpp index faa0549..96aba20 100644 --- a/ps2xRecomp/src/lib/code_generator.cpp +++ b/ps2xRecomp/src/lib/code_generator.cpp @@ -1,6 +1,7 @@ #include "ps2recomp/code_generator.h" #include "ps2recomp/instructions.h" #include "ps2recomp/ps2_recompiler.h" +#include "ps2recomp/r5900_decoder.h" #include "ps2recomp/types.h" #include "ps2_runtime_calls.h" #include @@ -12,6 +13,7 @@ #include #include #include +#include namespace ps2recomp { @@ -164,6 +166,11 @@ namespace ps2recomp } } + void CodeGenerator::setEmitInstructionComments(bool emitInstructionComments) + { + m_emitInstructionComments = emitInstructionComments; + } + std::string CodeGenerator::getFunctionName(uint32_t address) const { auto it = m_renamedFunctions.find(address); @@ -223,11 +230,16 @@ namespace ps2recomp std::string delaySlotSuffix = ""; if (hasValidDelaySlot) { delaySlotPrefix = "ctx->in_delay_slot = true; ctx->branch_pc = 0x" + fmt::format("{:X}", branchInst.address) + "u;\n "; - delaySlotCode = " // 0x" + fmt::format("{:x}", delaySlot.address) + ": 0x" + fmt::format("{:x}", delaySlot.raw); - if (!delaySlot.disassembly.empty()) { - delaySlotCode += " " + delaySlot.disassembly; + if (m_emitInstructionComments) + { + delaySlotCode = " // 0x" + fmt::format("{:x}", delaySlot.address) + ": 0x" + fmt::format("{:x}", delaySlot.raw); + std::string disassembly = R5900Decoder::disassembleInstruction(delaySlot); + if (!disassembly.empty()) { + delaySlotCode += " " + disassembly; + } + delaySlotCode += " (Delay Slot)\n "; } - delaySlotCode += " (Delay Slot)\n " + translateInstruction(delaySlot); + delaySlotCode += translateInstruction(delaySlot); delaySlotSuffix = "\n ctx->in_delay_slot = false;"; } @@ -259,20 +271,18 @@ namespace ps2recomp std::vector sortedInternalTargets; if (branchInst.opcode == OPCODE_SPECIAL && - (branchInst.function == SPECIAL_JR || branchInst.function == SPECIAL_JALR) && + ((branchInst.function == SPECIAL_JR && branchInst.rs != 31) || + branchInst.function == SPECIAL_JALR) && !internalTargets.empty()) { + // Only emit local indirect-jump switches for jump tables we actually resolved. + // Falling back to every internal target here can duplicate huge switches at each + // indirect branch. Unresolved JR/JALR targets are registered as resumable entries + // instead, so runtime dispatch can re-enter this function at ctx->pc. auto jtIt = analysisResult.jumpTableTargets.find(branchInst.address); if (jtIt != analysisResult.jumpTableTargets.end()) { sortedInternalTargets = jtIt->second; std::sort(sortedInternalTargets.begin(), sortedInternalTargets.end()); - } else { - sortedInternalTargets.reserve(internalTargets.size()); - for (uint32_t t : internalTargets) - { - sortedInternalTargets.push_back(t); - } - std::sort(sortedInternalTargets.begin(), sortedInternalTargets.end()); } } @@ -827,7 +837,7 @@ namespace ps2recomp if (hasIndirectRegisterJump) { - bool needsJrFallback = false; + bool needsIndirectFallback = false; for (const Instruction* jrInst : indirectJumps) { if (jrInst->function == SPECIAL_JALR) { @@ -997,19 +1007,19 @@ namespace ps2recomp } } if (!foundTable) { - if (!(jrInst->function == SPECIAL_JALR)) - { - needsJrFallback = true; - } + needsIndirectFallback = true; } } - if (needsJrFallback) { + if (needsIndirectFallback) { for (uint32_t addr : instructionAddresses) { if (addr >= function.start && addr < function.end) { result.entryPoints.insert(addr); + // Keep labels and runtime registration for unresolved JR/JALR targets + // without emitting a local switch over every possible target. + result.indirectFallbackEntryPoints.insert(addr); } } } @@ -1093,11 +1103,15 @@ namespace ps2recomp ss << "label_" << std::hex << inst.address << std::dec << ":\n"; } - ss << " // 0x" << std::hex << inst.address << ": 0x" << inst.raw << std::dec; - if (!inst.disassembly.empty()) { - ss << " " << inst.disassembly; + if (m_emitInstructionComments) + { + ss << " // 0x" << std::hex << inst.address << ": 0x" << inst.raw << std::dec; + std::string disassembly = R5900Decoder::disassembleInstruction(inst); + if (!disassembly.empty()) { + ss << " " << disassembly; + } + ss << "\n"; } - ss << "\n"; try { diff --git a/ps2xRecomp/src/lib/config_manager.cpp b/ps2xRecomp/src/lib/config_manager.cpp index ab15df1..f7b7321 100644 --- a/ps2xRecomp/src/lib/config_manager.cpp +++ b/ps2xRecomp/src/lib/config_manager.cpp @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include namespace ps2recomp { @@ -29,6 +32,21 @@ namespace ps2recomp config.ghidraMapPath = toml::find_or(general, "ghidra_output", ""); config.outputPath = toml::find(general, "output"); config.singleFileOutput = toml::find_or(general, "single_file_output", false); + config.lowMemoryMode = toml::find_or(general, "low_memory_mode", config.lowMemoryMode); + const int64_t configuredOutputWorkers = toml::find_or( + general, + "output_worker_threads", + toml::find_or(general, "output_worker_thread", config.outputWorkerThreads)); + const int64_t clampedOutputWorkers = std::clamp( + configuredOutputWorkers, + 0, + std::thread::hardware_concurrency() * 2); + if (configuredOutputWorkers != clampedOutputWorkers) + { + std::cerr << "Warning: output_worker_threads value " << configuredOutputWorkers + << " is out of range; clamped to " << clampedOutputWorkers << "." << std::endl; + } + config.outputWorkerThreads = static_cast(clampedOutputWorkers); config.patchSyscalls = toml::find_or(general, "patch_syscalls", config.patchSyscalls); config.patchCop0 = toml::find_or(general, "patch_cop0", config.patchCop0); config.patchCache = toml::find_or(general, "patch_cache", config.patchCache); @@ -234,6 +252,8 @@ namespace ps2recomp general["ghidra_output"] = config.ghidraMapPath; general["output"] = config.outputPath; general["single_file_output"] = config.singleFileOutput; + general["low_memory_mode"] = config.lowMemoryMode; + general["output_worker_threads"] = static_cast(config.outputWorkerThreads); general["patch_syscalls"] = config.patchSyscalls; general["patch_cop0"] = config.patchCop0; general["patch_cache"] = config.patchCache; diff --git a/ps2xRecomp/src/lib/ps2_recompiler.cpp b/ps2xRecomp/src/lib/ps2_recompiler.cpp index 428bbdf..0ac3ee5 100644 --- a/ps2xRecomp/src/lib/ps2_recompiler.cpp +++ b/ps2xRecomp/src/lib/ps2_recompiler.cpp @@ -11,10 +11,15 @@ #include #include #include +#include +#include +#include +#include #include #include #include #include +#include namespace fs = std::filesystem; @@ -77,6 +82,36 @@ namespace ps2recomp return function.isRecompiled || function.isStub || function.isSkipped; } + size_t resolveOutputWorkerCount(uint32_t configuredWorkerCount) + { + if (configuredWorkerCount > 0) + { + return configuredWorkerCount; + } + + const unsigned int hardwareWorkers = std::thread::hardware_concurrency(); + if (hardwareWorkers >= 2) + { + return static_cast(hardwareWorkers - 1); + } + + return 1; + } + + void writeCombinedOutputPreamble(std::ostream &output) + { + output << "#include \"ps2_recompiled_functions.h\"\n\n"; + output << "#include \"ps2_runtime_macros.h\"\n"; + output << "#include \"ps2_runtime.h\"\n"; + output << "#include \"ps2_recompiled_stubs.h\"\n"; + output << "#include \"ps2_syscalls.h\"\n"; + output << "#include \"ps2_stubs.h\"\n"; + output << "#ifdef _DEBUG\n"; + output << "#include \"ps2_log.h\"\n"; + output << "#endif\n"; + output << "\n"; + } + enum class PatchClass { Generic, @@ -850,6 +885,7 @@ namespace ps2recomp m_codeGenerator->setRelocationCallNames(relocationCallNames); m_codeGenerator->setBootstrapInfo(m_bootstrapInfo); m_codeGenerator->setConfiguredJumpTables(m_config.jumpTables); + m_codeGenerator->setEmitInstructionComments(true); fs::create_directories(m_config.outputPath); @@ -1050,110 +1086,451 @@ namespace ps2recomp generateFunctionHeader(); + std::vector outputFunctions; + outputFunctions.reserve(m_functions.size()); + for (const auto &function : m_functions) + { + if (shouldGenerateCodeForFunction(function)) + { + outputFunctions.push_back(&function); + } + } + + const size_t outputWorkerCount = m_config.lowMemoryMode ? 1 : resolveOutputWorkerCount(m_config.outputWorkerThreads); + if (outputFunctions.size() > 1 && outputWorkerCount > 1) + { + std::cout << "Generating function output with " << outputWorkerCount << " worker(s)." << std::endl; + } + + const auto &generatedStubs = m_generatedStubs; + const auto &decodedFunctions = m_decodedFunctions; + + auto generateFunctionCode = [&](CodeGenerator &codeGenerator, const Function &function, bool useHeaders) -> std::string + { + try + { + if (function.isStub || function.isSkipped) + { + if (!useHeaders) + { + return generatedStubs.at(function.start); + } + + std::stringstream stubFile; + stubFile << "#include \"ps2_runtime.h\"\n"; + stubFile << "#include \"ps2_syscalls.h\"\n"; + stubFile << "#include \"ps2_stubs.h\"\n"; + stubFile << "#ifdef _DEBUG\n"; + stubFile << "#include \"ps2_log.h\"\n"; + stubFile << "#endif\n"; + stubFile << "\n"; + stubFile << generatedStubs.at(function.start) << "\n"; + return stubFile.str(); + } + + const auto &instructions = decodedFunctions.at(function.start); + return codeGenerator.generateFunction(function, instructions, useHeaders); + } + catch (const std::exception &e) + { + std::cerr << "Error generating code for function " + << function.name << " (start 0x" + << std::hex << function.start << std::dec << "): " + << e.what() << std::endl; + throw; + } + }; + if (m_config.singleFileOutput) { - std::stringstream combinedOutput; - - combinedOutput << "#include \"ps2_recompiled_functions.h\"\n\n"; - combinedOutput << "#include \"ps2_runtime_macros.h\"\n"; - combinedOutput << "#include \"ps2_runtime.h\"\n"; - combinedOutput << "#include \"ps2_recompiled_stubs.h\"\n"; - combinedOutput << "#include \"ps2_syscalls.h\"\n"; - combinedOutput << "#include \"ps2_stubs.h\"\n"; - combinedOutput << "#ifdef _DEBUG\n"; - combinedOutput << "#include \"ps2_log.h\"\n"; - combinedOutput << "#endif\n"; - combinedOutput << "\n"; - - for (const auto &function : m_functions) + fs::path outputPath = fs::path(m_config.outputPath) / "ps2_recompiled_functions.cpp"; + std::ofstream combinedOutput(outputPath); + if (!combinedOutput) { - if (!shouldGenerateCodeForFunction(function)) + throw std::runtime_error("Failed to open combined output: " + outputPath.string()); + } + + writeCombinedOutputPreamble(combinedOutput); + + if (outputWorkerCount <= 1) + { + for (const Function *function : outputFunctions) { - continue; + combinedOutput << generateFunctionCode(*m_codeGenerator, *function, false) << "\n\n"; } + } + else + { + struct CompletedCode + { + size_t outputIndex = 0; + std::string code; + }; + + std::mutex outputMutex; + std::condition_variable workAvailable; + std::condition_variable resultAvailable; + std::queue pendingWork; + std::queue readyCode; + std::unordered_map completedCode; + std::exception_ptr workerException; + bool stopWorkers = false; + size_t outstandingWork = 0; + size_t nextFunction = 0; + size_t nextOutputIndex = 0; + const size_t maxBufferedOutput = std::max(outputWorkerCount * 2, outputWorkerCount + 1); + + auto workerMain = [&]() + { + CodeGenerator generator(*m_codeGenerator); + while (true) + { + size_t outputIndex = 0; + { + std::unique_lock lock(outputMutex); + workAvailable.wait(lock, [&]() + { return stopWorkers || !pendingWork.empty(); }); + if (stopWorkers) + { + return; + } + + outputIndex = pendingWork.front(); + pendingWork.pop(); + } + + try + { + std::string code = generateFunctionCode(generator, *outputFunctions[outputIndex], false); + { + std::lock_guard lock(outputMutex); + readyCode.push(CompletedCode{outputIndex, std::move(code)}); + --outstandingWork; + } + resultAvailable.notify_one(); + } + catch (...) + { + { + std::lock_guard lock(outputMutex); + if (!workerException) + { + workerException = std::current_exception(); + } + --outstandingWork; + stopWorkers = true; + } + resultAvailable.notify_one(); + workAvailable.notify_all(); + return; + } + } + }; + + std::vector workers; + workers.reserve(outputWorkerCount); + for (size_t i = 0; i < outputWorkerCount; ++i) + { + workers.emplace_back(workerMain); + } + + auto stopAndJoinWorkers = [&]() + { + { + std::lock_guard lock(outputMutex); + stopWorkers = true; + } + workAvailable.notify_all(); + for (std::thread &worker : workers) + { + if (worker.joinable()) + { + worker.join(); + } + } + }; + + auto scheduleAvailableWork = [&]() + { + bool scheduledAny = false; + { + std::lock_guard lock(outputMutex); + while (nextFunction < outputFunctions.size() && + outstandingWork + completedCode.size() < maxBufferedOutput && + !stopWorkers) + { + pendingWork.push(nextFunction++); + ++outstandingWork; + scheduledAny = true; + } + } + + if (scheduledAny) + { + workAvailable.notify_all(); + } + }; + + auto flushCompletedOutput = [&]() + { + while (true) + { + auto completedIt = completedCode.find(nextOutputIndex); + if (completedIt == completedCode.end()) + { + break; + } + + combinedOutput << completedIt->second << "\n\n"; + completedCode.erase(completedIt); + ++nextOutputIndex; + + if (!combinedOutput) + { + throw std::runtime_error("Failed while writing combined output: " + outputPath.string()); + } + } + }; try { - if (function.isStub || function.isSkipped) + scheduleAvailableWork(); + + while (nextOutputIndex < outputFunctions.size()) { - combinedOutput << m_generatedStubs.at(function.start) << "\n\n"; - } - else - { - const auto &instructions = m_decodedFunctions.at(function.start); - std::string code = m_codeGenerator->generateFunction(function, instructions, false); - combinedOutput << code << "\n\n"; + std::unique_lock lock(outputMutex); + resultAvailable.wait(lock, [&]() + { return workerException || !readyCode.empty() || (outstandingWork == 0 && nextFunction >= outputFunctions.size()); }); + + if (workerException) + { + lock.unlock(); + stopAndJoinWorkers(); + std::rethrow_exception(workerException); + } + + while (!readyCode.empty()) + { + CompletedCode completed = std::move(readyCode.front()); + readyCode.pop(); + completedCode.emplace(completed.outputIndex, std::move(completed.code)); + } + lock.unlock(); + + flushCompletedOutput(); + scheduleAvailableWork(); + + if (nextOutputIndex >= outputFunctions.size()) + { + break; + } + + std::lock_guard finalLock(outputMutex); + if (outstandingWork == 0 && nextFunction >= outputFunctions.size() && completedCode.empty()) + { + throw std::runtime_error("Internal error: combined output completion queue is missing index " + std::to_string(nextOutputIndex)); + } } + + stopAndJoinWorkers(); } - catch (const std::exception &e) + catch (...) { - std::cerr << "Error generating code for function " - << function.name << " (start 0x" - << std::hex << function.start << "): " - << e.what() << std::endl; + stopAndJoinWorkers(); throw; } } - fs::path outputPath = fs::path(m_config.outputPath) / "ps2_recompiled_functions.cpp"; - if (!writeToFile(outputPath.string(), combinedOutput.str())) + combinedOutput.close(); + if (!combinedOutput) { - throw std::runtime_error("Failed to write combined output: " + outputPath.string()); + throw std::runtime_error("Failed to finish combined output: " + outputPath.string()); } + std::cout << "Wrote recompiled to combined output to: " << outputPath << std::endl; } else { - for (const auto &function : m_functions) + struct GeneratedFile { - if (!shouldGenerateCodeForFunction(function)) + fs::path outputPath; + std::string code; + }; + + std::vector outputPaths; + outputPaths.reserve(outputFunctions.size()); + for (const Function *function : outputFunctions) + { + outputPaths.push_back(getOutputPath(*function)); + } + + auto generateFile = [&](CodeGenerator &codeGenerator, size_t outputIndex) -> GeneratedFile + { + return GeneratedFile{outputPaths[outputIndex], generateFunctionCode(codeGenerator, *outputFunctions[outputIndex], true)}; + }; + + auto writeGeneratedFile = [&](GeneratedFile generated) + { + fs::create_directories(generated.outputPath.parent_path()); + if (!writeToFile(generated.outputPath.string(), generated.code)) { - continue; + throw std::runtime_error("Failed to write function output: " + generated.outputPath.string()); + } + }; + + if (outputWorkerCount <= 1) + { + for (size_t outputIndex = 0; outputIndex < outputFunctions.size(); ++outputIndex) + { + writeGeneratedFile(generateFile(*m_codeGenerator, outputIndex)); + } + } + else + { + std::mutex outputMutex; + std::condition_variable workAvailable; + std::condition_variable resultAvailable; + std::queue pendingWork; + std::queue readyFiles; + std::exception_ptr workerException; + bool stopWorkers = false; + size_t outstandingWork = 0; + size_t nextFunction = 0; + const size_t maxBufferedOutput = std::max(outputWorkerCount * 2, outputWorkerCount + 1); + + auto workerMain = [&]() + { + CodeGenerator generator(*m_codeGenerator); + while (true) + { + size_t outputIndex = 0; + { + std::unique_lock lock(outputMutex); + workAvailable.wait(lock, [&]() + { return stopWorkers || !pendingWork.empty(); }); + if (stopWorkers) + { + return; + } + + outputIndex = pendingWork.front(); + pendingWork.pop(); + } + + try + { + GeneratedFile generated = generateFile(generator, outputIndex); + { + std::lock_guard lock(outputMutex); + readyFiles.push(std::move(generated)); + --outstandingWork; + } + resultAvailable.notify_one(); + } + catch (...) + { + { + std::lock_guard lock(outputMutex); + if (!workerException) + { + workerException = std::current_exception(); + } + --outstandingWork; + stopWorkers = true; + } + resultAvailable.notify_one(); + workAvailable.notify_all(); + return; + } + } + }; + + std::vector workers; + workers.reserve(outputWorkerCount); + for (size_t i = 0; i < outputWorkerCount; ++i) + { + workers.emplace_back(workerMain); } - std::string code; + auto stopAndJoinWorkers = [&]() + { + { + std::lock_guard lock(outputMutex); + stopWorkers = true; + } + workAvailable.notify_all(); + for (std::thread &worker : workers) + { + if (worker.joinable()) + { + worker.join(); + } + } + }; + + auto scheduleAvailableWork = [&]() + { + { + std::lock_guard lock(outputMutex); + while (nextFunction < outputFunctions.size() && + outstandingWork < maxBufferedOutput && + !stopWorkers) + { + pendingWork.push(nextFunction++); + ++outstandingWork; + } + } + workAvailable.notify_all(); + }; + try { - if (function.isStub || function.isSkipped) - { - std::stringstream stubFile; - stubFile << "#include \"ps2_runtime.h\"\n"; - stubFile << "#include \"ps2_syscalls.h\"\n"; - stubFile << "#include \"ps2_stubs.h\"\n"; - stubFile << "#ifdef _DEBUG\n"; - stubFile << "#include \"ps2_log.h\"\n"; - stubFile << "#endif\n"; - stubFile << "\n"; - stubFile << m_generatedStubs.at(function.start) << "\n"; - code = stubFile.str(); - } - else - { - const auto &instructions = m_decodedFunctions.at(function.start); - code = m_codeGenerator->generateFunction(function, instructions, true); - } - } - catch (const std::exception &e) - { - std::cerr << "Error generating code for function " - << function.name << " (start 0x" - << std::hex << function.start << "): " - << e.what() << std::endl; - throw; - } + scheduleAvailableWork(); - fs::path outputPath = getOutputPath(function); - fs::create_directories(outputPath.parent_path()); - if (!writeToFile(outputPath.string(), code)) + size_t writtenCount = 0; + while (writtenCount < outputFunctions.size()) + { + std::unique_lock lock(outputMutex); + resultAvailable.wait(lock, [&]() + { return workerException || !readyFiles.empty() || (outstandingWork == 0 && nextFunction >= outputFunctions.size()); }); + + if (workerException) + { + lock.unlock(); + stopAndJoinWorkers(); + std::rethrow_exception(workerException); + } + + if (readyFiles.empty()) + { + break; + } + + GeneratedFile generated = std::move(readyFiles.front()); + readyFiles.pop(); + lock.unlock(); + + writeGeneratedFile(std::move(generated)); + ++writtenCount; + scheduleAvailableWork(); + } + + stopAndJoinWorkers(); + } + catch (...) { - throw std::runtime_error("Failed to write function output: " + outputPath.string()); + stopAndJoinWorkers(); + throw; } } std::cout << "Wrote individual function files to: " << m_config.outputPath << std::endl; } + m_decodedFunctions.clear(); + std::string registerFunctions = m_codeGenerator->generateFunctionRegistration(m_functions, m_generatedStubs); + m_generatedStubs.clear(); fs::path registerPath = fs::path(m_config.outputPath) / "register_functions.cpp"; if (!writeToFile(registerPath.string(), registerFunctions)) @@ -1338,6 +1715,9 @@ namespace ps2recomp ownerTargets.insert(ownerTargets.end(), analysisResult.resumeEntryPoints.begin(), analysisResult.resumeEntryPoints.end()); + ownerTargets.insert(ownerTargets.end(), + analysisResult.indirectFallbackEntryPoints.begin(), + analysisResult.indirectFallbackEntryPoints.end()); for (uint32_t target : analysisResult.externalEntryPoints) { @@ -1428,7 +1808,7 @@ namespace ps2recomp } } - Instruction inst = m_decoder->decodeInstruction(address, rawInstruction); + Instruction inst = m_decoder->decodeInstruction(address, rawInstruction, !m_config.lowMemoryMode); auto mmioIt = m_config.mmioByInstructionAddress.find(address); if (mmioIt != m_config.mmioByInstructionAddress.end()) diff --git a/ps2xRecomp/src/lib/r5900_decoder.cpp b/ps2xRecomp/src/lib/r5900_decoder.cpp index 463bdda..9e627a3 100644 --- a/ps2xRecomp/src/lib/r5900_decoder.cpp +++ b/ps2xRecomp/src/lib/r5900_decoder.cpp @@ -1,9 +1,23 @@ #include "ps2recomp/r5900_decoder.h" #include "rabbitizer.h" #include +#include namespace ps2recomp { + static std::string disassembleRabbitizerInstruction(RabbitizerInstruction &rabbitizerInst) + { + std::string disassembly; + const size_t bufferSize = RabbitizerInstruction_getSizeForBuffer(&rabbitizerInst, 0, 0); + if (bufferSize > 0) + { + std::vector buffer(bufferSize + 1, '\0'); + RabbitizerInstruction_disassemble(&rabbitizerInst, buffer.data(), nullptr, 0, 0); + disassembly = buffer.data(); + } + + return disassembly; + } R5900Decoder::R5900Decoder() { @@ -13,7 +27,29 @@ namespace ps2recomp { } - Instruction R5900Decoder::decodeInstruction(uint32_t address, uint32_t rawInstruction) const + std::string R5900Decoder::disassembleInstruction(uint32_t address, uint32_t rawInstruction) + { + RabbitizerInstruction rabbitizerInst; + RabbitizerInstructionR5900_init(&rabbitizerInst, rawInstruction, address); + RabbitizerInstructionR5900_processUniqueId(&rabbitizerInst); + + std::string disassembly = disassembleRabbitizerInstruction(rabbitizerInst); + + RabbitizerInstructionR5900_destroy(&rabbitizerInst); + return disassembly; + } + + std::string R5900Decoder::disassembleInstruction(const Instruction &inst) + { + if (!inst.disassembly.empty()) + { + return inst.disassembly; + } + + return disassembleInstruction(inst.address, inst.raw); + } + + Instruction R5900Decoder::decodeInstruction(uint32_t address, uint32_t rawInstruction, bool includeDisassembly) const { Instruction inst; @@ -177,12 +213,9 @@ namespace ps2recomp inst.vectorInfo.isVector = inst.isVU; // Only VU ops are truly vector } - size_t bufferSize = RabbitizerInstruction_getSizeForBuffer(&rabbitizerInst, 0, 0); - if (bufferSize > 0) + if (includeDisassembly) { - std::vector buffer(bufferSize + 1, '\0'); - RabbitizerInstruction_disassemble(&rabbitizerInst, buffer.data(), nullptr, 0, 0); - inst.disassembly = buffer.data(); + inst.disassembly = disassembleRabbitizerInstruction(rabbitizerInst); } RabbitizerInstructionR5900_destroy(&rabbitizerInst); diff --git a/ps2xTest/src/code_generator_tests.cpp b/ps2xTest/src/code_generator_tests.cpp index 16f5a02..2ce6fde 100644 --- a/ps2xTest/src/code_generator_tests.cpp +++ b/ps2xTest/src/code_generator_tests.cpp @@ -305,6 +305,60 @@ void register_code_generator_tests() "JALR resume pc should also be emitted as an internal label"); }); + tc.Run("unresolved JR marks internal labels as indirect fallback resume entries", [](TestCase &t) { + Function func; + func.name = "unresolved_jr_fallback"; + func.start = 0x3100; + func.end = 0x3120; + func.isRecompiled = true; + func.isStub = false; + + std::vector instructions{ + makeNop(0x3100), + makeJr(0x3104, 8), + makeNop(0x3108), + makeNop(0x310C), + makeNop(0x3110), + }; + + CodeGenerator gen({}, {}); + CodeGenerator::AnalysisResult analysis = gen.collectInternalBranchTargets(func, instructions); + + t.IsTrue(analysis.indirectFallbackEntryPoints.contains(0x310Cu), + "unresolved JR should register internal labels as resumable entries for the owning function"); + t.IsTrue(analysis.entryPoints.contains(0x310Cu), + "unresolved JR fallback targets should still emit labels in the owner"); + t.IsFalse(analysis.jumpTableTargets.contains(0x3104u), + "unresolved JR should not pretend it has a resolved local jump table"); + }); + + tc.Run("unresolved JALR marks internal labels as indirect fallback resume entries", [](TestCase &t) { + Function func; + func.name = "unresolved_jalr_fallback"; + func.start = 0x3200; + func.end = 0x3220; + func.isRecompiled = true; + func.isStub = false; + + std::vector instructions{ + makeNop(0x3200), + makeJalr(0x3204, 25, 31), + makeNop(0x3208), + makeNop(0x320C), + makeNop(0x3210), + }; + + CodeGenerator gen({}, {}); + CodeGenerator::AnalysisResult analysis = gen.collectInternalBranchTargets(func, instructions); + + t.IsTrue(analysis.indirectFallbackEntryPoints.contains(0x320Cu), + "unresolved JALR should register internal labels as resumable entries for the owning function"); + t.IsTrue(analysis.entryPoints.contains(0x320Cu), + "unresolved JALR fallback targets should still emit labels in the owner"); + t.IsFalse(analysis.jumpTableTargets.contains(0x3204u), + "unresolved JALR should not pretend it has a resolved local jump table"); + }); + tc.Run("resume entry targets emit a top-level pc switch in the owner wrapper", [](TestCase &t) { Function func; func.name = "resume_owner"; @@ -1142,9 +1196,9 @@ void register_code_generator_tests() t.IsTrue(generated.find("if (branch_taken_0x1200)") != std::string::npos, "should generate branch_taken variable and if for likely branch"); }); - tc.Run("JR $31 emits switch for internal return targets", [](TestCase &t) { + tc.Run("JR $31 returns through dynamic target without broad local switch", [](TestCase &t) { Function func; - func.name = "jr_ra_switch"; + func.name = "jr_ra_return"; func.start = 0x1300; func.end = 0x1340; func.isRecompiled = true; @@ -1162,10 +1216,16 @@ void register_code_generator_tests() CodeGenerator gen({}, {}); std::string generated = gen.generateFunction(func, { jal, jalDelay, atReturn, atTarget, jr, jrDelay }, false); - printGeneratedCode("JR $31 emits switch for internal return targets", generated); + printGeneratedCode("JR $31 returns through dynamic target without broad local switch", generated); - t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos, "JR $31 should emit switch for internal targets"); - t.IsTrue(generated.find("case 0x1308u: goto label_1308;") != std::string::npos, "switch should include return address from internal JAL"); + t.IsTrue(generated.find("uint32_t jumpTarget = GPR_U32(ctx, 31);") != std::string::npos, + "JR $31 should still read the dynamic return target"); + t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos, + "JR $31 should not emit a broad local switch over internal labels"); + t.IsTrue(generated.find("label_1308:") != std::string::npos, + "internal JAL return address should still be emitted as a label"); + t.IsTrue(generated.find(" return;") != std::string::npos, + "JR $31 should return to the dispatcher/runtime after setting ctx->pc"); }); tc.Run("trailing JR $31 without decoded delay slot still emits return flow", [](TestCase &t) { @@ -1188,17 +1248,17 @@ void register_code_generator_tests() t.IsTrue(generated.find("uint32_t jumpTarget = GPR_U32(ctx, 31);") != std::string::npos, "truncated trailing JR should still read the return target"); - t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos, - "truncated trailing JR should still emit the return-target switch"); - t.IsTrue(generated.find("case 0x1508u: goto label_1508;") != std::string::npos, - "truncated trailing JR should still include internal return targets"); + t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos, + "truncated trailing JR should not emit a broad local return-target switch"); + t.IsTrue(generated.find("label_1508:") != std::string::npos, + "truncated trailing JR should still include the internal return label"); t.IsTrue(generated.find("// JR $31 - Handled by branch logic") == std::string::npos, "truncated trailing JR must not degrade to comment-only output"); }); - tc.Run("JR non-RA emits switch for in-function jump targets", [](TestCase &t) { + tc.Run("unresolved JR non-RA uses dispatcher resume entries without broad local switch", [](TestCase &t) { Function func; - func.name = "jr_non_ra_switch"; + func.name = "jr_non_ra_dispatcher_resume"; func.start = 0x1400; func.end = 0x1420; func.isRecompiled = true; @@ -1214,17 +1274,21 @@ void register_code_generator_tests() Instruction i3 = makeNop(0x140c); CodeGenerator gen({}, {}); + CodeGenerator::AnalysisResult analysis = gen.collectInternalBranchTargets(func, {i0, jr, delay, i3}); + gen.setResumeEntryTargets({{func.start, std::vector( + analysis.indirectFallbackEntryPoints.begin(), + analysis.indirectFallbackEntryPoints.end())}}); std::string generated = gen.generateFunction(func, {i0, jr, delay, i3}, false); - printGeneratedCode("JR non-RA emits switch for in-function jump targets", generated); + printGeneratedCode("unresolved JR non-RA uses dispatcher resume entries without broad local switch", generated); - t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos, - "JR via non-RA register should emit switch for internal targets"); - t.IsTrue(generated.find("switch (ctx->pc)") == std::string::npos, - "JR fallback labels should not be promoted to dispatcher resume sites"); - t.IsTrue(generated.find("case 0x1400u: goto label_1400;") != std::string::npos, - "switch should include in-function entry label"); - t.IsTrue(generated.find("case 0x140Cu: goto label_140c;") != std::string::npos, - "switch should include other in-function labels"); + t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos, + "unresolved JR via non-RA register should not emit a broad local switch over internal labels"); + t.IsTrue(generated.find("switch (ctx->pc)") != std::string::npos, + "JR fallback labels should be promoted to dispatcher resume sites"); + t.IsTrue(generated.find("case 0x140cu: goto label_140c;") != std::string::npos, + "owner resume switch should include internal fallback labels"); + t.IsTrue(generated.find("ctx->pc = jumpTarget;") != std::string::npos, + "unresolved JR should hand the dynamic target back through ctx->pc"); }); tc.Run("configured jump table addresses drive JR dispatch targets", [](TestCase &t) { @@ -1291,6 +1355,8 @@ void register_code_generator_tests() func, {lui, addiu, sll, addu, lw, jr, jrDelay, target0, target1}); + t.IsTrue(analysis.jumpTableTargets.contains(0x1614u), + "configured JR table should be tracked as a resolved local jump table"); t.IsFalse(analysis.resumeEntryPoints.contains(0x1620u), "configured JR table targets should stay in-function dispatch labels"); t.IsFalse(analysis.resumeEntryPoints.contains(0x1630u), @@ -1314,15 +1380,15 @@ void register_code_generator_tests() "configured table should avoid broad JR fallback labels"); }); - tc.Run("JALR includes switch and fallback/guard pair", [](TestCase &t) { + tc.Run("unresolved JALR uses runtime dispatch without broad local switch", [](TestCase &t) { Function func; - func.name = "jalr_switch_and_fallback"; + func.name = "jalr_runtime_fallback"; func.start = 0x1500; func.end = 0x1530; func.isRecompiled = true; func.isStub = false; - // A call-like setup so there are multiple in-function labels to dispatch to. + // A call-like setup so there are multiple in-function labels available. Instruction jal = makeJal(0x1500, 0x1510); Instruction jalDelay = makeNop(0x1504); Instruction atReturn = makeNop(0x1508); @@ -1332,12 +1398,12 @@ void register_code_generator_tests() CodeGenerator gen({}, {}); std::string generated = gen.generateFunction(func, {jal, jalDelay, atReturn, atTarget, jalr, jalrDelay}, false); - printGeneratedCode("JALR includes switch and fallback/guard pair", generated); + printGeneratedCode("unresolved JALR uses runtime dispatch without broad local switch", generated); - t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos, - "JALR should emit switch when in-function register-jump targets exist"); - t.IsTrue(generated.find("case 0x1508u: goto label_1508;") != std::string::npos, - "switch should include internal return label from JAL in same function"); + t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos, + "unresolved JALR should not emit a broad local switch over every internal label"); + t.IsTrue(generated.find("auto targetFn = runtime->lookupFunction(jumpTarget);") != std::string::npos, + "unresolved JALR should dispatch through the runtime"); t.IsTrue(generated.find("if (ctx->pc == __entryPc) { ctx->pc = 0x151Cu; }") != std::string::npos, "JALR should contain unchanged-PC fallback to fallthrough"); t.IsTrue(generated.find("if (ctx->pc != 0x151Cu) { return; }") != std::string::npos,