Reduce recompiler output memory usage, and added multi-threading to recomp process (#128)

* feat(recomp): Reduce recompiler output memory usage

Stream output generation, add low-memory config controls, and avoid
pathological indirect-jump switch expansion in generated C++.

Low-memory mode now avoids retaining per-instruction disassembly strings
while still emitting asm comments during output generation. Output workers
are bounded/configurable, combined output is streamed, and decoded buffers
are released after generation.

Also document the new output memory settings.

* fix(recomp): added tests for unregistered JR/JALR, updated fallback logic to cover JR/JALR, moved Rabbitizer formatting into R5900Decoder
This commit is contained in:
Sinan
2026-06-06 05:15:14 +02:00
committed by GitHub
parent 93e221feaa
commit ed8b3ebee1
10 changed files with 659 additions and 126 deletions
+4
View File
@@ -92,6 +92,8 @@ Main fields in `config.toml`:
* `general.ghidra_output`: recommended function map CSV exported from Ghidra.
* `general.output`: generated C++ output folder.
* `general.single_file_output`: one combined cpp or one file per function.
* `general.low_memory_mode`: reduce peak output-generation memory by avoiding retained disassembly strings and forcing serial output generation. Generated instruction comments are still emitted; disassembly text is produced while writing each output file instead of being kept in memory.
* `general.output_worker_threads`: number of output-generation workers (clamped to nproc * 2). A positive value uses exactly that many workers. `0` uses `nproc - 1` when at least two hardware threads are available, otherwise serial output generation. `1` forces serial output generation.
* `general.patch_syscalls`: apply configured patches to `SYSCALL` instructions (`false` recommended).
* `general.patch_cop0`: apply configured patches to COP0 instructions.
* `general.patch_cache`: apply configured patches to CACHE instructions.
@@ -118,6 +120,8 @@ ghidra_output = ""
output = "output/"
single_file_output = true
low_memory_mode = true
output_worker_threads = 0
patch_syscalls = false
patch_cop0 = true
patch_cache = true
+7
View File
@@ -8,6 +8,13 @@ output = "output/"
# Single file output mode (false for one file per function)
single_file_output = false
# Lower peak memory by avoiding retained disassembly strings and forcing serial output generation.
low_memory_mode = false
# Function generation workers. 0 uses nproc - 1 when at least 2 hardware threads are available; 1 disables parallel generation.
# Limited to nproc * 2 to avoid oversubscription.
output_worker_threads = 0
# Path to runtime header (optional)
runtime_header = "include/ps2_runtime.h"
@@ -39,6 +39,7 @@ namespace ps2recomp
std::unordered_set<uint32_t> entryPoints;
std::unordered_set<uint32_t> externalEntryPoints;
std::unordered_set<uint32_t> resumeEntryPoints;
std::unordered_set<uint32_t> indirectFallbackEntryPoints;
std::unordered_map<uint32_t, std::vector<uint32_t>> jumpTableTargets;
};
@@ -52,6 +53,7 @@ namespace ps2recomp
void setRelocationCallNames(const std::unordered_map<uint32_t, std::string> &callNames);
void setConfiguredJumpTables(const std::vector<JumpTable> &jumpTables);
void setResumeEntryTargets(const std::unordered_map<uint32_t, std::vector<uint32_t>> &resumeTargetsByOwner);
void setEmitInstructionComments(bool emitInstructionComments);
AnalysisResult collectInternalBranchTargets(const Function &function,
const std::vector<Instruction> &instructions,
@@ -65,6 +67,7 @@ namespace ps2recomp
std::unordered_map<uint32_t, std::vector<uint32_t>> m_resumeEntryTargetsByOwner;
const std::vector<Section>& m_sections;
BootstrapInfo m_bootstrapInfo;
bool m_emitInstructionComments = true;
std::string translateInstruction(const Instruction &inst);
std::string translateMMIInstruction(const Instruction &inst);
+6 -2
View File
@@ -4,6 +4,7 @@
#include "ps2recomp/types.h"
#include "ps2recomp/instructions.h"
#include <cstdint>
#include <string>
namespace ps2recomp
{
@@ -14,7 +15,10 @@ namespace ps2recomp
R5900Decoder();
~R5900Decoder();
Instruction decodeInstruction(uint32_t address, uint32_t rawInstruction) const;
static std::string disassembleInstruction(uint32_t address, uint32_t rawInstruction);
static std::string disassembleInstruction(const Instruction &inst);
Instruction decodeInstruction(uint32_t address, uint32_t rawInstruction, bool includeDisassembly = true) const;
bool isBranchInstruction(const Instruction &inst) const;
bool isJumpInstruction(const Instruction &inst) const;
@@ -52,4 +56,4 @@ namespace ps2recomp
} // namespace ps2recomp
#endif // PS2RECOMP_R5900_DECODER_H
#endif // PS2RECOMP_R5900_DECODER_H
+2
View File
@@ -173,6 +173,8 @@ namespace ps2recomp
std::string outputPath;
std::string ghidraMapPath;
bool singleFileOutput = false;
bool lowMemoryMode = false;
uint32_t outputWorkerThreads = 0;
bool patchSyscalls = false;
bool patchCop0 = true;
bool patchCache = true;
+36 -22
View File
@@ -1,6 +1,7 @@
#include "ps2recomp/code_generator.h"
#include "ps2recomp/instructions.h"
#include "ps2recomp/ps2_recompiler.h"
#include "ps2recomp/r5900_decoder.h"
#include "ps2recomp/types.h"
#include "ps2_runtime_calls.h"
#include <fmt/format.h>
@@ -12,6 +13,7 @@
#include <iostream>
#include <cctype>
#include <cmath>
#include <vector>
namespace ps2recomp
{
@@ -164,6 +166,11 @@ namespace ps2recomp
}
}
void CodeGenerator::setEmitInstructionComments(bool emitInstructionComments)
{
m_emitInstructionComments = emitInstructionComments;
}
std::string CodeGenerator::getFunctionName(uint32_t address) const
{
auto it = m_renamedFunctions.find(address);
@@ -223,11 +230,16 @@ namespace ps2recomp
std::string delaySlotSuffix = "";
if (hasValidDelaySlot) {
delaySlotPrefix = "ctx->in_delay_slot = true; ctx->branch_pc = 0x" + fmt::format("{:X}", branchInst.address) + "u;\n ";
delaySlotCode = " // 0x" + fmt::format("{:x}", delaySlot.address) + ": 0x" + fmt::format("{:x}", delaySlot.raw);
if (!delaySlot.disassembly.empty()) {
delaySlotCode += " " + delaySlot.disassembly;
if (m_emitInstructionComments)
{
delaySlotCode = " // 0x" + fmt::format("{:x}", delaySlot.address) + ": 0x" + fmt::format("{:x}", delaySlot.raw);
std::string disassembly = R5900Decoder::disassembleInstruction(delaySlot);
if (!disassembly.empty()) {
delaySlotCode += " " + disassembly;
}
delaySlotCode += " (Delay Slot)\n ";
}
delaySlotCode += " (Delay Slot)\n " + translateInstruction(delaySlot);
delaySlotCode += translateInstruction(delaySlot);
delaySlotSuffix = "\n ctx->in_delay_slot = false;";
}
@@ -259,20 +271,18 @@ namespace ps2recomp
std::vector<uint32_t> sortedInternalTargets;
if (branchInst.opcode == OPCODE_SPECIAL &&
(branchInst.function == SPECIAL_JR || branchInst.function == SPECIAL_JALR) &&
((branchInst.function == SPECIAL_JR && branchInst.rs != 31) ||
branchInst.function == SPECIAL_JALR) &&
!internalTargets.empty())
{
// Only emit local indirect-jump switches for jump tables we actually resolved.
// Falling back to every internal target here can duplicate huge switches at each
// indirect branch. Unresolved JR/JALR targets are registered as resumable entries
// instead, so runtime dispatch can re-enter this function at ctx->pc.
auto jtIt = analysisResult.jumpTableTargets.find(branchInst.address);
if (jtIt != analysisResult.jumpTableTargets.end()) {
sortedInternalTargets = jtIt->second;
std::sort(sortedInternalTargets.begin(), sortedInternalTargets.end());
} else {
sortedInternalTargets.reserve(internalTargets.size());
for (uint32_t t : internalTargets)
{
sortedInternalTargets.push_back(t);
}
std::sort(sortedInternalTargets.begin(), sortedInternalTargets.end());
}
}
@@ -827,7 +837,7 @@ namespace ps2recomp
if (hasIndirectRegisterJump)
{
bool needsJrFallback = false;
bool needsIndirectFallback = false;
for (const Instruction* jrInst : indirectJumps) {
if (jrInst->function == SPECIAL_JALR)
{
@@ -997,19 +1007,19 @@ namespace ps2recomp
}
}
if (!foundTable) {
if (!(jrInst->function == SPECIAL_JALR))
{
needsJrFallback = true;
}
needsIndirectFallback = true;
}
}
if (needsJrFallback) {
if (needsIndirectFallback) {
for (uint32_t addr : instructionAddresses)
{
if (addr >= function.start && addr < function.end)
{
result.entryPoints.insert(addr);
// Keep labels and runtime registration for unresolved JR/JALR targets
// without emitting a local switch over every possible target.
result.indirectFallbackEntryPoints.insert(addr);
}
}
}
@@ -1093,11 +1103,15 @@ namespace ps2recomp
ss << "label_" << std::hex << inst.address << std::dec << ":\n";
}
ss << " // 0x" << std::hex << inst.address << ": 0x" << inst.raw << std::dec;
if (!inst.disassembly.empty()) {
ss << " " << inst.disassembly;
if (m_emitInstructionComments)
{
ss << " // 0x" << std::hex << inst.address << ": 0x" << inst.raw << std::dec;
std::string disassembly = R5900Decoder::disassembleInstruction(inst);
if (!disassembly.empty()) {
ss << " " << disassembly;
}
ss << "\n";
}
ss << "\n";
try
{
+20
View File
@@ -4,6 +4,9 @@
#include <iostream>
#include <stdexcept>
#include <sstream>
#include <algorithm>
#include <limits>
#include <thread>
namespace ps2recomp
{
@@ -29,6 +32,21 @@ namespace ps2recomp
config.ghidraMapPath = toml::find_or<std::string>(general, "ghidra_output", "");
config.outputPath = toml::find<std::string>(general, "output");
config.singleFileOutput = toml::find_or<bool>(general, "single_file_output", false);
config.lowMemoryMode = toml::find_or<bool>(general, "low_memory_mode", config.lowMemoryMode);
const int64_t configuredOutputWorkers = toml::find_or<int64_t>(
general,
"output_worker_threads",
toml::find_or<int64_t>(general, "output_worker_thread", config.outputWorkerThreads));
const int64_t clampedOutputWorkers = std::clamp<int64_t>(
configuredOutputWorkers,
0,
std::thread::hardware_concurrency() * 2);
if (configuredOutputWorkers != clampedOutputWorkers)
{
std::cerr << "Warning: output_worker_threads value " << configuredOutputWorkers
<< " is out of range; clamped to " << clampedOutputWorkers << "." << std::endl;
}
config.outputWorkerThreads = static_cast<uint32_t>(clampedOutputWorkers);
config.patchSyscalls = toml::find_or<bool>(general, "patch_syscalls", config.patchSyscalls);
config.patchCop0 = toml::find_or<bool>(general, "patch_cop0", config.patchCop0);
config.patchCache = toml::find_or<bool>(general, "patch_cache", config.patchCache);
@@ -234,6 +252,8 @@ namespace ps2recomp
general["ghidra_output"] = config.ghidraMapPath;
general["output"] = config.outputPath;
general["single_file_output"] = config.singleFileOutput;
general["low_memory_mode"] = config.lowMemoryMode;
general["output_worker_threads"] = static_cast<int64_t>(config.outputWorkerThreads);
general["patch_syscalls"] = config.patchSyscalls;
general["patch_cop0"] = config.patchCop0;
general["patch_cache"] = config.patchCache;
+448 -68
View File
@@ -11,10 +11,15 @@
#include <stdexcept>
#include <filesystem>
#include <cctype>
#include <condition_variable>
#include <exception>
#include <mutex>
#include <queue>
#include <unordered_set>
#include <optional>
#include <limits>
#include <functional>
#include <thread>
namespace fs = std::filesystem;
@@ -77,6 +82,36 @@ namespace ps2recomp
return function.isRecompiled || function.isStub || function.isSkipped;
}
size_t resolveOutputWorkerCount(uint32_t configuredWorkerCount)
{
if (configuredWorkerCount > 0)
{
return configuredWorkerCount;
}
const unsigned int hardwareWorkers = std::thread::hardware_concurrency();
if (hardwareWorkers >= 2)
{
return static_cast<size_t>(hardwareWorkers - 1);
}
return 1;
}
void writeCombinedOutputPreamble(std::ostream &output)
{
output << "#include \"ps2_recompiled_functions.h\"\n\n";
output << "#include \"ps2_runtime_macros.h\"\n";
output << "#include \"ps2_runtime.h\"\n";
output << "#include \"ps2_recompiled_stubs.h\"\n";
output << "#include \"ps2_syscalls.h\"\n";
output << "#include \"ps2_stubs.h\"\n";
output << "#ifdef _DEBUG\n";
output << "#include \"ps2_log.h\"\n";
output << "#endif\n";
output << "\n";
}
enum class PatchClass
{
Generic,
@@ -850,6 +885,7 @@ namespace ps2recomp
m_codeGenerator->setRelocationCallNames(relocationCallNames);
m_codeGenerator->setBootstrapInfo(m_bootstrapInfo);
m_codeGenerator->setConfiguredJumpTables(m_config.jumpTables);
m_codeGenerator->setEmitInstructionComments(true);
fs::create_directories(m_config.outputPath);
@@ -1050,110 +1086,451 @@ namespace ps2recomp
generateFunctionHeader();
std::vector<const Function *> outputFunctions;
outputFunctions.reserve(m_functions.size());
for (const auto &function : m_functions)
{
if (shouldGenerateCodeForFunction(function))
{
outputFunctions.push_back(&function);
}
}
const size_t outputWorkerCount = m_config.lowMemoryMode ? 1 : resolveOutputWorkerCount(m_config.outputWorkerThreads);
if (outputFunctions.size() > 1 && outputWorkerCount > 1)
{
std::cout << "Generating function output with " << outputWorkerCount << " worker(s)." << std::endl;
}
const auto &generatedStubs = m_generatedStubs;
const auto &decodedFunctions = m_decodedFunctions;
auto generateFunctionCode = [&](CodeGenerator &codeGenerator, const Function &function, bool useHeaders) -> std::string
{
try
{
if (function.isStub || function.isSkipped)
{
if (!useHeaders)
{
return generatedStubs.at(function.start);
}
std::stringstream stubFile;
stubFile << "#include \"ps2_runtime.h\"\n";
stubFile << "#include \"ps2_syscalls.h\"\n";
stubFile << "#include \"ps2_stubs.h\"\n";
stubFile << "#ifdef _DEBUG\n";
stubFile << "#include \"ps2_log.h\"\n";
stubFile << "#endif\n";
stubFile << "\n";
stubFile << generatedStubs.at(function.start) << "\n";
return stubFile.str();
}
const auto &instructions = decodedFunctions.at(function.start);
return codeGenerator.generateFunction(function, instructions, useHeaders);
}
catch (const std::exception &e)
{
std::cerr << "Error generating code for function "
<< function.name << " (start 0x"
<< std::hex << function.start << std::dec << "): "
<< e.what() << std::endl;
throw;
}
};
if (m_config.singleFileOutput)
{
std::stringstream combinedOutput;
combinedOutput << "#include \"ps2_recompiled_functions.h\"\n\n";
combinedOutput << "#include \"ps2_runtime_macros.h\"\n";
combinedOutput << "#include \"ps2_runtime.h\"\n";
combinedOutput << "#include \"ps2_recompiled_stubs.h\"\n";
combinedOutput << "#include \"ps2_syscalls.h\"\n";
combinedOutput << "#include \"ps2_stubs.h\"\n";
combinedOutput << "#ifdef _DEBUG\n";
combinedOutput << "#include \"ps2_log.h\"\n";
combinedOutput << "#endif\n";
combinedOutput << "\n";
for (const auto &function : m_functions)
fs::path outputPath = fs::path(m_config.outputPath) / "ps2_recompiled_functions.cpp";
std::ofstream combinedOutput(outputPath);
if (!combinedOutput)
{
if (!shouldGenerateCodeForFunction(function))
throw std::runtime_error("Failed to open combined output: " + outputPath.string());
}
writeCombinedOutputPreamble(combinedOutput);
if (outputWorkerCount <= 1)
{
for (const Function *function : outputFunctions)
{
continue;
combinedOutput << generateFunctionCode(*m_codeGenerator, *function, false) << "\n\n";
}
}
else
{
struct CompletedCode
{
size_t outputIndex = 0;
std::string code;
};
std::mutex outputMutex;
std::condition_variable workAvailable;
std::condition_variable resultAvailable;
std::queue<size_t> pendingWork;
std::queue<CompletedCode> readyCode;
std::unordered_map<size_t, std::string> completedCode;
std::exception_ptr workerException;
bool stopWorkers = false;
size_t outstandingWork = 0;
size_t nextFunction = 0;
size_t nextOutputIndex = 0;
const size_t maxBufferedOutput = std::max<size_t>(outputWorkerCount * 2, outputWorkerCount + 1);
auto workerMain = [&]()
{
CodeGenerator generator(*m_codeGenerator);
while (true)
{
size_t outputIndex = 0;
{
std::unique_lock<std::mutex> lock(outputMutex);
workAvailable.wait(lock, [&]()
{ return stopWorkers || !pendingWork.empty(); });
if (stopWorkers)
{
return;
}
outputIndex = pendingWork.front();
pendingWork.pop();
}
try
{
std::string code = generateFunctionCode(generator, *outputFunctions[outputIndex], false);
{
std::lock_guard<std::mutex> lock(outputMutex);
readyCode.push(CompletedCode{outputIndex, std::move(code)});
--outstandingWork;
}
resultAvailable.notify_one();
}
catch (...)
{
{
std::lock_guard<std::mutex> lock(outputMutex);
if (!workerException)
{
workerException = std::current_exception();
}
--outstandingWork;
stopWorkers = true;
}
resultAvailable.notify_one();
workAvailable.notify_all();
return;
}
}
};
std::vector<std::thread> workers;
workers.reserve(outputWorkerCount);
for (size_t i = 0; i < outputWorkerCount; ++i)
{
workers.emplace_back(workerMain);
}
auto stopAndJoinWorkers = [&]()
{
{
std::lock_guard<std::mutex> lock(outputMutex);
stopWorkers = true;
}
workAvailable.notify_all();
for (std::thread &worker : workers)
{
if (worker.joinable())
{
worker.join();
}
}
};
auto scheduleAvailableWork = [&]()
{
bool scheduledAny = false;
{
std::lock_guard<std::mutex> lock(outputMutex);
while (nextFunction < outputFunctions.size() &&
outstandingWork + completedCode.size() < maxBufferedOutput &&
!stopWorkers)
{
pendingWork.push(nextFunction++);
++outstandingWork;
scheduledAny = true;
}
}
if (scheduledAny)
{
workAvailable.notify_all();
}
};
auto flushCompletedOutput = [&]()
{
while (true)
{
auto completedIt = completedCode.find(nextOutputIndex);
if (completedIt == completedCode.end())
{
break;
}
combinedOutput << completedIt->second << "\n\n";
completedCode.erase(completedIt);
++nextOutputIndex;
if (!combinedOutput)
{
throw std::runtime_error("Failed while writing combined output: " + outputPath.string());
}
}
};
try
{
if (function.isStub || function.isSkipped)
scheduleAvailableWork();
while (nextOutputIndex < outputFunctions.size())
{
combinedOutput << m_generatedStubs.at(function.start) << "\n\n";
}
else
{
const auto &instructions = m_decodedFunctions.at(function.start);
std::string code = m_codeGenerator->generateFunction(function, instructions, false);
combinedOutput << code << "\n\n";
std::unique_lock<std::mutex> lock(outputMutex);
resultAvailable.wait(lock, [&]()
{ return workerException || !readyCode.empty() || (outstandingWork == 0 && nextFunction >= outputFunctions.size()); });
if (workerException)
{
lock.unlock();
stopAndJoinWorkers();
std::rethrow_exception(workerException);
}
while (!readyCode.empty())
{
CompletedCode completed = std::move(readyCode.front());
readyCode.pop();
completedCode.emplace(completed.outputIndex, std::move(completed.code));
}
lock.unlock();
flushCompletedOutput();
scheduleAvailableWork();
if (nextOutputIndex >= outputFunctions.size())
{
break;
}
std::lock_guard<std::mutex> finalLock(outputMutex);
if (outstandingWork == 0 && nextFunction >= outputFunctions.size() && completedCode.empty())
{
throw std::runtime_error("Internal error: combined output completion queue is missing index " + std::to_string(nextOutputIndex));
}
}
stopAndJoinWorkers();
}
catch (const std::exception &e)
catch (...)
{
std::cerr << "Error generating code for function "
<< function.name << " (start 0x"
<< std::hex << function.start << "): "
<< e.what() << std::endl;
stopAndJoinWorkers();
throw;
}
}
fs::path outputPath = fs::path(m_config.outputPath) / "ps2_recompiled_functions.cpp";
if (!writeToFile(outputPath.string(), combinedOutput.str()))
combinedOutput.close();
if (!combinedOutput)
{
throw std::runtime_error("Failed to write combined output: " + outputPath.string());
throw std::runtime_error("Failed to finish combined output: " + outputPath.string());
}
std::cout << "Wrote recompiled to combined output to: " << outputPath << std::endl;
}
else
{
for (const auto &function : m_functions)
struct GeneratedFile
{
if (!shouldGenerateCodeForFunction(function))
fs::path outputPath;
std::string code;
};
std::vector<fs::path> outputPaths;
outputPaths.reserve(outputFunctions.size());
for (const Function *function : outputFunctions)
{
outputPaths.push_back(getOutputPath(*function));
}
auto generateFile = [&](CodeGenerator &codeGenerator, size_t outputIndex) -> GeneratedFile
{
return GeneratedFile{outputPaths[outputIndex], generateFunctionCode(codeGenerator, *outputFunctions[outputIndex], true)};
};
auto writeGeneratedFile = [&](GeneratedFile generated)
{
fs::create_directories(generated.outputPath.parent_path());
if (!writeToFile(generated.outputPath.string(), generated.code))
{
continue;
throw std::runtime_error("Failed to write function output: " + generated.outputPath.string());
}
};
if (outputWorkerCount <= 1)
{
for (size_t outputIndex = 0; outputIndex < outputFunctions.size(); ++outputIndex)
{
writeGeneratedFile(generateFile(*m_codeGenerator, outputIndex));
}
}
else
{
std::mutex outputMutex;
std::condition_variable workAvailable;
std::condition_variable resultAvailable;
std::queue<size_t> pendingWork;
std::queue<GeneratedFile> readyFiles;
std::exception_ptr workerException;
bool stopWorkers = false;
size_t outstandingWork = 0;
size_t nextFunction = 0;
const size_t maxBufferedOutput = std::max<size_t>(outputWorkerCount * 2, outputWorkerCount + 1);
auto workerMain = [&]()
{
CodeGenerator generator(*m_codeGenerator);
while (true)
{
size_t outputIndex = 0;
{
std::unique_lock<std::mutex> lock(outputMutex);
workAvailable.wait(lock, [&]()
{ return stopWorkers || !pendingWork.empty(); });
if (stopWorkers)
{
return;
}
outputIndex = pendingWork.front();
pendingWork.pop();
}
try
{
GeneratedFile generated = generateFile(generator, outputIndex);
{
std::lock_guard<std::mutex> lock(outputMutex);
readyFiles.push(std::move(generated));
--outstandingWork;
}
resultAvailable.notify_one();
}
catch (...)
{
{
std::lock_guard<std::mutex> lock(outputMutex);
if (!workerException)
{
workerException = std::current_exception();
}
--outstandingWork;
stopWorkers = true;
}
resultAvailable.notify_one();
workAvailable.notify_all();
return;
}
}
};
std::vector<std::thread> workers;
workers.reserve(outputWorkerCount);
for (size_t i = 0; i < outputWorkerCount; ++i)
{
workers.emplace_back(workerMain);
}
std::string code;
auto stopAndJoinWorkers = [&]()
{
{
std::lock_guard<std::mutex> lock(outputMutex);
stopWorkers = true;
}
workAvailable.notify_all();
for (std::thread &worker : workers)
{
if (worker.joinable())
{
worker.join();
}
}
};
auto scheduleAvailableWork = [&]()
{
{
std::lock_guard<std::mutex> lock(outputMutex);
while (nextFunction < outputFunctions.size() &&
outstandingWork < maxBufferedOutput &&
!stopWorkers)
{
pendingWork.push(nextFunction++);
++outstandingWork;
}
}
workAvailable.notify_all();
};
try
{
if (function.isStub || function.isSkipped)
{
std::stringstream stubFile;
stubFile << "#include \"ps2_runtime.h\"\n";
stubFile << "#include \"ps2_syscalls.h\"\n";
stubFile << "#include \"ps2_stubs.h\"\n";
stubFile << "#ifdef _DEBUG\n";
stubFile << "#include \"ps2_log.h\"\n";
stubFile << "#endif\n";
stubFile << "\n";
stubFile << m_generatedStubs.at(function.start) << "\n";
code = stubFile.str();
}
else
{
const auto &instructions = m_decodedFunctions.at(function.start);
code = m_codeGenerator->generateFunction(function, instructions, true);
}
}
catch (const std::exception &e)
{
std::cerr << "Error generating code for function "
<< function.name << " (start 0x"
<< std::hex << function.start << "): "
<< e.what() << std::endl;
throw;
}
scheduleAvailableWork();
fs::path outputPath = getOutputPath(function);
fs::create_directories(outputPath.parent_path());
if (!writeToFile(outputPath.string(), code))
size_t writtenCount = 0;
while (writtenCount < outputFunctions.size())
{
std::unique_lock<std::mutex> lock(outputMutex);
resultAvailable.wait(lock, [&]()
{ return workerException || !readyFiles.empty() || (outstandingWork == 0 && nextFunction >= outputFunctions.size()); });
if (workerException)
{
lock.unlock();
stopAndJoinWorkers();
std::rethrow_exception(workerException);
}
if (readyFiles.empty())
{
break;
}
GeneratedFile generated = std::move(readyFiles.front());
readyFiles.pop();
lock.unlock();
writeGeneratedFile(std::move(generated));
++writtenCount;
scheduleAvailableWork();
}
stopAndJoinWorkers();
}
catch (...)
{
throw std::runtime_error("Failed to write function output: " + outputPath.string());
stopAndJoinWorkers();
throw;
}
}
std::cout << "Wrote individual function files to: " << m_config.outputPath << std::endl;
}
m_decodedFunctions.clear();
std::string registerFunctions = m_codeGenerator->generateFunctionRegistration(m_functions, m_generatedStubs);
m_generatedStubs.clear();
fs::path registerPath = fs::path(m_config.outputPath) / "register_functions.cpp";
if (!writeToFile(registerPath.string(), registerFunctions))
@@ -1338,6 +1715,9 @@ namespace ps2recomp
ownerTargets.insert(ownerTargets.end(),
analysisResult.resumeEntryPoints.begin(),
analysisResult.resumeEntryPoints.end());
ownerTargets.insert(ownerTargets.end(),
analysisResult.indirectFallbackEntryPoints.begin(),
analysisResult.indirectFallbackEntryPoints.end());
for (uint32_t target : analysisResult.externalEntryPoints)
{
@@ -1428,7 +1808,7 @@ namespace ps2recomp
}
}
Instruction inst = m_decoder->decodeInstruction(address, rawInstruction);
Instruction inst = m_decoder->decodeInstruction(address, rawInstruction, !m_config.lowMemoryMode);
auto mmioIt = m_config.mmioByInstructionAddress.find(address);
if (mmioIt != m_config.mmioByInstructionAddress.end())
+39 -6
View File
@@ -1,9 +1,23 @@
#include "ps2recomp/r5900_decoder.h"
#include "rabbitizer.h"
#include <iostream>
#include <vector>
namespace ps2recomp
{
static std::string disassembleRabbitizerInstruction(RabbitizerInstruction &rabbitizerInst)
{
std::string disassembly;
const size_t bufferSize = RabbitizerInstruction_getSizeForBuffer(&rabbitizerInst, 0, 0);
if (bufferSize > 0)
{
std::vector<char> buffer(bufferSize + 1, '\0');
RabbitizerInstruction_disassemble(&rabbitizerInst, buffer.data(), nullptr, 0, 0);
disassembly = buffer.data();
}
return disassembly;
}
R5900Decoder::R5900Decoder()
{
@@ -13,7 +27,29 @@ namespace ps2recomp
{
}
Instruction R5900Decoder::decodeInstruction(uint32_t address, uint32_t rawInstruction) const
std::string R5900Decoder::disassembleInstruction(uint32_t address, uint32_t rawInstruction)
{
RabbitizerInstruction rabbitizerInst;
RabbitizerInstructionR5900_init(&rabbitizerInst, rawInstruction, address);
RabbitizerInstructionR5900_processUniqueId(&rabbitizerInst);
std::string disassembly = disassembleRabbitizerInstruction(rabbitizerInst);
RabbitizerInstructionR5900_destroy(&rabbitizerInst);
return disassembly;
}
std::string R5900Decoder::disassembleInstruction(const Instruction &inst)
{
if (!inst.disassembly.empty())
{
return inst.disassembly;
}
return disassembleInstruction(inst.address, inst.raw);
}
Instruction R5900Decoder::decodeInstruction(uint32_t address, uint32_t rawInstruction, bool includeDisassembly) const
{
Instruction inst;
@@ -177,12 +213,9 @@ namespace ps2recomp
inst.vectorInfo.isVector = inst.isVU; // Only VU ops are truly vector
}
size_t bufferSize = RabbitizerInstruction_getSizeForBuffer(&rabbitizerInst, 0, 0);
if (bufferSize > 0)
if (includeDisassembly)
{
std::vector<char> buffer(bufferSize + 1, '\0');
RabbitizerInstruction_disassemble(&rabbitizerInst, buffer.data(), nullptr, 0, 0);
inst.disassembly = buffer.data();
inst.disassembly = disassembleRabbitizerInstruction(rabbitizerInst);
}
RabbitizerInstructionR5900_destroy(&rabbitizerInst);
+94 -28
View File
@@ -305,6 +305,60 @@ void register_code_generator_tests()
"JALR resume pc should also be emitted as an internal label");
});
tc.Run("unresolved JR marks internal labels as indirect fallback resume entries", [](TestCase &t) {
Function func;
func.name = "unresolved_jr_fallback";
func.start = 0x3100;
func.end = 0x3120;
func.isRecompiled = true;
func.isStub = false;
std::vector<Instruction> instructions{
makeNop(0x3100),
makeJr(0x3104, 8),
makeNop(0x3108),
makeNop(0x310C),
makeNop(0x3110),
};
CodeGenerator gen({}, {});
CodeGenerator::AnalysisResult analysis = gen.collectInternalBranchTargets(func, instructions);
t.IsTrue(analysis.indirectFallbackEntryPoints.contains(0x310Cu),
"unresolved JR should register internal labels as resumable entries for the owning function");
t.IsTrue(analysis.entryPoints.contains(0x310Cu),
"unresolved JR fallback targets should still emit labels in the owner");
t.IsFalse(analysis.jumpTableTargets.contains(0x3104u),
"unresolved JR should not pretend it has a resolved local jump table");
});
tc.Run("unresolved JALR marks internal labels as indirect fallback resume entries", [](TestCase &t) {
Function func;
func.name = "unresolved_jalr_fallback";
func.start = 0x3200;
func.end = 0x3220;
func.isRecompiled = true;
func.isStub = false;
std::vector<Instruction> instructions{
makeNop(0x3200),
makeJalr(0x3204, 25, 31),
makeNop(0x3208),
makeNop(0x320C),
makeNop(0x3210),
};
CodeGenerator gen({}, {});
CodeGenerator::AnalysisResult analysis = gen.collectInternalBranchTargets(func, instructions);
t.IsTrue(analysis.indirectFallbackEntryPoints.contains(0x320Cu),
"unresolved JALR should register internal labels as resumable entries for the owning function");
t.IsTrue(analysis.entryPoints.contains(0x320Cu),
"unresolved JALR fallback targets should still emit labels in the owner");
t.IsFalse(analysis.jumpTableTargets.contains(0x3204u),
"unresolved JALR should not pretend it has a resolved local jump table");
});
tc.Run("resume entry targets emit a top-level pc switch in the owner wrapper", [](TestCase &t) {
Function func;
func.name = "resume_owner";
@@ -1142,9 +1196,9 @@ void register_code_generator_tests()
t.IsTrue(generated.find("if (branch_taken_0x1200)") != std::string::npos, "should generate branch_taken variable and if for likely branch");
});
tc.Run("JR $31 emits switch for internal return targets", [](TestCase &t) {
tc.Run("JR $31 returns through dynamic target without broad local switch", [](TestCase &t) {
Function func;
func.name = "jr_ra_switch";
func.name = "jr_ra_return";
func.start = 0x1300;
func.end = 0x1340;
func.isRecompiled = true;
@@ -1162,10 +1216,16 @@ void register_code_generator_tests()
CodeGenerator gen({}, {});
std::string generated = gen.generateFunction(func, { jal, jalDelay, atReturn, atTarget, jr, jrDelay }, false);
printGeneratedCode("JR $31 emits switch for internal return targets", generated);
printGeneratedCode("JR $31 returns through dynamic target without broad local switch", generated);
t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos, "JR $31 should emit switch for internal targets");
t.IsTrue(generated.find("case 0x1308u: goto label_1308;") != std::string::npos, "switch should include return address from internal JAL");
t.IsTrue(generated.find("uint32_t jumpTarget = GPR_U32(ctx, 31);") != std::string::npos,
"JR $31 should still read the dynamic return target");
t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos,
"JR $31 should not emit a broad local switch over internal labels");
t.IsTrue(generated.find("label_1308:") != std::string::npos,
"internal JAL return address should still be emitted as a label");
t.IsTrue(generated.find(" return;") != std::string::npos,
"JR $31 should return to the dispatcher/runtime after setting ctx->pc");
});
tc.Run("trailing JR $31 without decoded delay slot still emits return flow", [](TestCase &t) {
@@ -1188,17 +1248,17 @@ void register_code_generator_tests()
t.IsTrue(generated.find("uint32_t jumpTarget = GPR_U32(ctx, 31);") != std::string::npos,
"truncated trailing JR should still read the return target");
t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos,
"truncated trailing JR should still emit the return-target switch");
t.IsTrue(generated.find("case 0x1508u: goto label_1508;") != std::string::npos,
"truncated trailing JR should still include internal return targets");
t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos,
"truncated trailing JR should not emit a broad local return-target switch");
t.IsTrue(generated.find("label_1508:") != std::string::npos,
"truncated trailing JR should still include the internal return label");
t.IsTrue(generated.find("// JR $31 - Handled by branch logic") == std::string::npos,
"truncated trailing JR must not degrade to comment-only output");
});
tc.Run("JR non-RA emits switch for in-function jump targets", [](TestCase &t) {
tc.Run("unresolved JR non-RA uses dispatcher resume entries without broad local switch", [](TestCase &t) {
Function func;
func.name = "jr_non_ra_switch";
func.name = "jr_non_ra_dispatcher_resume";
func.start = 0x1400;
func.end = 0x1420;
func.isRecompiled = true;
@@ -1214,17 +1274,21 @@ void register_code_generator_tests()
Instruction i3 = makeNop(0x140c);
CodeGenerator gen({}, {});
CodeGenerator::AnalysisResult analysis = gen.collectInternalBranchTargets(func, {i0, jr, delay, i3});
gen.setResumeEntryTargets({{func.start, std::vector<uint32_t>(
analysis.indirectFallbackEntryPoints.begin(),
analysis.indirectFallbackEntryPoints.end())}});
std::string generated = gen.generateFunction(func, {i0, jr, delay, i3}, false);
printGeneratedCode("JR non-RA emits switch for in-function jump targets", generated);
printGeneratedCode("unresolved JR non-RA uses dispatcher resume entries without broad local switch", generated);
t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos,
"JR via non-RA register should emit switch for internal targets");
t.IsTrue(generated.find("switch (ctx->pc)") == std::string::npos,
"JR fallback labels should not be promoted to dispatcher resume sites");
t.IsTrue(generated.find("case 0x1400u: goto label_1400;") != std::string::npos,
"switch should include in-function entry label");
t.IsTrue(generated.find("case 0x140Cu: goto label_140c;") != std::string::npos,
"switch should include other in-function labels");
t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos,
"unresolved JR via non-RA register should not emit a broad local switch over internal labels");
t.IsTrue(generated.find("switch (ctx->pc)") != std::string::npos,
"JR fallback labels should be promoted to dispatcher resume sites");
t.IsTrue(generated.find("case 0x140cu: goto label_140c;") != std::string::npos,
"owner resume switch should include internal fallback labels");
t.IsTrue(generated.find("ctx->pc = jumpTarget;") != std::string::npos,
"unresolved JR should hand the dynamic target back through ctx->pc");
});
tc.Run("configured jump table addresses drive JR dispatch targets", [](TestCase &t) {
@@ -1291,6 +1355,8 @@ void register_code_generator_tests()
func,
{lui, addiu, sll, addu, lw, jr, jrDelay, target0, target1});
t.IsTrue(analysis.jumpTableTargets.contains(0x1614u),
"configured JR table should be tracked as a resolved local jump table");
t.IsFalse(analysis.resumeEntryPoints.contains(0x1620u),
"configured JR table targets should stay in-function dispatch labels");
t.IsFalse(analysis.resumeEntryPoints.contains(0x1630u),
@@ -1314,15 +1380,15 @@ void register_code_generator_tests()
"configured table should avoid broad JR fallback labels");
});
tc.Run("JALR includes switch and fallback/guard pair", [](TestCase &t) {
tc.Run("unresolved JALR uses runtime dispatch without broad local switch", [](TestCase &t) {
Function func;
func.name = "jalr_switch_and_fallback";
func.name = "jalr_runtime_fallback";
func.start = 0x1500;
func.end = 0x1530;
func.isRecompiled = true;
func.isStub = false;
// A call-like setup so there are multiple in-function labels to dispatch to.
// A call-like setup so there are multiple in-function labels available.
Instruction jal = makeJal(0x1500, 0x1510);
Instruction jalDelay = makeNop(0x1504);
Instruction atReturn = makeNop(0x1508);
@@ -1332,12 +1398,12 @@ void register_code_generator_tests()
CodeGenerator gen({}, {});
std::string generated = gen.generateFunction(func, {jal, jalDelay, atReturn, atTarget, jalr, jalrDelay}, false);
printGeneratedCode("JALR includes switch and fallback/guard pair", generated);
printGeneratedCode("unresolved JALR uses runtime dispatch without broad local switch", generated);
t.IsTrue(generated.find("switch (jumpTarget)") != std::string::npos,
"JALR should emit switch when in-function register-jump targets exist");
t.IsTrue(generated.find("case 0x1508u: goto label_1508;") != std::string::npos,
"switch should include internal return label from JAL in same function");
t.IsFalse(generated.find("switch (jumpTarget)") != std::string::npos,
"unresolved JALR should not emit a broad local switch over every internal label");
t.IsTrue(generated.find("auto targetFn = runtime->lookupFunction(jumpTarget);") != std::string::npos,
"unresolved JALR should dispatch through the runtime");
t.IsTrue(generated.find("if (ctx->pc == __entryPc) { ctx->pc = 0x151Cu; }") != std::string::npos,
"JALR should contain unchanged-PC fallback to fallthrough");
t.IsTrue(generated.find("if (ctx->pc != 0x151Cu) { return; }") != std::string::npos,