recompiler: split SCUS_*_full.c into parallel-compilable shards

Emit a shared <stem>_decls.h (runtime extern prologue, static-inline
unaligned helpers, all func + alias-body forward decls) and distribute
function bodies across N ~40k-line <stem>_full_NN.c shards instead of one
monolithic <stem>_full.c. Overlapping-alias host bodies become externally
linked so shard bucketing has no atomicity constraint. runtime.cmake's
GAME_GENERATED_FULL_C is now multi-value (single path still works, so
other games are unaffected until they regen + glob).

Tomba: SCUS_942.36_full.c 41MB single TU -> 36 shards; compile (-O3)
4m35s -> 51s (-j16, ~5.4x). Generated function bytes verified byte-identical
(6245/6245 functions).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -98,3 +98,6 @@ tools/_options_session/
 
 # Compiled test binaries (build artifacts; the runtime/tests/*.c sources are tracked)
 runtime/tests/*.exe
+
+# transient split-gen build dirs
+recompiler/build-splitgen/

recompiler/include/code_generator.h

@@ -4,6 +4,7 @@
 #include <string>
 #include <vector>
 #include <sstream>
+#include <ostream>
 #include <set>
 #include <map>
 #include <array>
@@ -200,6 +201,20 @@ public:
         const std::vector<Function>& functions,
         const std::map<uint32_t, ControlFlowGraph>& cfgs);
 
+    // The generated functions from the most recent generate_file() call
+    // (post mid-function-split pass). Consumed by main_psx to write the
+    // split full.c shards + shared decls header. Not populated until
+    // generate_file() has run.
+    const std::vector<GeneratedFunction>& last_gen_funcs() const { return last_gen_funcs_; }
+
+    // Build the shared-declarations header shared by every full.c shard:
+    // the runtime extern prologue, the unaligned-access helpers (as static
+    // inline, since this header is included by multiple translation units),
+    // and forward declarations for every recompiled function plus every
+    // overlapping-alias shared body. Byte-for-byte the same declaration set
+    // the monolith emits inline, just hoisted into a header.
+    std::string build_shared_decls_header(const std::vector<GeneratedFunction>& gen_funcs) const;
+
     // Generate the per-function code-range manifest consumed by the overlay
     // loader's per-entry validity hash (design §8). For each function, emits its
     // compiled code byte-ranges (union of basic-block extents), coalesced;
@@ -231,6 +246,29 @@ private:
     std::set<uint32_t> extra_labels_;    // Mid-block addresses that need inline labels (jump table targets)
     const AnnotationTable* annotations_ = nullptr;
 
+    // Forward declarations for every psx_alias_body_XXXXXXXX() emitted by
+    // generate_alias_group() during the most recent generate_all_functions()
+    // pass. These bodies are externally linked (split-TU build) so every
+    // shard/header that calls into another shard's alias body needs the
+    // prototype. Cleared at the top of generate_all_functions().
+    std::vector<std::string> alias_body_decls_;
+
+    // The functions emitted by the most recent generate_file() call, stashed
+    // for main_psx's split-shard writer. See last_gen_funcs().
+    std::vector<GeneratedFunction> last_gen_funcs_;
+
+    // Emit the runtime-hook extern prologue (debug_server_log_call_entry ...
+    // g_debug_last_store_pc). Shared verbatim between the monolith path
+    // (generate_file) and the split-shard shared header
+    // (build_shared_decls_header).
+    void emit_runtime_externs(std::ostream& ss) const;
+
+    // Emit the LWL/LWR/SWL/SWR unaligned-access reference helpers.
+    // as_inline selects `static inline` (shared header, included by many
+    // TUs) vs `static` (monolith, single TU) linkage for the 4 helper
+    // function definitions; everything else is identical.
+    void emit_unaligned_helpers(std::ostream& ss, bool as_inline) const;
+
     // RECURSION_BUG.md §25 — continuation-passing call/return (the universal fix
     // for the idle-freeze host-stack leak). When set (gen-time env PSX_CPS),
     // guest calls (jal/jalr/jr-table/external) are emitted as TAIL-TRANSFERS

recompiler/src/code_generator.cpp

@@ -2456,8 +2456,9 @@ std::vector<GeneratedFunction> CodeGenerator::generate_alias_group(
     std::stringstream body;
     body << fmt::format("/* Overlapping-alias body for host func_{:08X}: {} entries */\n",
                         host, aliases.size());
-    body << fmt::format("static void psx_alias_body_{:08X}(CPUState* cpu, uint32_t entry)\n{{\n",
+    body << fmt::format("void psx_alias_body_{:08X}(CPUState* cpu, uint32_t entry)\n{{\n",
                         host);
+    alias_body_decls_.push_back(fmt::format("void psx_alias_body_{:08X}(CPUState* cpu, uint32_t entry)", host));
     // CPS (§25): a dispatched continuation (callee published cpu->pc = $ra) is
     // routed into its block here, BEFORE the entry switch — cpu->pc overrides
     // the `entry` arg. The dispatch routes a continuation by calling the first
@@ -2538,6 +2539,7 @@ std::vector<GeneratedFunction> CodeGenerator::generate_all_functions(
     // game dispatch table (psx_dispatch_game_compiled) reflects exactly the
     // functions emitted here (the final generate_file pass is authoritative).
     cps_continuation_owner_.clear();
+    alias_body_decls_.clear();
 
     fmt::print("\n=== C Code Generation ===\n\n");
     fmt::print("Generating C code for {} functions...\n", functions.size());
@@ -2658,18 +2660,7 @@ std::vector<GeneratedFunction> CodeGenerator::generate_all_functions(
     return results;
 }
 
-std::string CodeGenerator::generate_file(
-    const std::vector<Function>& functions,
-    const std::map<uint32_t, ControlFlowGraph>& cfgs) {
-
-    std::stringstream ss;
-
-    // File header
-    ss << "/* Generated by PSXRecomp - PlayStation 1 Static Recompiler */\n\n";
-
-    // Include the generic PSX runtime header.
-    // This provides CPUState, GTE/trap declarations, and call_by_address().
-    ss << "#include \"psx_runtime.h\"\n\n";
+void CodeGenerator::emit_runtime_externs(std::ostream& ss) const {
     ss << "extern void debug_server_log_call_entry(uint32_t func_addr);\n";
     ss << "#ifndef PSX_NO_DEBUG_TOOLS\n";
     ss << "extern void debug_server_cyc_observe(uint32_t block_leader_phys);\n";
@@ -2701,6 +2692,10 @@ std::string CodeGenerator::generate_file(
     ss << "extern uint32_t psx_ws_backdrop_value(uint32_t orig, int is_end, int window_cols);  /* ws backdrop preload (gpu.c) */\n";
     ss << "extern void gte_ws_set_suppress(int on);  /* widescreen far-backdrop un-squash (gte.cpp) */\n";
     ss << "extern uint32_t g_debug_last_store_pc;  /* exact PC of the executing SW/SH/SB — wtrace/readtrace producer attribution (debug_server.c) */\n\n";
+}
+
+void CodeGenerator::emit_unaligned_helpers(std::ostream& ss, bool as_inline) const {
+    const char* kw = as_inline ? "static inline " : "static ";
 
     // Emit reference implementations for unaligned memory helpers.
     // These implement the MIPS lwl/lwr/swl/swr semantics.
@@ -2717,7 +2712,7 @@ std::string CodeGenerator::generate_file(
     ss << " * addr is the effective (possibly unaligned) address.\n";
     ss << " * rt_value is the current value of the destination register.\n";
     ss << " */\n";
-    ss << "static uint32_t psx_lwl(CPUState* cpu, uint32_t addr, uint32_t rt_value, uint32_t rt, uint32_t reg_mask) {\n";
+    ss << kw << "uint32_t psx_lwl(CPUState* cpu, uint32_t addr, uint32_t rt_value, uint32_t rt, uint32_t reg_mask) {\n";
     ss << "    uint32_t word = psx_cyc_load_word(cpu, addr & ~3u, rt, reg_mask);  /* full load interlock */\n";
     ss << "    switch (addr & 3u) {\n";
     ss << "        case 0: return (rt_value & 0x00FFFFFFu) | (word << 24);\n";
@@ -2731,7 +2726,7 @@ std::string CodeGenerator::generate_file(
     ss << " * addr is the effective (possibly unaligned) address.\n";
     ss << " * rt_value is the current value of the destination register.\n";
     ss << " */\n";
-    ss << "static uint32_t psx_lwr(CPUState* cpu, uint32_t addr, uint32_t rt_value, uint32_t rt, uint32_t reg_mask) {\n";
+    ss << kw << "uint32_t psx_lwr(CPUState* cpu, uint32_t addr, uint32_t rt_value, uint32_t rt, uint32_t reg_mask) {\n";
     ss << "    uint32_t word = psx_cyc_load_word(cpu, addr & ~3u, rt, reg_mask);  /* full load interlock */\n";
     ss << "    switch (addr & 3u) {\n";
     ss << "        case 0: return word;\n";
@@ -2745,7 +2740,7 @@ std::string CodeGenerator::generate_file(
     ss << " * addr is the effective (possibly unaligned) address.\n";
     ss << " * rt_value is the value of the source register.\n";
     ss << " */\n";
-    ss << "static void psx_swl(CPUState* cpu, uint32_t addr, uint32_t rt_value) {\n";
+    ss << kw << "void psx_swl(CPUState* cpu, uint32_t addr, uint32_t rt_value) {\n";
     ss << "    uint32_t aligned_addr = addr & ~3u;\n";
     ss << "    uint32_t word = cpu->read_word(aligned_addr);\n";
     ss << "    switch (addr & 3u) {\n";
@@ -2761,7 +2756,7 @@ std::string CodeGenerator::generate_file(
     ss << " * addr is the effective (possibly unaligned) address.\n";
     ss << " * rt_value is the value of the source register.\n";
     ss << " */\n";
-    ss << "static void psx_swr(CPUState* cpu, uint32_t addr, uint32_t rt_value) {\n";
+    ss << kw << "void psx_swr(CPUState* cpu, uint32_t addr, uint32_t rt_value) {\n";
     ss << "    uint32_t aligned_addr = addr & ~3u;\n";
     ss << "    uint32_t word = cpu->read_word(aligned_addr);\n";
     ss << "    switch (addr & 3u) {\n";
@@ -2773,6 +2768,38 @@ std::string CodeGenerator::generate_file(
     ss << "    cpu->write_word(aligned_addr, word);\n";
     ss << "}\n";
     ss << "\n";
+}
+
+std::string CodeGenerator::build_shared_decls_header(const std::vector<GeneratedFunction>& gen_funcs) const {
+    std::stringstream h;
+    h << "/* Generated by PSXRecomp - shared declarations for split full.c shards. DO NOT EDIT. */\n";
+    h << "#pragma once\n";
+    h << "#include \"psx_runtime.h\"\n\n";
+    emit_runtime_externs(h);
+    emit_unaligned_helpers(h, /*as_inline=*/true);
+    if (!gen_funcs.empty()) {
+        h << "/* Forward declarations for all recompiled functions */\n";
+        for (const auto& gf : gen_funcs) h << gf.signature << ";\n";
+        for (const auto& decl : alias_body_decls_) h << decl << ";\n";
+        h << "\n";
+    }
+    return h.str();
+}
+
+std::string CodeGenerator::generate_file(
+    const std::vector<Function>& functions,
+    const std::map<uint32_t, ControlFlowGraph>& cfgs) {
+
+    std::stringstream ss;
+
+    // File header
+    ss << "/* Generated by PSXRecomp - PlayStation 1 Static Recompiler */\n\n";
+
+    // Include the generic PSX runtime header.
+    // This provides CPUState, GTE/trap declarations, and call_by_address().
+    ss << "#include \"psx_runtime.h\"\n\n";
+    emit_runtime_externs(ss);
+    emit_unaligned_helpers(ss, /*as_inline=*/false);
 
     // Make mutable copies for potential function splitting
     std::vector<Function> functions_mut(functions);
@@ -2981,6 +3008,7 @@ std::string CodeGenerator::generate_file(
         for (const auto& gen_func : gen_funcs) {
             ss << gen_func.signature << ";\n";
         }
+        for (const auto& decl : alias_body_decls_) ss << decl << ";\n";
         ss << "\n";
     }
 
@@ -2989,6 +3017,10 @@ std::string CodeGenerator::generate_file(
         ss << gen_func.full_code << "\n";
     }
 
+    // Stash for the split-shard writer (main_psx); copy is intentional so the
+    // caller can consume it after generate_file() returns the monolith text.
+    last_gen_funcs_ = gen_funcs;
+
     // Preserve the exact post-split CFG inventory used by the emitted C. The
     // dispatch generator consumes this so synthesized fallthrough functions
     // receive the same byte-precise native-validity ranges as ordinary entries.

recompiler/src/main_psx.cpp

@@ -920,15 +920,81 @@ int main(int argc, char** argv) {
         }
     }
 
-    std::ofstream out_file(output_filename);
-    if (out_file.is_open()) {
-        out_file << full_c_code;
-        out_file.close();
-        fmt::print("✓ Saved {} lines to {}\n\n",
-                  std::count(full_c_code.begin(), full_c_code.end(), '\n'),
-                  output_filename.string());
-    } else {
-        fmt::print(stderr, "⚠ Failed to write output file\n\n");
+    // Split-TU output: instead of one monolithic <exe_stem>_full.c, write a
+    // shared declarations header plus N shards bucketed by cumulative line
+    // count, so the game build can compile the shards in parallel. The
+    // in-memory full_c_code / dispatch_addrs scan above is unaffected — only
+    // the on-disk representation changes. Function bodies are byte-identical
+    // to what full_c_code would have contained; see
+    // CodeGenerator::last_gen_funcs() / build_shared_decls_header().
+    {
+        // 1. Remove stale outputs: the old monolith and any previously
+        //    written shards (a shard count shrink must not leave orphans).
+        std::error_code rm_ec;
+        std::filesystem::remove(output_filename, rm_ec);
+        for (int stale = 0; stale < 256; stale++) {
+            std::filesystem::path stale_shard =
+                out_dir / fmt::format("{}_full_{:02d}.c", exe_stem, stale);
+            std::filesystem::remove(stale_shard, rm_ec);
+        }
+
+        // 2. Shared declarations header.
+        std::filesystem::path decls_filename = out_dir / (exe_stem + "_decls.h");
+        {
+            std::ofstream decls_file(decls_filename);
+            if (decls_file.is_open()) {
+                decls_file << codegen.build_shared_decls_header(codegen.last_gen_funcs());
+                decls_file.close();
+                fmt::print("✓ Saved shared decls header to {}\n", decls_filename.string());
+            } else {
+                fmt::print(stderr, "⚠ Failed to write shared decls header\n\n");
+            }
+        }
+
+        // 3. Bucket generated functions into shards by cumulative line count.
+        const int SHARD_LINE_BUDGET = 40000;
+        const std::vector<PSXRecomp::GeneratedFunction>& gen_funcs_for_shards = codegen.last_gen_funcs();
+        std::string decls_basename = exe_stem + "_decls.h";
+
+        int shard_index = 0;
+        int shard_lines = 0;
+        int total_shard_lines = 0;
+        std::stringstream shard_buf;
+        auto flush_shard = [&]() {
+            if (shard_buf.tellp() == std::streampos(0)) return;
+            std::filesystem::path shard_filename =
+                out_dir / fmt::format("{}_full_{:02d}.c", exe_stem, shard_index);
+            std::ofstream shard_file(shard_filename);
+            if (shard_file.is_open()) {
+                shard_file << fmt::format(
+                    "/* Generated by PSXRecomp - full.c shard {}. DO NOT EDIT. */\n",
+                    shard_index);
+                shard_file << fmt::format("#include \"{}\"\n\n", decls_basename);
+                shard_file << shard_buf.str();
+                shard_file.close();
+                fmt::print("✓ Saved shard {} ({} lines) to {}\n",
+                           shard_index, shard_lines, shard_filename.string());
+            } else {
+                fmt::print(stderr, "⚠ Failed to write shard {}\n\n", shard_index);
+            }
+            total_shard_lines += shard_lines;
+            shard_index++;
+            shard_lines = 0;
+            shard_buf.str(std::string());
+            shard_buf.clear();
+        };
+
+        for (const auto& gf : gen_funcs_for_shards) {
+            if (shard_lines > 0 && shard_lines + gf.line_count > SHARD_LINE_BUDGET) {
+                flush_shard();
+            }
+            shard_buf << gf.full_code << "\n";
+            shard_lines += gf.line_count;
+        }
+        flush_shard();
+
+        fmt::print("✓ Wrote {} shards ({} total lines) for {}\n\n",
+                   shard_index, total_shard_lines, exe_stem);
     }
 
     // Per-function code-range manifest (design §8): consumed by the overlay

runtime/runtime.cmake

@@ -263,7 +263,6 @@ endif()
 function(psxrecomp_add_runtime_target target)
     set(options ORACLE COSIM)
     set(oneValueArgs
-        GAME_GENERATED_FULL_C
         GAME_GENERATED_DISPATCH_C
         GAME_OVERLAY_STATIC_C
         DEBUG_PORT
@@ -272,7 +271,12 @@ function(psxrecomp_add_runtime_target target)
         DEFAULT_GAME_CONFIG_PATH
         EXE_NAME
     )
-    set(multiValueArgs EXTRAS_SOURCES)
+    # GAME_GENERATED_FULL_C is a list (not a single value): the split-TU build
+    # writes the recompiled game as N full_NN.c shards instead of one
+    # monolithic full.c, so this argument may carry 1..N paths. A single path
+    # is just a one-element list, so games still passing one file are
+    # unaffected.
+    set(multiValueArgs EXTRAS_SOURCES GAME_GENERATED_FULL_C)
     cmake_parse_arguments(PSXRT "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
     # DEBUG_PORT and WINDOW_TITLE were previously required cmake-time defaults.
@@ -307,8 +311,10 @@ function(psxrecomp_add_runtime_target target)
 
     set(generated_sources ${PSXRECOMP_BIOS_GENERATED})
     if(PSXRT_GAME_GENERATED_FULL_C)
-        set_source_files_properties("${PSXRT_GAME_GENERATED_FULL_C}" PROPERTIES GENERATED TRUE)
-        list(APPEND generated_sources "${PSXRT_GAME_GENERATED_FULL_C}")
+        foreach(_full_src IN LISTS PSXRT_GAME_GENERATED_FULL_C)
+            set_source_files_properties("${_full_src}" PROPERTIES GENERATED TRUE)
+            list(APPEND generated_sources "${_full_src}")
+        endforeach()
         set(has_game_dispatch TRUE)
     endif()
     if(PSXRT_GAME_GENERATED_DISPATCH_C)