From 1c0a3edee9a97deff13ea92fb023ee3c577cfb8c Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:00:47 +0200 Subject: [PATCH] Add synthetic recompilation CI workflow (#161) --- .github/workflows/build.yml | 4 + .github/workflows/package.yml | 6 +- .github/workflows/recomp-test.yml | 41 ++++++++ Launcher/Test-Recompilation.ps1 | 146 +++++++++++++++++++++++++++++ Launcher/prepare-portable-tools.sh | 26 ++--- 5 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/recomp-test.yml create mode 100644 Launcher/Test-Recompilation.ps1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f258e14..267a34e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,10 @@ concurrency: cancel-in-progress: true jobs: + recompilation: + name: Recompilation test + uses: ./.github/workflows/recomp-test.yml + translator: name: Translator (build + test) runs-on: windows-latest diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 742418f..b97c62a 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -20,6 +20,10 @@ concurrency: cancel-in-progress: true jobs: + recompilation: + name: Recompilation test + uses: ./.github/workflows/recomp-test.yml + linux-appimage: name: Linux (AppImage, ${{ matrix.arch }}) strategy: @@ -93,7 +97,7 @@ jobs: release: name: Publish GitHub Release if: startsWith(github.ref, 'refs/tags/v') - needs: [linux-appimage, windows-installer] + needs: [linux-appimage, windows-installer, recompilation] runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/recomp-test.yml b/.github/workflows/recomp-test.yml new file mode 100644 index 0000000..0908d15 --- /dev/null +++ b/.github/workflows/recomp-test.yml @@ -0,0 +1,41 @@ +name: Synthetic recompilation + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + windows: + name: Windows runtime (synthetic DOL) + runs-on: windows-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: '8.0.x' + + # Cache downloads only. Preparation still validates pins, and every run + # compiles current Aurora, runtime, and generated sources from scratch. + - uses: actions/cache@v5 + with: + path: Launcher/artifacts/downloads + key: windows-recomp-downloads-${{ hashFiles('Launcher/Prepare-PortableTools.ps1', 'Launcher/Prepare-Dependencies.ps1') }} + + - name: Prepare the shipped Windows toolchain + shell: pwsh + run: ./Launcher/Prepare-PortableTools.ps1 + + - name: Prepare pinned native dependencies + shell: pwsh + run: ./Launcher/Prepare-Dependencies.ps1 + + - name: Translate, compile the full runtime, and link + shell: pwsh + run: ./Launcher/Test-Recompilation.ps1 -Parallel 3 diff --git a/Launcher/Test-Recompilation.ps1 b/Launcher/Test-Recompilation.ps1 new file mode 100644 index 0000000..9a8d8a2 --- /dev/null +++ b/Launcher/Test-Recompilation.ps1 @@ -0,0 +1,146 @@ +# Build the real Windows runtime with translated, entirely synthetic PowerPC code. +# No game dump, game symbol map, REL, mod download, or existing generated/ output is used. +[CmdletBinding()] +param( + [string]$PortableToolsDirectory = 'Launcher/artifacts/portable-tools', + [string]$DependencySourceDirectory = 'Launcher/artifacts/dependencies', + [string]$StageDirectory = 'build/recomp-test', + [ValidateRange(1, 64)] [int]$Parallel = 3 +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 3.0 +. (Join-Path $PSScriptRoot 'NativeBuildFlags.ps1') + +$repoRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +function Full([string]$Path) { + if ([IO.Path]::IsPathRooted($Path)) { return [IO.Path]::GetFullPath($Path) } + return [IO.Path]::GetFullPath((Join-Path $repoRoot $Path)) +} +$portableTools = Full $PortableToolsDirectory +$dependencies = Full $DependencySourceDirectory +$stage = Full $StageDirectory +$dotnet = (Get-Command dotnet -CommandType Application).Source +$cmake = Join-Path $portableTools 'CMake/bin/cmake.exe' +$ninja = Join-Path $portableTools 'Ninja/ninja.exe' +$compilerBin = Join-Path $portableTools 'llvm-mingw/bin' +Assert-File $cmake 'Pinned CMake (run Prepare-PortableTools.ps1 first)' +Assert-File $ninja 'Pinned Ninja' +Assert-File (Join-Path $dependencies 'cppwinrt/winrt/base.h') 'Pinned dependencies (run Prepare-Dependencies.ps1 first)' + +# Refuse reuse so a developer's game translation or an earlier build cannot make +# the test pass. Keep the staging tree after the run for diagnostics. +if (Test-Path -LiteralPath $stage) { throw "Test stage already exists; choose a fresh -StageDirectory: $stage" } +[IO.Directory]::CreateDirectory($stage) | Out-Null +Write-Host "Synthetic recompilation workspace: $stage" + +# Copy current sources, including uncommitted edits, but no ignored build output. +# An isolated workspace preserves the developer's real generated/ directory. +$sourceFiles = & git -C $repoRoot -c core.quotepath=false ls-files --cached --others --exclude-standard -- runtime aurora-main +if ($LASTEXITCODE -ne 0) { throw 'Could not enumerate runtime and Aurora sources.' } +foreach ($relative in $sourceFiles | Sort-Object -Unique) { + $destination = Join-Path $stage $relative + [IO.Directory]::CreateDirectory([IO.Path]::GetDirectoryName($destination)) | Out-Null + Copy-Item -LiteralPath (Join-Path $repoRoot $relative) -Destination $destination +} + +# One synthetic text section: li r3,40; addi r3,r3,2; nop; blr. +# Native HLE wrappers also call these eight guest symbols directly. Give each +# its own generated blr function so the real product can link without game code. +# Keep this list explicit: a new unresolved guest dependency must fail the test. +[uint32]$entry = 0x80001000L +[uint32[]]$guestCallbacks = @( + 0x8012B830L, 0x801A0620L, 0x801A1ED8L, 0x801A961CL, + 0x801AADE0L, 0x801D8D30L, 0x801D9E94L, 0x8055531CL +) +$textSize = [int]($guestCallbacks[-1] - $entry + 4) +$dataOffset = 0x100 + $textSize +$dol = [byte[]]::new($dataOffset + 4) +function Write-BigEndian32([int]$Offset, [uint32]$Value) { + $dol[$Offset] = [byte](($Value -shr 24) -band 255) + $dol[$Offset + 1] = [byte](($Value -shr 16) -band 255) + $dol[$Offset + 2] = [byte](($Value -shr 8) -band 255) + $dol[$Offset + 3] = [byte]($Value -band 255) +} +Write-BigEndian32 0x00 0x100 # text[0] file offset +Write-BigEndian32 0x48 $entry # text[0] guest address +Write-BigEndian32 0x90 $textSize # text[0] length (unreachable gaps are zero) +Write-BigEndian32 0x1C $dataOffset # data[0] file offset +Write-BigEndian32 0x64 0x80600000L # data[0] guest address +Write-BigEndian32 0xAC 4 # data[0] length +Write-BigEndian32 0xD8 0x80601000L # BSS address +Write-BigEndian32 0xDC 32 # BSS length +Write-BigEndian32 0xE0 $entry # entry point +Write-BigEndian32 0x100 0x38600028 # li r3,40 +Write-BigEndian32 0x104 0x38630002 # addi r3,r3,2 +Write-BigEndian32 0x108 0x60000000 # nop +Write-BigEndian32 0x10C 0x4E800020 # blr +foreach ($address in $guestCallbacks) { + Write-BigEndian32 ([int](0x100 + $address - $entry)) 0x4E800020 +} +Write-BigEndian32 $dataOffset 0x12345678 +[IO.File]::WriteAllBytes((Join-Path $stage 'synthetic.dol'), $dol) +$entryPoints = (@($entry) + $guestCallbacks | ForEach-Object { '0x{0:X8}' -f $_ }) -join ', ' +$functionMap = (@($entry) + $guestCallbacks | ForEach-Object { '{0:X8} func_{0:X8}' -f $_ }) -join "`n" +[IO.File]::WriteAllText((Join-Path $stage 'synthetic-functions.txt'), $functionMap) +$manifest = Join-Path $stage 'recomp.yml' +[IO.File]::WriteAllText($manifest, @" +schema_version: 1 +workspace_root: . +project: + id: ci-synthetic-dol + display_name: CI Synthetic DOL +memory: + base: 0x80000000 + size: 0x01800000 + sda_base: 0x80600000 + sda2_base: 0x80600000 +inputs: + dol: + path: synthetic.dol +translation: + entry_points: [$entryPoints] + function_map: + path: synthetic-functions.txt + allow_unsupported_instructions: false +runtime: + native_abi_directories: [] + native_registration_root: runtime/src +output: + root: generated +"@) + +$translatorProject = Join-Path $repoRoot 'translator/src/Translator.Cli/Translator.Cli.csproj' +Invoke-Checked $dotnet @('build', $translatorProject, '-c', 'Release', '--disable-build-servers') 'Building the translator' +$translator = Join-Path $repoRoot 'translator/src/Translator.Cli/bin/Release/net8.0/Translator.Cli.dll' +$metadata = Join-Path $stage 'generated/base_translation_output.json' +Invoke-Checked $dotnet @($translator, 'translate-recursive', '0x80001000', '--project', $manifest, + '--output-metadata', $metadata, '--threads', "$Parallel") ` + 'Translating the synthetic DOL' +# Function-map seeds can be skipped by discovery; do not accept a partial fixture. +$translated = Get-Content -LiteralPath $metadata -Raw | ConvertFrom-Json +foreach ($address in @($entry) + $guestCallbacks) { + if ($address -notin $translated.functions.entryPoint) { + throw ('Synthetic function 0x{0:X8} was not translated.' -f $address) + } +} +Invoke-Checked $dotnet @($translator, 'generate-data-init', '--project', $manifest) 'Generating synthetic data and runtime configuration' +Invoke-Checked $dotnet @($translator, 'emit-build-shards', '--project', $manifest) 'Emitting the production build graph' + +$nativeBuild = Join-Path $stage 'native-build' +$oldPath = $env:PATH +try { + $env:PATH = Get-MkwToolchainPath $portableTools + $configure = Get-MkwNativeConfigureArguments -SourceDirectory (Join-Path $stage 'runtime') -BuildDirectory $nativeBuild ` + -Ninja $ninja -CCompiler (Join-Path $compilerBin 'x86_64-w64-mingw32-clang.exe') ` + -CxxCompiler (Join-Path $compilerBin 'x86_64-w64-mingw32-clang++.exe') ` + -ResourceCompiler (Join-Path $compilerBin 'x86_64-w64-mingw32-windres.exe') ` + -DependenciesDirectory $dependencies -AdditionalArguments @('-DMKW_BUILD_PRODUCTS=ON') + Invoke-Checked $cmake $configure 'Configuring the production Windows runtime' + Invoke-Checked $cmake @('--build', $nativeBuild, '--target', 'WiiCompiled', '--parallel', "$Parallel") ` + 'Compiling and linking the synthetic product with the full runtime' + Assert-File (Join-Path $nativeBuild 'WiiCompiled.exe') 'Linked synthetic product' +} finally { + $env:PATH = $oldPath +} +Write-Host 'Synthetic recompilation passed (translation, data generation, runtime compilation, and product link).' diff --git a/Launcher/prepare-portable-tools.sh b/Launcher/prepare-portable-tools.sh index 8e812ff..69ea00d 100755 --- a/Launcher/prepare-portable-tools.sh +++ b/Launcher/prepare-portable-tools.sh @@ -201,10 +201,10 @@ Ninja $ninja_version Apache License 2.0 EOF -echo "prepare-portable-tools.sh: smoke-testing the toolchain..." -smoke_dir=$(mktemp -d) -trap 'rm -rf "$smoke_dir"' EXIT -cat > "$smoke_dir/t.cpp" <<'EOF' +echo "prepare-portable-tools.sh: testing the toolchain..." +test_dir=$(mktemp -d) +trap 'rm -rf "$test_dir"' EXIT +cat > "$test_dir/t.cpp" <<'EOF' #include #include int main() { @@ -214,23 +214,23 @@ int main() { return sum == 6 ? 0 : 1; } EOF -"$work/bin/clang++" -std=c++20 -fuse-ld=lld "$smoke_dir/t.cpp" -o "$smoke_dir/t" -"$smoke_dir/t" +"$work/bin/clang++" -std=c++20 -fuse-ld=lld "$test_dir/t.cpp" -o "$test_dir/t" +"$test_dir/t" # Also exercised together through CMake+Ninja, exactly how local-build.sh drives them - a plain # clang++ invocation above would not catch a broken CMAKE_ROOT (Modules/Templates) or a Ninja that # can't find the compiler. -cat > "$smoke_dir/CMakeLists.txt" <<'EOF' +cat > "$test_dir/CMakeLists.txt" <<'EOF' cmake_minimum_required(VERSION 3.16) -project(smoke CXX) -add_executable(smoke t.cpp) +project(test CXX) +add_executable(test t.cpp) EOF -"$work/bin/cmake" -S "$smoke_dir" -B "$smoke_dir/build" -G Ninja \ +"$work/bin/cmake" -S "$test_dir" -B "$test_dir/build" -G Ninja \ -DCMAKE_MAKE_PROGRAM="$work/bin/ninja" -DCMAKE_CXX_COMPILER="$work/bin/clang++" >/dev/null -"$work/bin/cmake" --build "$smoke_dir/build" >/dev/null -"$smoke_dir/build/smoke" +"$work/bin/cmake" --build "$test_dir/build" >/dev/null +"$test_dir/build/test" -rm -rf "$smoke_dir" +rm -rf "$test_dir" trap - EXIT mv "$work" "$toolchain_dir"