Add synthetic recompilation CI workflow (#161)

This commit is contained in:
patchzyy
2026-09-05 23:00:47 +02:00
committed by GitHub
parent d1d80613cc
commit 1c0a3edee9
5 changed files with 209 additions and 14 deletions
+4
View File
@@ -14,6 +14,10 @@ concurrency:
cancel-in-progress: true
jobs:
recompilation:
name: Recompilation test
uses: ./.github/workflows/recomp-test.yml
translator:
name: Translator (build + test)
runs-on: windows-latest
+5 -1
View File
@@ -20,6 +20,10 @@ concurrency:
cancel-in-progress: true
jobs:
recompilation:
name: Recompilation test
uses: ./.github/workflows/recomp-test.yml
linux-appimage:
name: Linux (AppImage, ${{ matrix.arch }})
strategy:
@@ -93,7 +97,7 @@ jobs:
release:
name: Publish GitHub Release
if: startsWith(github.ref, 'refs/tags/v')
needs: [linux-appimage, windows-installer]
needs: [linux-appimage, windows-installer, recompilation]
runs-on: ubuntu-latest
permissions:
contents: write
+41
View File
@@ -0,0 +1,41 @@
name: Synthetic recompilation
on:
workflow_call:
workflow_dispatch:
permissions:
contents: read
jobs:
windows:
name: Windows runtime (synthetic DOL)
runs-on: windows-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-dotnet@v6
with:
dotnet-version: '8.0.x'
# Cache downloads only. Preparation still validates pins, and every run
# compiles current Aurora, runtime, and generated sources from scratch.
- uses: actions/cache@v5
with:
path: Launcher/artifacts/downloads
key: windows-recomp-downloads-${{ hashFiles('Launcher/Prepare-PortableTools.ps1', 'Launcher/Prepare-Dependencies.ps1') }}
- name: Prepare the shipped Windows toolchain
shell: pwsh
run: ./Launcher/Prepare-PortableTools.ps1
- name: Prepare pinned native dependencies
shell: pwsh
run: ./Launcher/Prepare-Dependencies.ps1
- name: Translate, compile the full runtime, and link
shell: pwsh
run: ./Launcher/Test-Recompilation.ps1 -Parallel 3
+146
View File
@@ -0,0 +1,146 @@
# Build the real Windows runtime with translated, entirely synthetic PowerPC code.
# No game dump, game symbol map, REL, mod download, or existing generated/ output is used.
[CmdletBinding()]
param(
[string]$PortableToolsDirectory = 'Launcher/artifacts/portable-tools',
[string]$DependencySourceDirectory = 'Launcher/artifacts/dependencies',
[string]$StageDirectory = 'build/recomp-test',
[ValidateRange(1, 64)] [int]$Parallel = 3
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 3.0
. (Join-Path $PSScriptRoot 'NativeBuildFlags.ps1')
$repoRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
function Full([string]$Path) {
if ([IO.Path]::IsPathRooted($Path)) { return [IO.Path]::GetFullPath($Path) }
return [IO.Path]::GetFullPath((Join-Path $repoRoot $Path))
}
$portableTools = Full $PortableToolsDirectory
$dependencies = Full $DependencySourceDirectory
$stage = Full $StageDirectory
$dotnet = (Get-Command dotnet -CommandType Application).Source
$cmake = Join-Path $portableTools 'CMake/bin/cmake.exe'
$ninja = Join-Path $portableTools 'Ninja/ninja.exe'
$compilerBin = Join-Path $portableTools 'llvm-mingw/bin'
Assert-File $cmake 'Pinned CMake (run Prepare-PortableTools.ps1 first)'
Assert-File $ninja 'Pinned Ninja'
Assert-File (Join-Path $dependencies 'cppwinrt/winrt/base.h') 'Pinned dependencies (run Prepare-Dependencies.ps1 first)'
# Refuse reuse so a developer's game translation or an earlier build cannot make
# the test pass. Keep the staging tree after the run for diagnostics.
if (Test-Path -LiteralPath $stage) { throw "Test stage already exists; choose a fresh -StageDirectory: $stage" }
[IO.Directory]::CreateDirectory($stage) | Out-Null
Write-Host "Synthetic recompilation workspace: $stage"
# Copy current sources, including uncommitted edits, but no ignored build output.
# An isolated workspace preserves the developer's real generated/ directory.
$sourceFiles = & git -C $repoRoot -c core.quotepath=false ls-files --cached --others --exclude-standard -- runtime aurora-main
if ($LASTEXITCODE -ne 0) { throw 'Could not enumerate runtime and Aurora sources.' }
foreach ($relative in $sourceFiles | Sort-Object -Unique) {
$destination = Join-Path $stage $relative
[IO.Directory]::CreateDirectory([IO.Path]::GetDirectoryName($destination)) | Out-Null
Copy-Item -LiteralPath (Join-Path $repoRoot $relative) -Destination $destination
}
# One synthetic text section: li r3,40; addi r3,r3,2; nop; blr.
# Native HLE wrappers also call these eight guest symbols directly. Give each
# its own generated blr function so the real product can link without game code.
# Keep this list explicit: a new unresolved guest dependency must fail the test.
[uint32]$entry = 0x80001000L
[uint32[]]$guestCallbacks = @(
0x8012B830L, 0x801A0620L, 0x801A1ED8L, 0x801A961CL,
0x801AADE0L, 0x801D8D30L, 0x801D9E94L, 0x8055531CL
)
$textSize = [int]($guestCallbacks[-1] - $entry + 4)
$dataOffset = 0x100 + $textSize
$dol = [byte[]]::new($dataOffset + 4)
function Write-BigEndian32([int]$Offset, [uint32]$Value) {
$dol[$Offset] = [byte](($Value -shr 24) -band 255)
$dol[$Offset + 1] = [byte](($Value -shr 16) -band 255)
$dol[$Offset + 2] = [byte](($Value -shr 8) -band 255)
$dol[$Offset + 3] = [byte]($Value -band 255)
}
Write-BigEndian32 0x00 0x100 # text[0] file offset
Write-BigEndian32 0x48 $entry # text[0] guest address
Write-BigEndian32 0x90 $textSize # text[0] length (unreachable gaps are zero)
Write-BigEndian32 0x1C $dataOffset # data[0] file offset
Write-BigEndian32 0x64 0x80600000L # data[0] guest address
Write-BigEndian32 0xAC 4 # data[0] length
Write-BigEndian32 0xD8 0x80601000L # BSS address
Write-BigEndian32 0xDC 32 # BSS length
Write-BigEndian32 0xE0 $entry # entry point
Write-BigEndian32 0x100 0x38600028 # li r3,40
Write-BigEndian32 0x104 0x38630002 # addi r3,r3,2
Write-BigEndian32 0x108 0x60000000 # nop
Write-BigEndian32 0x10C 0x4E800020 # blr
foreach ($address in $guestCallbacks) {
Write-BigEndian32 ([int](0x100 + $address - $entry)) 0x4E800020
}
Write-BigEndian32 $dataOffset 0x12345678
[IO.File]::WriteAllBytes((Join-Path $stage 'synthetic.dol'), $dol)
$entryPoints = (@($entry) + $guestCallbacks | ForEach-Object { '0x{0:X8}' -f $_ }) -join ', '
$functionMap = (@($entry) + $guestCallbacks | ForEach-Object { '{0:X8} func_{0:X8}' -f $_ }) -join "`n"
[IO.File]::WriteAllText((Join-Path $stage 'synthetic-functions.txt'), $functionMap)
$manifest = Join-Path $stage 'recomp.yml'
[IO.File]::WriteAllText($manifest, @"
schema_version: 1
workspace_root: .
project:
id: ci-synthetic-dol
display_name: CI Synthetic DOL
memory:
base: 0x80000000
size: 0x01800000
sda_base: 0x80600000
sda2_base: 0x80600000
inputs:
dol:
path: synthetic.dol
translation:
entry_points: [$entryPoints]
function_map:
path: synthetic-functions.txt
allow_unsupported_instructions: false
runtime:
native_abi_directories: []
native_registration_root: runtime/src
output:
root: generated
"@)
$translatorProject = Join-Path $repoRoot 'translator/src/Translator.Cli/Translator.Cli.csproj'
Invoke-Checked $dotnet @('build', $translatorProject, '-c', 'Release', '--disable-build-servers') 'Building the translator'
$translator = Join-Path $repoRoot 'translator/src/Translator.Cli/bin/Release/net8.0/Translator.Cli.dll'
$metadata = Join-Path $stage 'generated/base_translation_output.json'
Invoke-Checked $dotnet @($translator, 'translate-recursive', '0x80001000', '--project', $manifest,
'--output-metadata', $metadata, '--threads', "$Parallel") `
'Translating the synthetic DOL'
# Function-map seeds can be skipped by discovery; do not accept a partial fixture.
$translated = Get-Content -LiteralPath $metadata -Raw | ConvertFrom-Json
foreach ($address in @($entry) + $guestCallbacks) {
if ($address -notin $translated.functions.entryPoint) {
throw ('Synthetic function 0x{0:X8} was not translated.' -f $address)
}
}
Invoke-Checked $dotnet @($translator, 'generate-data-init', '--project', $manifest) 'Generating synthetic data and runtime configuration'
Invoke-Checked $dotnet @($translator, 'emit-build-shards', '--project', $manifest) 'Emitting the production build graph'
$nativeBuild = Join-Path $stage 'native-build'
$oldPath = $env:PATH
try {
$env:PATH = Get-MkwToolchainPath $portableTools
$configure = Get-MkwNativeConfigureArguments -SourceDirectory (Join-Path $stage 'runtime') -BuildDirectory $nativeBuild `
-Ninja $ninja -CCompiler (Join-Path $compilerBin 'x86_64-w64-mingw32-clang.exe') `
-CxxCompiler (Join-Path $compilerBin 'x86_64-w64-mingw32-clang++.exe') `
-ResourceCompiler (Join-Path $compilerBin 'x86_64-w64-mingw32-windres.exe') `
-DependenciesDirectory $dependencies -AdditionalArguments @('-DMKW_BUILD_PRODUCTS=ON')
Invoke-Checked $cmake $configure 'Configuring the production Windows runtime'
Invoke-Checked $cmake @('--build', $nativeBuild, '--target', 'WiiCompiled', '--parallel', "$Parallel") `
'Compiling and linking the synthetic product with the full runtime'
Assert-File (Join-Path $nativeBuild 'WiiCompiled.exe') 'Linked synthetic product'
} finally {
$env:PATH = $oldPath
}
Write-Host 'Synthetic recompilation passed (translation, data generation, runtime compilation, and product link).'
+13 -13
View File
@@ -201,10 +201,10 @@ Ninja $ninja_version
Apache License 2.0
EOF
echo "prepare-portable-tools.sh: smoke-testing the toolchain..."
smoke_dir=$(mktemp -d)
trap 'rm -rf "$smoke_dir"' EXIT
cat > "$smoke_dir/t.cpp" <<'EOF'
echo "prepare-portable-tools.sh: testing the toolchain..."
test_dir=$(mktemp -d)
trap 'rm -rf "$test_dir"' EXIT
cat > "$test_dir/t.cpp" <<'EOF'
#include <vector>
#include <cstdio>
int main() {
@@ -214,23 +214,23 @@ int main() {
return sum == 6 ? 0 : 1;
}
EOF
"$work/bin/clang++" -std=c++20 -fuse-ld=lld "$smoke_dir/t.cpp" -o "$smoke_dir/t"
"$smoke_dir/t"
"$work/bin/clang++" -std=c++20 -fuse-ld=lld "$test_dir/t.cpp" -o "$test_dir/t"
"$test_dir/t"
# Also exercised together through CMake+Ninja, exactly how local-build.sh drives them - a plain
# clang++ invocation above would not catch a broken CMAKE_ROOT (Modules/Templates) or a Ninja that
# can't find the compiler.
cat > "$smoke_dir/CMakeLists.txt" <<'EOF'
cat > "$test_dir/CMakeLists.txt" <<'EOF'
cmake_minimum_required(VERSION 3.16)
project(smoke CXX)
add_executable(smoke t.cpp)
project(test CXX)
add_executable(test t.cpp)
EOF
"$work/bin/cmake" -S "$smoke_dir" -B "$smoke_dir/build" -G Ninja \
"$work/bin/cmake" -S "$test_dir" -B "$test_dir/build" -G Ninja \
-DCMAKE_MAKE_PROGRAM="$work/bin/ninja" -DCMAKE_CXX_COMPILER="$work/bin/clang++" >/dev/null
"$work/bin/cmake" --build "$smoke_dir/build" >/dev/null
"$smoke_dir/build/smoke"
"$work/bin/cmake" --build "$test_dir/build" >/dev/null
"$test_dir/build/test"
rm -rf "$smoke_dir"
rm -rf "$test_dir"
trap - EXIT
mv "$work" "$toolchain_dir"