Compare commits

..

6 Commits

Author SHA1 Message Date
patchzyy f02d000ba8 update to 0.9.3 2026-10-25 14:01:50 +01:00
patchzyy 814aa304bb keep audio poll time remainder (#360) 2026-10-25 13:41:53 +01:00
patchzyy 48aa27d77e bump aurora for instant resolution changes 2026-10-25 12:51:59 +01:00
patchzyy 27e9be32ed let wrapper natives run mod overlays (#359) 2026-10-25 12:47:16 +01:00
patchzyy cc6abd49ab faster deterministic translation (#358) 2026-10-25 10:30:08 +01:00
patchzyy bb5e2b1166 Fix prebuilt aurora package being rejected (#357)
* fix prebuilt package rejected on user machines

* Time out a hung translator step

The translator can hang after a crash instead of exiting, and the
installer's clean retry runs only on a failed exit. Each translator
step now gets 1800 s; past that its process tree is killed and the
step fails, so the retry runs.

* time out mod awareness check too

---------

Co-authored-by: therenow9 <60898774+therenow9@users.noreply.github.com>
2026-10-25 09:47:03 +01:00
23 changed files with 943 additions and 531 deletions
+18
View File
@@ -278,6 +278,24 @@ $packagedRuntimeFingerprint = (Get-Content -LiteralPath (Join-Path $workspace 'D
if ($stagedRuntimeFingerprint -ne $packagedRuntimeFingerprint) {
throw "The staged runtime ($stagedRuntimeFingerprint) does not match the precompiled runtime archive ($packagedRuntimeFingerprint)."
}
# Users' builds run LocalBuild.ps1 under Windows PowerShell 5.1, whatever shell built this
# installer, so the staged package is accepted (or not) exactly the way theirs will be.
$prebuiltGate = Join-Path $workRoot 'native-prebuilt-gate.ps1'
@'
param([string]$Workspace, [string]$Toolkit)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 3.0
. (Join-Path $Workspace 'NativeBuildFlags.ps1')
$package = Resolve-MkwNativePrebuiltDirectory (Join-Path $Workspace 'Dependencies') `
(Join-Path $Toolkit 'llvm-mingw\bin') (Join-Path $Workspace 'aurora-main') (Join-Path $Workspace 'runtime\third_party')
if (-not $package) { exit 3 }
'@ | Set-Content -LiteralPath $prebuiltGate -Encoding UTF8
& (Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe') -NoLogo -NoProfile -NonInteractive `
-ExecutionPolicy Bypass -File $prebuiltGate -Workspace $workspace -Toolkit $toolkit
if ($LASTEXITCODE -ne 0) {
throw "Windows PowerShell rejects the staged precompiled aurora package (exit $LASTEXITCODE); every installation would build aurora from source."
}
Remove-Item -LiteralPath $prebuiltGate -Force
[IO.Directory]::CreateDirectory((Join-Path $payloadRoot 'host')) | Out-Null
Copy-Item $setupHost (Join-Path $payloadRoot 'host\WiiCompiled-Setup.exe')
+1 -1
View File
@@ -1,7 +1,7 @@
<Project>
<PropertyGroup>
<!-- The one product version. Release tags must match it. -->
<Version>0.9.2</Version>
<Version>0.9.3</Version>
<IncludeSourceRevisionInInformationalVersion>false</IncludeSourceRevisionInInformationalVersion>
</PropertyGroup>
</Project>
+26 -71
View File
@@ -34,65 +34,6 @@ Set-StrictMode -Version 3.0
# See NativeBuildFlags.ps1; Build-Installer.ps1 stages it beside this script.
. (Join-Path $PSScriptRoot 'NativeBuildFlags.ps1')
function Resolve-NativePrebuiltDirectory([string]$DependenciesDirectory, [string]$ToolchainBin, [string]$AuroraDirectory, [string]$ThirdPartyDirectory) {
<#
The shipped precompiled aurora/third-party package, verified against its recorded provenance
(exact compiler, exact flags) before use. Anything unexpected falls back to building from source.
#>
$package = Join-Path $DependenciesDirectory 'native_prebuilt'
$provenancePath = Join-Path $package 'provenance.json'
if (-not (Test-Path -LiteralPath $provenancePath -PathType Leaf)) {
Write-Host 'MKWCBUILD: No precompiled aurora package was shipped; building it from source'
return ''
}
try {
$provenance = Get-Content -LiteralPath $provenancePath -Raw | ConvertFrom-Json
$compiler = Join-Path $ToolchainBin 'clang-22.exe'
$compilerHash = Get-MkwFileSha256 $compiler
if ($provenance.CompilerSha256 -ne $compilerHash) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built by a different compiler; building it from source'
return ''
}
if ($provenance.FlagFingerprint -ne (Get-MkwNativeFlagFingerprint)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built with different flags; building it from source'
return ''
}
# The archives embed Dawn's headers and link its import library, so a
# dawn_prebuilt tree that is not the one they were harvested against is
# an ABI mismatch that only shows up as a crash inside the GPU backend.
$recordedDawn = $provenance.PSObject.Properties['DawnRuntimeSha256']
if ($null -eq $recordedDawn -or
$recordedDawn.Value -ne (Get-MkwDawnRuntimeSha256 $DependenciesDirectory)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built against a different Dawn; building it from source'
return ''
}
# The archives are aurora; the runtime compiled here calls its headers.
# If this workspace's aurora sources are not the ones that were harvested
# (a newer toolkit, or a locally modified tree), linking the package would
# mix two versions of aurora, so build it from source instead.
$recordedAurora = $provenance.PSObject.Properties['AuroraSourceFingerprint']
if ($null -eq $recordedAurora -or
$recordedAurora.Value -ne (Get-MkwAuroraSourceFingerprint $AuroraDirectory)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built from different aurora sources; building it from source'
return ''
}
# Same reasoning for the vendored Crypto++ the package also carries:
# first-party TUs compile against this workspace's third_party headers,
# so a package harvested from an older tree must not be linked here.
$recordedThirdParty = $provenance.PSObject.Properties['ThirdPartySourceFingerprint']
if ($null -eq $recordedThirdParty -or
$recordedThirdParty.Value -ne (Get-MkwThirdPartySourceFingerprint $ThirdPartyDirectory)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built from different vendored third-party sources; building it from source'
return ''
}
} catch {
Write-Host "MKWCBUILD: The precompiled aurora package could not be validated ($($_.Exception.Message)); building it from source"
return ''
}
Write-Host 'MKWCBUILD: Using the precompiled aurora and third-party libraries'
return $package
}
function Write-MkwBuildStep([string]$StepId, [string]$Message) {
<#
Announces a build step. The identifier is the contract (BuildStepIds in
@@ -160,6 +101,8 @@ if (-not [string]::IsNullOrWhiteSpace($RetroRewindPackageDirectory) -and
$RetroRewindPackageDirectory = $Workspace + $RetroRewindPackageDirectory.Substring($realWorkspace.Length)
}
$translator = Join-Path $Toolkit 'Translator\Translator.Cli.exe'
# A translator step takes 3-19 s on 16 threads; past this it has hung, and failing lets the clean retry run.
$translatorTimeoutSeconds = 1800
$toolchain = Get-MkwBuildSafePath $Toolkit 'toolchain' 'CMake\bin\cmake.exe'
$cmake = Join-Path $toolchain 'CMake\bin\cmake.exe'
$ninja = Join-Path $toolchain 'Ninja\ninja.exe'
@@ -208,8 +151,9 @@ if ($Parallel -gt 0) {
$globalJobs = $Parallel
} else {
$translatorThreads = [math]::Max(1, [math]::Min([Environment]::ProcessorCount, 16))
# A translated TU peaks around 0.26 GB in clang; 0.5 GiB each after a 2 GiB reserve.
$translatedJobs = [math]::Max(1, [math]::Min([Environment]::ProcessorCount,
[math]::Floor($memoryGiB / 2)))
[math]::Floor(($memoryGiB - 2) / 0.5)))
$globalJobs = [math]::Max($translatedJobs, [Environment]::ProcessorCount)
}
@@ -286,11 +230,18 @@ try {
# same translated functions produces the same base tree. Only the translator can
# decide that - it takes parsing the Kamek patch set against the recorded translated
# function ranges - and it fails closed, so anything but exit 0 retranslates.
& $translator @(
'check-base-mod-awareness', '--project', $project, '--profile', 'retro-rewind',
'--translation-output-metadata', $baseMetadata, '--code-pul', $retroCodePul
) | Write-Host
if ($LASTEXITCODE -ne 0) {
$modAware = $true
try {
Invoke-Checked $translator @(
'check-base-mod-awareness', '--project', $project, '--profile', 'retro-rewind',
'--translation-output-metadata', $baseMetadata, '--code-pul', $retroCodePul
) 'Checking whether the base translation covers the new Code.pul' `
-TimeoutSeconds $translatorTimeoutSeconds
} catch {
Write-Host "MKWCBUILD: $($_.Exception.Message)"
$modAware = $false
}
if (-not $modAware) {
Write-MkwBuildStep 'retranslate-base' 'The base translation is stale; retranslating the base game for the new Code.pul'
$reuseBase = $false
}
@@ -313,12 +264,14 @@ try {
'--outdir', $functions, '--output-metadata', $baseMetadata,
'--production-source-bundle', (Join-Path $generated 'base_translation_sources.bin'),
'--no-function-files', '--prune-stale', '--threads', $translatorThreads
) 'Translating the user-owned base game' -StepId 'translate-base'
) 'Translating the user-owned base game' -StepId 'translate-base' `
-TimeoutSeconds $translatorTimeoutSeconds
Invoke-Checked $translator @(
'emit-base-manifest', '--project', $project, '--out', $baseManifestDir,
'--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', $pins.Region
) 'Creating the local base translation manifest' -StepId 'emit-base-manifest'
) 'Creating the local base translation manifest' -StepId 'emit-base-manifest' `
-TimeoutSeconds $translatorTimeoutSeconds
if (-not [string]::IsNullOrWhiteSpace($TranslationFingerprint)) {
[ordered]@{ SchemaVersion = 1; TranslationFingerprint = $TranslationFingerprint; DolSha256 = $pins.DolSha256 } |
@@ -348,11 +301,12 @@ try {
throw 'Retro Rewind requires either -RetroWfcOfflineDirectory or -SkipRetroWfcPayload.'
}
Invoke-Checked $translator $translateModArguments 'Translating the selected Retro Rewind Code.pul' `
-StepId 'translate-mod'
-StepId 'translate-mod' -TimeoutSeconds $translatorTimeoutSeconds
}
Invoke-Checked $translator @('generate-data-init', '--project', $project) `
'Generating local game data initialization' -StepId 'generate-data-init'
'Generating local game data initialization' -StepId 'generate-data-init' `
-TimeoutSeconds $translatorTimeoutSeconds
$shardArgs = @(
'emit-build-shards', '--project', $project, '--base-metadata', $baseMetadata,
@@ -364,7 +318,8 @@ try {
$shardArgs += @('--resolved-profile', (Join-Path $retroOut 'resolved_dispatch_profile.json'),
'--retro-cpp-dir', (Join-Path $retroOut 'cpp'))
}
Invoke-Checked $translator $shardArgs 'Preparing local native build shards' -StepId 'emit-build-shards'
Invoke-Checked $translator $shardArgs 'Preparing local native build shards' -StepId 'emit-build-shards' `
-TimeoutSeconds $translatorTimeoutSeconds
# The build directory is kept whenever the exact toolchain that filled it is the one
# installed now and its CMake cache still belongs to this workspace path. Object files are
@@ -396,7 +351,7 @@ try {
}
}
$dependencies = Join-Path $Workspace 'Dependencies'
$nativePrebuilt = Resolve-NativePrebuiltDirectory $dependencies $toolchainBin `
$nativePrebuilt = Resolve-MkwNativePrebuiltDirectory $dependencies $toolchainBin `
(Join-Path $Workspace 'aurora-main') (Join-Path $Workspace 'runtime\third_party')
if ($X86CpuProfile -eq 'v2') {
Write-Host 'MKWCBUILD: This CPU does not support x86-64-v3; building for x86-64-v2'
+118 -54
View File
@@ -157,7 +157,8 @@ function Resolve-MkwProjectForAssets([string]$Workspace, [string]$AssetsDirector
}
function Invoke-Checked([string]$FilePath, [string[]]$Arguments, [string]$Description,
[string]$LogPrefix = 'MKWCBUILD', [string]$StepId = '', [bool]$WaitForProcessTree = $true) {
[string]$LogPrefix = 'MKWCBUILD', [string]$StepId = '', [bool]$WaitForProcessTree = $true,
[int]$TimeoutSeconds = 0) {
<#
Runs a build tool and turns a non-zero exit code into a described failure. By default,
Start-Process -Wait waits for the whole process tree, since a .NET single-file bundle host may
@@ -166,6 +167,7 @@ function Invoke-Checked([string]$FilePath, [string[]]$Arguments, [string]$Descri
Start-Process doesn't publish $LASTEXITCODE, so this sets it manually for callers that check it.
-StepId emits the machine-readable form the installer's progress bar consumes (BuildStepIds in
WiiCompiled.Setup/InstallProgress.cs); the human sentence stays on the same log line.
-TimeoutSeconds waits on the process itself and kills its tree when time runs out.
#>
if ($StepId) { Write-Host "${LogPrefix}:STEP:${StepId} $Description" }
else { Write-Host "${LogPrefix}: $Description" }
@@ -174,7 +176,17 @@ function Invoke-Checked([string]$FilePath, [string[]]$Arguments, [string]$Descri
# Backslashes directly before the closing quote would otherwise escape it.
'"' + ($_ -replace '(\\+)$', '$1$1') + '"'
})
if ($WaitForProcessTree) {
if ($TimeoutSeconds -gt 0) {
$process = Start-Process -FilePath $FilePath -ArgumentList $quotedArguments -NoNewWindow -PassThru
# Holding the handle keeps ExitCode readable once the process has exited.
$null = $process.Handle
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
# taskkill's stderr would end the script under 'Stop' before the message below.
try { & taskkill.exe /T /F /PID $process.Id 2>&1 | Out-Null } catch {}
throw "$Description did not finish within $TimeoutSeconds seconds and was stopped."
}
$exitCode = $process.ExitCode
} elseif ($WaitForProcessTree) {
$process = Start-Process -FilePath $FilePath -ArgumentList $quotedArguments `
-NoNewWindow -Wait -PassThru
$exitCode = $process.ExitCode
@@ -236,69 +248,121 @@ function Get-MkwDawnRuntimeSha256([string]$DependenciesDirectory) {
return Get-MkwFileSha256 $runtime
}
function Get-MkwThirdPartySourceFingerprint([string]$ThirdPartyDirectory) {
function ConvertTo-MkwHex([byte[]]$Bytes) {
return [BitConverter]::ToString($Bytes).Replace('-', '').ToLowerInvariant()
}
function Get-MkwSourceTreeFingerprint([string]$Directory, [string[]]$ExcludedTopDirectories = @()) {
<#
SHA-256 over runtime/third_party, the vendored sources behind the prebuilt package. The
precompiled Crypto++ archive is linked while first-party TUs compile against the same headers,
so a workspace whose vendored tree moved on must not consume archives harvested from an older
one (same reasoning as the aurora fingerprint below), skipping dot-paths by name the same way.
Returns '' when absent.
SHA-256 over "relative-path content-sha256" lines of a source tree. It must come out the same
under powershell.exe 5.1 and pwsh 7 and for any checkout's line endings, because the package is
harvested on a release machine and re-checked on every user's: paths are sorted ordinally (not
with Sort-Object, whose culture collation differs between NLS and ICU) and text content is hashed
with CRLF folded to LF. Dot-paths are skipped by name, not by the hidden attribute, which the
payload zip drops. Returns '' when the tree is absent.
#>
if ([string]::IsNullOrWhiteSpace($ThirdPartyDirectory)) { return '' }
if (-not (Test-Path -LiteralPath $ThirdPartyDirectory -PathType Container)) { return '' }
$root = [IO.Path]::GetFullPath($ThirdPartyDirectory).TrimEnd('\')
$files = Get-ChildItem -LiteralPath $root -Recurse -File -Force |
Where-Object { -not ($_.FullName.Substring($root.Length + 1) -split '\\' | Where-Object { $_.StartsWith('.') }) } |
Sort-Object { $_.FullName.Substring($root.Length + 1).Replace('\', '/') }
if ([string]::IsNullOrWhiteSpace($Directory)) { return '' }
if (-not (Test-Path -LiteralPath $Directory -PathType Container)) { return '' }
$root = [IO.Path]::GetFullPath($Directory).TrimEnd('\')
$relatives = [Collections.Generic.List[string]]::new()
foreach ($file in [IO.Directory]::EnumerateFiles($root, '*', [IO.SearchOption]::AllDirectories)) {
$relative = $file.Substring($root.Length + 1).Replace('\', '/')
$parts = $relative.Split('/')
if ($parts.Length -gt 1 -and $ExcludedTopDirectories -contains $parts[0]) { continue }
$hidden = $false
foreach ($part in $parts) { if ($part.StartsWith('.', [StringComparison]::Ordinal)) { $hidden = $true; break } }
if (-not $hidden) { $relatives.Add($relative) }
}
$sorted = $relatives.ToArray()
[Array]::Sort($sorted, [StringComparer]::Ordinal)
$latin1 = [Text.Encoding]::GetEncoding(28591)
$sha = [Security.Cryptography.SHA256]::Create()
try {
$builder = [Text.StringBuilder]::new()
foreach ($file in $files) {
$relative = $file.FullName.Substring($root.Length + 1).Replace('\', '/')
$content = Get-MkwFileSha256 $file.FullName
[void]$builder.Append($relative).Append(' ').Append($content).Append("`n")
foreach ($relative in $sorted) {
$bytes = [IO.File]::ReadAllBytes($root + '\' + $relative.Replace('/', '\'))
# Latin-1 maps bytes 1:1, so the round trip only changes the line endings.
$text = $latin1.GetString($bytes)
if ($text.IndexOf([char]0) -lt 0 -and $text.Contains("`r`n")) {
$bytes = $latin1.GetBytes($text.Replace("`r`n", "`n"))
}
[void]$builder.Append($relative).Append(' ').Append((ConvertTo-MkwHex $sha.ComputeHash($bytes))).Append("`n")
}
$bytes = $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($builder.ToString()))
return ConvertTo-MkwHex $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($builder.ToString()))
} finally { $sha.Dispose() }
return (($bytes | ForEach-Object { $_.ToString('x2') }) -join '')
}
function Get-MkwThirdPartySourceFingerprint([string]$ThirdPartyDirectory) {
<#
runtime/third_party, the vendored sources behind the prebuilt package. The precompiled Crypto++
archive is linked while first-party TUs compile against the same headers, so a workspace whose
vendored tree moved on must not consume archives harvested from an older one.
#>
return Get-MkwSourceTreeFingerprint $ThirdPartyDirectory
}
function Get-MkwAuroraSourceFingerprint([string]$AuroraDirectory) {
<#
SHA-256 over the aurora sources the precompiled archives were harvested from. The compiler,
flag, and Dawn digests only prove the package agrees with its target environment; none notice
aurora itself moving on, and since the runtime calls aurora headers directly, a package
harvested before an aurora change links old code against new call sites (undefined symbol at
best, silently stale renderer at worst). extern/ is excluded because the payload ships those
trees separately. Returns '' when the tree is absent.
The aurora sources the precompiled archives were harvested from. The compiler, flag, and Dawn
digests only prove the package agrees with its target environment; none notice aurora itself
moving on, and since the runtime calls aurora headers directly, a package harvested before an
aurora change links old code against new call sites. extern/ is excluded because the payload
ships those trees separately.
#>
if ([string]::IsNullOrWhiteSpace($AuroraDirectory)) { return '' }
if (-not (Test-Path -LiteralPath $AuroraDirectory -PathType Container)) { return '' }
$root = [IO.Path]::GetFullPath($AuroraDirectory).TrimEnd('\')
$excludedPrefixes = @(
(Join-Path $root 'extern') + '\',
(Join-Path $root 'build') + '\'
)
# Dot-paths (the submodule's .git pointer) are skipped by name, not by the hidden attribute:
# the payload zip drops that attribute, so the installed side would otherwise hash them.
$files = Get-ChildItem -LiteralPath $root -Recurse -File -Force |
Where-Object {
$full = $_.FullName
-not ($excludedPrefixes | Where-Object { $full.StartsWith($_, [StringComparison]::OrdinalIgnoreCase) }) -and
-not ($full.Substring($root.Length + 1) -split '\\' | Where-Object { $_.StartsWith('.') })
} |
Sort-Object { $_.FullName.Substring($root.Length + 1).Replace('\', '/') }
$sha = [Security.Cryptography.SHA256]::Create()
return Get-MkwSourceTreeFingerprint $AuroraDirectory @('extern', 'build')
}
function Resolve-MkwNativePrebuiltDirectory([string]$DependenciesDirectory, [string]$ToolchainBin,
[string]$AuroraDirectory, [string]$ThirdPartyDirectory) {
<#
The shipped precompiled aurora/third-party package, verified against its recorded provenance
(exact compiler, exact flags, sources) before use. Anything unexpected falls back to building
from source. Shared with Build-Installer.ps1, which runs it against the staged payload under
powershell.exe so a package users would reject never ships.
#>
$package = Join-Path $DependenciesDirectory 'native_prebuilt'
$provenancePath = Join-Path $package 'provenance.json'
if (-not (Test-Path -LiteralPath $provenancePath -PathType Leaf)) {
Write-Host 'MKWCBUILD: No precompiled aurora package was shipped; building it from source'
return ''
}
try {
$builder = [Text.StringBuilder]::new()
foreach ($file in $files) {
$relative = $file.FullName.Substring($root.Length + 1).Replace('\', '/')
$content = Get-MkwFileSha256 $file.FullName
[void]$builder.Append($relative).Append(' ').Append($content).Append("`n")
$provenance = Get-Content -LiteralPath $provenancePath -Raw | ConvertFrom-Json
if ($provenance.CompilerSha256 -ne (Get-MkwFileSha256 (Join-Path $ToolchainBin 'clang-22.exe'))) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built by a different compiler; building it from source'
return ''
}
$bytes = $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($builder.ToString()))
} finally { $sha.Dispose() }
return (($bytes | ForEach-Object { $_.ToString('x2') }) -join '')
if ($provenance.FlagFingerprint -ne (Get-MkwNativeFlagFingerprint)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built with different flags; building it from source'
return ''
}
# The archives embed Dawn's headers and link its import library; a different Dawn is an
# ABI mismatch that only shows up as a crash inside the GPU backend.
$recordedDawn = $provenance.PSObject.Properties['DawnRuntimeSha256']
if ($null -eq $recordedDawn -or
$recordedDawn.Value -ne (Get-MkwDawnRuntimeSha256 $DependenciesDirectory)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built against a different Dawn; building it from source'
return ''
}
$recordedAurora = $provenance.PSObject.Properties['AuroraSourceFingerprint']
if ($null -eq $recordedAurora -or
$recordedAurora.Value -ne (Get-MkwAuroraSourceFingerprint $AuroraDirectory)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built from different aurora sources; building it from source'
return ''
}
$recordedThirdParty = $provenance.PSObject.Properties['ThirdPartySourceFingerprint']
if ($null -eq $recordedThirdParty -or
$recordedThirdParty.Value -ne (Get-MkwThirdPartySourceFingerprint $ThirdPartyDirectory)) {
Write-Host 'MKWCBUILD: The precompiled aurora package was built from different vendored third-party sources; building it from source'
return ''
}
} catch {
Write-Host "MKWCBUILD: The precompiled aurora package could not be validated ($($_.Exception.Message)); building it from source"
return ''
}
Write-Host 'MKWCBUILD: Using the precompiled aurora and third-party libraries'
return $package
}
function ConvertTo-MkwCMakePath([string]$Path) {
@@ -347,9 +411,9 @@ function Get-MkwNativeConfigureArguments {
}
}
if (-not [string]::IsNullOrWhiteSpace($NativePrebuiltDirectory)) {
$arguments += "-DMKW_NATIVE_PREBUILT_DIR=$(ConvertTo-MkwCMakePath $NativePrebuiltDirectory)"
}
# Always set, so a reused CMake cache can never keep a package this run rejected.
$arguments += if ([string]::IsNullOrWhiteSpace($NativePrebuiltDirectory)) { '-DMKW_NATIVE_PREBUILT_DIR=' }
else { "-DMKW_NATIVE_PREBUILT_DIR=$(ConvertTo-MkwCMakePath $NativePrebuiltDirectory)" }
$arguments += $AdditionalArguments
return $arguments
+7 -1
View File
@@ -157,7 +157,7 @@ public:
#define MKW_DETAIL_CAT(a, b) a##b
#define MKW_DETAIL_MAKE_UNIQUE(a, b) MKW_DETAIL_CAT(a, b)
// Three hand-written registration macros, given the PAL address. Generated code never registers
// Hand-written registration macros, given the PAL address. Generated code never registers
// per function; every translated function reaches the registry via the bulk
// BulkTranslatedFunctionRecord tables in the translator's *_registration TUs.
#define REGISTER_TRANSLATED_FUNCTION(address, fn) \
@@ -168,3 +168,9 @@ public:
#define REGISTER_NATIVE_FUNCTION_AS(address, fn, pretty_name) \
static AbiTrampoline<decltype(fn)> MKW_DETAIL_MAKE_UNIQUE(_abi_native_trampoline_named_, __COUNTER__)(::GuestRegion::Translate(address), pretty_name, fn, FunctionKind::Native, false, kPpcAllNonvolatileFprMask, 0, 0, &AbiRawCpuThunk<&fn>::Invoke)
// A native that calls through with MKW_GUEST_NEXT, so a mod overlay at its address still runs.
#define REGISTER_NATIVE_WRAPPER_AS(address, fn, pretty_name) REGISTER_NATIVE_FUNCTION_AS(address, fn, pretty_name)
// A native that deliberately takes over its address from any mod overlay there.
#define REGISTER_NATIVE_REPLACEMENT_AS(address, fn, pretty_name) REGISTER_NATIVE_FUNCTION_AS(address, fn, pretty_name)
+12
View File
@@ -3,6 +3,8 @@
// Base translations the runtime calls directly, named by PAL identity and resolved through the
// function registry on first use.
// MKW_GUEST_FUNC(801A0620)(ctx);
// A native registered with REGISTER_NATIVE_WRAPPER_AS calls through with MKW_GUEST_NEXT instead,
// so a mod overlay at the same address still runs.
#include "guest_abi/cpu_context.h"
#include "guest_abi/guest_region.h"
@@ -12,6 +14,10 @@ using Entry = void (*)(CpuContext*);
// The base translation at a guest address, also where a native or mod overrides it.
Entry BaseTranslation(uint32_t address);
// The translation a native wrapper sits on top of: the winning mod overlay, else the base
// translation.
Entry NextTranslation(uint32_t address);
} // namespace GuestFunctions
#define MKW_GUEST_FUNC(pal) \
@@ -19,3 +25,9 @@ Entry BaseTranslation(uint32_t address);
static const ::GuestFunctions::Entry entry = ::GuestFunctions::BaseTranslation(MKW_GADDR(pal)); \
return entry; \
}())
#define MKW_GUEST_NEXT(pal) \
(*[] { \
static const ::GuestFunctions::Entry entry = ::GuestFunctions::NextTranslation(MKW_GADDR(pal)); \
return entry; \
}())
+41 -14
View File
@@ -426,20 +426,10 @@ const TranslatedFunctionInfo* TranslatedFunctionRegistry::FindByAddressPtrSlow(u
return nullptr;
}
GuestFunctions::Entry GuestFunctions::BaseTranslation(uint32_t address) {
std::unique_lock<std::mutex> lock(RegistryMutex(), std::defer_lock);
if (!RegistryFrozen().load(std::memory_order_acquire)) {
lock.lock();
}
for (const auto& entry : Registry()) {
if (entry.address == address && entry.kind == FunctionKind::BaseTranslated && entry.rawCpuInvoker) {
return entry.rawCpuInvoker;
}
}
if (lock.owns_lock()) {
lock.unlock();
}
RT_LOG(RT_TAG_RUNTIME) << "ERROR: no base translation registered at 0x" << std::hex << address << std::dec
namespace {
[[noreturn]] void ReportMissingGuestFunction(uint32_t address) {
RT_LOG(RT_TAG_RUNTIME) << "ERROR: no translation registered at 0x" << std::hex << address << std::dec
<< std::endl;
std::ostringstream message;
message << "The runtime calls the translated function at guest address 0x" << std::hex << address
@@ -451,6 +441,43 @@ GuestFunctions::Entry GuestFunctions::BaseTranslation(uint32_t address) {
std::exit(EXIT_FAILURE);
}
} // namespace
GuestFunctions::Entry GuestFunctions::BaseTranslation(uint32_t address) {
{
std::unique_lock<std::mutex> lock(RegistryMutex(), std::defer_lock);
if (!RegistryFrozen().load(std::memory_order_acquire)) {
lock.lock();
}
for (const auto& entry : Registry()) {
if (entry.address == address && entry.kind == FunctionKind::BaseTranslated && entry.rawCpuInvoker) {
return entry.rawCpuInvoker;
}
}
}
ReportMissingGuestFunction(address);
}
GuestFunctions::Entry GuestFunctions::NextTranslation(uint32_t address) {
{
std::unique_lock<std::mutex> lock(RegistryMutex(), std::defer_lock);
if (!RegistryFrozen().load(std::memory_order_acquire)) {
lock.lock();
}
const TranslatedFunctionInfo* best = nullptr;
for (const auto& entry : Registry()) {
if (entry.address == address && entry.kind != FunctionKind::Native && entry.rawCpuInvoker &&
(!best || IsBetterCandidate(entry, *best))) {
best = &entry;
}
}
if (best) {
return best->rawCpuInvoker;
}
}
ReportMissingGuestFunction(address);
}
const RawDispatchRecord* TranslatedFunctionRegistry::FindRawByAddressPtrSlow(uint32_t address) {
if (!RegistryFrozen().load(std::memory_order_acquire)) {
return nullptr;
+11 -10
View File
@@ -503,19 +503,20 @@ int64_t ConsumeAudioPollDeltaMicros()
using Clock = std::chrono::steady_clock;
static thread_local Clock::time_point lastPoll = Clock::now();
const Clock::time_point now = Clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - lastPoll).count();
lastPoll = now;
if (elapsed < 0) {
elapsed = 0;
}
// Cap the catch-up interval after a debugger pause or host stall; backlog drains via
// kMaxBlocksPerTick per pass. Never return early on a zero delta: two scheduler passes
// can land in the same microsecond and the backlog still needs servicing.
constexpr int64_t kMaxPollDeltaMicros = 100'000;
return std::min(elapsed, kMaxPollDeltaMicros);
constexpr std::chrono::microseconds kMaxPollDelta{100'000};
const Clock::time_point now = Clock::now();
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - lastPoll);
if (elapsed.count() < 0 || elapsed > kMaxPollDelta) {
lastPoll = now;
return elapsed.count() < 0 ? 0 : kMaxPollDelta.count();
}
// Keep the sub-microsecond remainder for the next poll.
lastPoll += elapsed;
return elapsed.count();
}
} // namespace
+3 -3
View File
@@ -162,7 +162,7 @@ REGISTER_NATIVE_FUNCTION_AS(0x805B9344, ArgumentParser_Parse_805b9344, "Argument
// RacedataScenario::InitScreens; local racers keep no real controller channel.
extern "C" void RacedataScenario_InitScreens_8052f4e8(CpuContext* ctx) {
if (!g_cpuDriven) {
MKW_GUEST_FUNC(8052F4E8)(ctx);
MKW_GUEST_NEXT(8052F4E8)(ctx);
return;
}
const uint32_t players = ctx->gpr[3] + kPlayersOffset;
@@ -170,7 +170,7 @@ extern "C" void RacedataScenario_InitScreens_8052f4e8(CpuContext* ctx) {
for (uint32_t i = 0; i < kPlayerCount; ++i) {
channels[i] = Memory::Read8(players + i * kPlayerSize + kPlayerControllerChannelOffset);
}
MKW_GUEST_FUNC(8052F4E8)(ctx);
MKW_GUEST_NEXT(8052F4E8)(ctx);
for (uint32_t i = 0; i < kPlayerCount; ++i) {
const uint32_t player = players + i * kPlayerSize;
if (Memory::Read32(player + kPlayerTypeOffset) == kPlayerRealLocal) {
@@ -178,7 +178,7 @@ extern "C" void RacedataScenario_InitScreens_8052f4e8(CpuContext* ctx) {
}
}
}
REGISTER_NATIVE_FUNCTION_AS(0x8052F4E8, RacedataScenario_InitScreens_8052f4e8, "RacedataScenario_InitScreens_8052f4e8");
REGISTER_NATIVE_WRAPPER_AS(0x8052F4E8, RacedataScenario_InitScreens_8052f4e8, "RacedataScenario_InitScreens_8052f4e8");
// KartAIController::IsCPU
extern "C" void KartAIController_IsCPU_8072624c(CpuContext* ctx) {
+1 -1
View File
@@ -436,7 +436,7 @@ extern "C" void NAND_IOS_OpenBody_HLE_801938FC(CpuContext* ctx) {
ctx->gpr[3] = static_cast<uint32_t>(result);
ctx->gpr[1] = ctx->gpr[1] + 32u;
}
REGISTER_NATIVE_FUNCTION_AS(0x801938FC, NAND_IOS_OpenBody_HLE_801938FC, "NAND_IOS_OpenBody_HLE_801938FC");
REGISTER_NATIVE_REPLACEMENT_AS(0x801938FC, NAND_IOS_OpenBody_HLE_801938FC, "NAND_IOS_OpenBody_HLE_801938FC");
extern "C" int32_t NAND_IOS_Close_HLE(uint32_t fd) {
if (fd == ISFS_DEV_FD) {
+131 -105
View File
@@ -139,6 +139,15 @@ TranslationOptions WithProjectFrontEndPolicy(
LeafInliningBlockedTargets = leafInliningBlockedTargets.Value
};
string? translateModInputCacheDirectory = null;
// translate-mod reads the same (large) base metadata from several places.
var baseMetadataCache = new Dictionary<string, BaseTranslationOutputMetadata>(StringComparer.OrdinalIgnoreCase);
BaseTranslationOutputMetadata ReadBaseMetadataCached(string path)
{
var fullPath = Path.GetFullPath(path);
if (!baseMetadataCache.TryGetValue(fullPath, out var metadata))
baseMetadataCache[fullPath] = metadata = BaseTranslationOutputMetadataFile.Read(fullPath);
return metadata;
}
string? translateModPublishedOutputDirectory = null;
List<AddressRange>? _executableRanges = null;
// Callback discovery now runs on the discovery worker pool, so the range cache
@@ -472,10 +481,6 @@ int RunTranslateRecursive(string[] argsTail)
var pruneStale = HasFlag(argsTail, "--prune-stale");
var threadsOption = OptionValue(argsTail, "--threads");
var maxThreads = threadsOption != null ? Math.Max(1, ParseInt(threadsOption)) : Math.Max(1, Environment.ProcessorCount);
// Discovery batches: cheap front-end-only work items, so the batch is sized
// generously to keep queue overhead off the hot path. Deliberately larger
// than the mod emission wave below, which carries whole C++ emissions.
var batchSize = Math.Max(64, maxThreads * 8);
Console.WriteLine($"[translator] Using up to {maxThreads} worker thread(s).");
// Log start time
@@ -493,7 +498,6 @@ int RunTranslateRecursive(string[] argsTail)
// callees are decoded separately and never appear here). Feeds the
// fall-through interior-entry detection for the base manifest.
var translatedFlowCoverage = new Dictionary<uint, uint[]>();
var gqrUnknownEntryRoots = new HashSet<uint> { startAddr };
var translatedFunctionEnds = new Dictionary<uint, uint>();
// Every map entry is a function start, so the whole boundary set is known
// before the first decode instead of accumulating as discovery proceeds.
@@ -565,13 +569,15 @@ int RunTranslateRecursive(string[] argsTail)
void ProcessQueue(bool speculative = false)
{
// Each round decodes the whole frontier: the output depends only on the final
// boundary set (boundary repair re-decodes stale owners), never on batching.
while (queue.Count > 0)
{
var batch = new List<(uint Address, int Depth, string Name)>(batchSize);
var batch = new List<(uint Address, int Depth, string Name)>(queue.Count);
var batchAddresses = new HashSet<uint>();
var candidates = QueueBatch.Dequeue(
queue,
batchSize,
queue.Count,
next => ShouldSkip(next) || !batchAddresses.Add(next.addr));
foreach (var next in candidates)
{
@@ -598,23 +604,19 @@ int RunTranslateRecursive(string[] argsTail)
}
}
// Batches drain one at a time so a callee of a prior speculative entry arrives with a known
// caller/GQR state instead of as another unknown root. Everything here is speculative, so a
// "call" decoded from an unreached map entry isn't proven code; treat failures as soft, not fatal.
// Everything here is speculative, so a "call" decoded from an unreached map entry
// isn't proven code; treat failures as soft, not fatal.
void ProcessSpeculativeSeeds()
{
while (speculativeSeeds.Count > 0)
{
var batch = new List<(uint Address, int Depth, string Name)>(batchSize);
var batch = new List<(uint Address, int Depth, string Name)>(speculativeSeeds.Count);
var candidates = QueueBatch.Dequeue(
speculativeSeeds,
batchSize,
speculativeSeeds.Count,
address => translated.Contains(address) || !visited.Add(address));
foreach (var address in candidates)
{
// Nothing calls this from the translated set, so its entry
// GQR state is unconstrained.
gqrUnknownEntryRoots.Add(address);
batch.Add((address, 0, ResolveFunctionName(address, preferred: null, $"func_{address:X8}")));
}
@@ -652,22 +654,42 @@ int RunTranslateRecursive(string[] argsTail)
// decoded out of one - is dropped with a counted warning.
void TranslateBatch(List<(uint Address, int Depth, string Name)> batch, bool speculative = false)
{
var results = new FunctionDiscoveryResult?[batch.Count];
IndexedParallel.For(batch.Count, parallelOptions, i => results[i] = Attempt(batch[i], speculative));
var metrics = new TranslationMetrics?[batch.Count];
var entryPoints = new uint[batch.Count];
var discoveryTargets = new uint[batch.Count][];
var swallowedSiblingEntries = new uint[batch.Count][];
var summaries = new GqrFunctionSummary[batch.Count];
var functionEnds = new uint[batch.Count];
var flowCoverage = new uint[batch.Count][];
// Everything kept from a decode is derived in the decode's own pass, so the full
// result is never retained; the few results a newly revealed boundary invalidates
// are simply decoded again. Commits stay serial and in batch order.
void Decode(int i, bool findSwallowedSiblings)
{
var result = Attempt(batch[i], speculative);
metrics[i] = result?.Metrics;
if (result is null)
{
canonicalIrStore.Remove(batch[i].Address);
return;
}
entryPoints[i] = result.EntryPoint;
if (findSwallowedSiblings)
swallowedSiblingEntries[i] = DiscoverSwallowedSiblingEntries(result).ToArray();
discoveryTargets[i] = DiscoverDiscoveryTranslationTargets(result).ToArray();
summaries[i] = GqrFunctionSummary.Create(result.Ssa.Function);
canonicalIrStore.Put(batch[i].Address, result.Ssa.Function);
functionEnds[i] = GetFunctionInstructionEnd(result);
flowCoverage[i] = CoalesceInstructionRanges(result);
}
IndexedParallel.For(batch.Count, parallelOptions, i => Decode(i, findSwallowedSiblings: true));
// Re-run only translations that swallowed a newly revealed sibling entry, or direct
// branches could bypass a mod overlay registered there. Both scans are pure per-result
// so they run on the same worker pool; the dedup filter below keeps first sighting.
var discoveryTargets = new uint[results.Length][];
var swallowedSiblingEntries = new uint[results.Length][];
IndexedParallel.For(results.Length, parallelOptions, i =>
{
if (results[i] is not { } result) return;
discoveryTargets[i] = DiscoverDiscoveryTranslationTargets(result).ToArray();
swallowedSiblingEntries[i] = DiscoverSwallowedSiblingEntries(result).ToArray();
});
var newlyKnownEntries = Enumerable.Range(0, results.Length)
.Where(i => results[i] is not null)
// branches could bypass a mod overlay registered there. The dedup filter keeps first sighting.
var newlyKnownEntries = Enumerable.Range(0, batch.Count)
.Where(i => metrics[i] is not null)
.SelectMany(i => discoveryTargets[i].Concat(swallowedSiblingEntries[i]))
.Where(knownBaseFunctionEntryPoints.Add)
.ToHashSet();
@@ -675,49 +697,39 @@ int RunTranslateRecursive(string[] argsTail)
{
// The boundary set is final for this batch once the filter above has run,
// so the affected re-translations are independent of one another.
var affected = new List<int>();
for (var i = 0; i < results.Length; i++)
{
if (results[i] is not { } result) continue;
var entry = result.EntryPoint;
if (result.Instructions.Any(instruction =>
instruction.Address != entry && newlyKnownEntries.Contains(instruction.Address)))
{
affected.Add(i);
}
}
IndexedParallel.For(affected.Count, parallelOptions, k =>
{
var i = affected[k];
results[i] = Attempt(batch[i], speculative);
if (results[i] is { } result)
{
discoveryTargets[i] = DiscoverDiscoveryTranslationTargets(result).ToArray();
}
});
var sortedNewEntries = newlyKnownEntries.Order().ToArray();
var affected = Enumerable.Range(0, batch.Count)
.Where(i => metrics[i] is not null &&
CoversInteriorInstruction(flowCoverage[i], entryPoints[i], sortedNewEntries))
.ToArray();
IndexedParallel.For(affected.Length, parallelOptions, k => Decode(affected[k], findSwallowedSiblings: false));
}
// The compact GQR projection is a pure walk of the final SSA graph and the
// canonical store serializes its own dictionary, so both move off the serial
// commit. Every address in a batch is distinct and untranslated, so the store
// sees one Put per key regardless of completion order.
var summaries = new GqrFunctionSummary[results.Length];
IndexedParallel.For(results.Length, parallelOptions, i =>
{
if (results[i] is not { } result) return;
summaries[i] = GqrFunctionSummary.Create(result.Ssa.Function);
canonicalIrStore.Put(batch[i].Address, result.Ssa.Function);
});
// Update state sequentially (thread-safe)
for (var i = 0; i < batch.Count; i++)
{
if (results[i] is not { } result) continue;
CommitResult(batch[i], result, discoveryTargets[i], summaries[i]);
if (metrics[i] is not { } functionMetrics) continue;
CommitResult(batch[i], functionMetrics, discoveryTargets[i], summaries[i], functionEnds[i], flowCoverage[i]);
}
}
// True when an instruction of the coalesced [start, end) coverage, other than the
// entry, starts at one of the sorted addresses.
static bool CoversInteriorInstruction(uint[] coverage, uint entry, uint[] sortedAddresses)
{
for (var r = 0; r < coverage.Length; r += 2)
{
var index = Array.BinarySearch(sortedAddresses, coverage[r]);
for (index = index >= 0 ? index : ~index;
index < sortedAddresses.Length && sortedAddresses[index] < coverage[r + 1];
index++)
{
var address = sortedAddresses[index];
if (address != entry && (address & 3u) == 0) return true;
}
}
return false;
}
FunctionDiscoveryResult? Attempt((uint Address, int Depth, string Name) work, bool speculative)
{
if (!speculative)
@@ -773,16 +785,18 @@ int RunTranslateRecursive(string[] argsTail)
void CommitResult(
(uint Address, int Depth, string Name) work,
FunctionDiscoveryResult result,
TranslationMetrics functionMetrics,
IReadOnlyList<uint> discoveryTargets,
GqrFunctionSummary summary)
GqrFunctionSummary summary,
uint functionEnd,
uint[] flowCoverage)
{
// File writing moved to parallel section above
totals.Add(result.Metrics);
totals.Add(functionMetrics);
count++;
translated.Add(work.Address);
translatedFunctionEnds[work.Address] = GetFunctionInstructionEnd(result);
translatedFlowCoverage[work.Address] = CoalesceInstructionRanges(result);
translatedFunctionEnds[work.Address] = functionEnd;
translatedFlowCoverage[work.Address] = flowCoverage;
foreach (var target in discoveryTargets)
{
@@ -821,6 +835,7 @@ int RunTranslateRecursive(string[] argsTail)
// list that existed to paper over the addresses they still missed.
ProcessQueue();
var walkReachable = count;
var walkTranslated = translated.ToHashSet();
ProcessSpeculativeSeeds();
if (functionMap is not null)
{
@@ -864,26 +879,31 @@ int RunTranslateRecursive(string[] argsTail)
if (boundaryRepairAddresses.Length != 0)
{
Console.WriteLine($"[translator] Reanalyzing {boundaryRepairAddresses.Length:N0} function(s) affected by later-discovered boundaries...");
var repaired = new FunctionDiscoveryResult[boundaryRepairAddresses.Length];
var repairedSummaries = new GqrFunctionSummary[boundaryRepairAddresses.Length];
var repairedEnds = new uint[boundaryRepairAddresses.Length];
var repairedCoverage = new uint[boundaryRepairAddresses.Length][];
IndexedParallel.For(boundaryRepairAddresses.Length, parallelOptions, i =>
{
var address = boundaryRepairAddresses[i];
var work = baseWork[address];
repaired[i] = translator.Value.Discover(address, WithProjectFrontEndPolicy(new TranslationOptions(
var repaired = translator.Value.Discover(address, WithProjectFrontEndPolicy(new TranslationOptions(
work.Name,
MaxInstructions: work.MaxInstructions,
MaxBytes: work.MaxBytes,
AllowUnsupportedInstructions: work.AllowUnsupported,
KnownFunctionEntryPoints: knownBaseFunctionEntryPoints),
RequireProject().Translation));
repairedSummaries[i] = GqrFunctionSummary.Create(repaired.Ssa.Function);
canonicalIrStore.Put(address, repaired.Ssa.Function);
repairedEnds[i] = GetFunctionInstructionEnd(repaired);
repairedCoverage[i] = CoalesceInstructionRanges(repaired);
});
for (var i = 0; i < boundaryRepairAddresses.Length; i++)
{
var address = boundaryRepairAddresses[i];
baseSummaries[address] = GqrFunctionSummary.Create(repaired[i].Ssa.Function);
canonicalIrStore.Put(address, repaired[i].Ssa.Function);
translatedFunctionEnds[address] = GetFunctionInstructionEnd(repaired[i]);
translatedFlowCoverage[address] = CoalesceInstructionRanges(repaired[i]);
baseSummaries[address] = repairedSummaries[i];
translatedFunctionEnds[address] = repairedEnds[i];
translatedFlowCoverage[address] = repairedCoverage[i];
}
}
@@ -906,6 +926,12 @@ int RunTranslateRecursive(string[] argsTail)
}
Console.WriteLine("[translator] Computing interprocedural GQR constants and callee write summaries...");
// Map entries the walk never reached have an unconstrained entry GQR state unless some
// translated function calls them directly.
var gqrUnknownEntryRoots = GqrInterproceduralAnalysis.SelectUnknownEntryRoots(
baseSummaries,
[startAddr],
(functionMap?.Addresses ?? Array.Empty<uint>()).Where(address => !walkTranslated.Contains(address)));
var gqrAnalysis = GqrInterproceduralAnalysis.Analyze(
baseSummaries,
gqrUnknownEntryRoots);
@@ -1183,7 +1209,7 @@ int RunTranslateRecursive(string[] argsTail)
// Populate the recursive ABI inference cache once, in stable global order, before
// refinement and parallel emission, so cyclic call graphs don't depend on thread count.
LogPhase("state-free selection + leaf residency");
ReportAbiPrewarm(inferredGuestAbi.Value.Prewarm(emissionAddresses));
ReportAbiPrewarm(inferredGuestAbi.Value.Prewarm(emissionAddresses, maxThreads));
LogPhase("ABI prewarm");
// State-free entries use the transformed leaf's exact live-at-entry
@@ -1404,6 +1430,7 @@ int RunTranslateRecursive(string[] argsTail)
// source visible. `completionPulse` only wakes the writer; it carries
// no state of its own.
var emissionDone = new bool[pendingSources.Length];
var unsupportedByIndex = new int[emissionAddresses.Length];
using var completionPulse = new ManualResetEventSlim(false);
var emissionAborted = false;
Task? pendingBundleWrite = null;
@@ -1459,6 +1486,8 @@ int RunTranslateRecursive(string[] argsTail)
var options = BuildEmissionOptions(address, stateFreeAbiContracts);
if (!canonicalIrStore.TryGet(address, out var canonicalSsa))
throw new InvalidOperationException($"Missing canonical SSA for final lowering at 0x{address:X8}.");
unsupportedByIndex[i] = canonicalSsa.Blocks.Sum(block =>
block.Instructions.Count(static instruction => instruction is IrUndefined));
var emitted = translator.Value.LowerCanonical(address, canonicalSsa, options);
var path = work.Path;
if (outputMetadata is not null)
@@ -1517,16 +1546,7 @@ int RunTranslateRecursive(string[] argsTail)
: Path.GetRelativePath(
Path.GetDirectoryName(Path.GetFullPath(outputMetadataPath!))!,
Path.GetFullPath(productionSourceBundlePath)).Replace('\\', '/');
var metadataUnsupportedInstructionCount = emissionAddresses.Sum(address =>
{
if (!canonicalIrStore.TryGet(address, out var canonicalSsa))
{
throw new InvalidOperationException(
$"Missing canonical SSA while computing translation quality for 0x{address:X8}.");
}
return canonicalSsa.Blocks.Sum(block =>
block.Instructions.Count(static instruction => instruction is IrUndefined));
});
var metadataUnsupportedInstructionCount = unsupportedByIndex.Sum();
completedOutputMetadata = BaseTranslationOutputMetadata.Create(
outputMetadata,
new TranslationQualityMetadata(
@@ -1701,7 +1721,7 @@ int RunTranslateModCore(string[] argsTail, string? outputDirectoryOverride)
// win (this shipped once as a character-select LinkList.h:573 panic).
if (File.Exists(baseTranslationOutputMetadataPath) && File.Exists(codePul))
{
var guardedBaseMetadata = BaseTranslationOutputMetadataFile.Read(baseTranslationOutputMetadataPath);
var guardedBaseMetadata = ReadBaseMetadataCached(baseTranslationOutputMetadataPath);
var codePulSha256 = ChecksumUtilities.Sha256HexOfFile(codePul);
var baseKnowsThisPul = guardedBaseMetadata.ModPatchAwareness is { } awareness &&
awareness.Any(entry => string.Equals(entry.CodePulSha256, codePulSha256, StringComparison.OrdinalIgnoreCase));
@@ -2079,7 +2099,7 @@ void WriteResolvedDispatchProfile(
var metadataPath = baseTranslationOutputMetadataPath;
if (File.Exists(metadataPath))
{
var metadata = BaseTranslationOutputMetadataFile.Read(metadataPath);
var metadata = ReadBaseMetadataCached(metadataPath);
if (!string.IsNullOrWhiteSpace(metadata.SourceBundlePath))
{
var bundlePath = Path.IsPathRooted(metadata.SourceBundlePath)
@@ -2154,7 +2174,7 @@ void WriteResolvedDispatchProfile(
BaseTranslationOutputMetadata? metadata = null;
if (File.Exists(baseTranslationOutputMetadataPath))
{
metadata = BaseTranslationOutputMetadataFile.Read(baseTranslationOutputMetadataPath);
metadata = ReadBaseMetadataCached(baseTranslationOutputMetadataPath);
}
if (metadata is null || metadata.Functions.Any(static function => function.Build is null))
{
@@ -2258,12 +2278,21 @@ void WriteResolvedDispatchProfile(
// functions the runtime must own (hardware, the strap scene). But when the mod's whole point
// is replacing that function, e.g. Retro Rewind's DVDOpen override at 0x8015E2BC, the overlay
// is silently dropped. Name every such address so the tradeoff is a choice, not an accident.
// Wrapper natives still run the overlay, and replacement natives already made that choice.
var nativesKeepingModOverlayDecision = runtimeNativeIndex.Value.Registrations
.Where(static registration => registration.ModOverlay != NativeModOverlay.Unspecified)
.Select(static registration => (registration.Address, registration.Symbol))
.ToHashSet();
var nativeOverriddenModPatches = groups
.Where(group =>
group.Any(entry => entry.Kind == "rr") &&
group.OrderByDescending(entry => entry.Priority)
.ThenByDescending(entry => KindRank(entry.Kind))
.First().Kind == "native")
{
var winner = group.OrderByDescending(entry => entry.Priority)
.ThenByDescending(entry => KindRank(entry.Kind))
.First();
return group.Any(entry => entry.Kind == "rr") &&
winner.Kind == "native" &&
!nativesKeepingModOverlayDecision.Contains((winner.Address, winner.Symbol));
})
.Select(group => new
{
Address = group.Key,
@@ -2326,12 +2355,9 @@ int EmitModCpp(
: new CompositeGuestFunctionAbiProvider(inferredAbi);
var modTranslator = new FunctionTranslator(modImage, abiProvider);
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = maxThreads };
// Mod emission waves: each item is a full translate+emit, and each wave is a
// barrier that also drains newly discovered continuations, so it is
// deliberately smaller than the discovery batch above - a long wave delays
// the discoveries the next wave has to translate.
var waveSize = Math.Max(16, maxThreads * 4);
Console.WriteLine($" translating mod C++ with up to {maxThreads} worker thread(s) in waves of {waveSize}");
// Each wave drains its whole queue: a wave is a barrier that publishes newly discovered
// entries, and draining everything makes the output independent of any wave size.
Console.WriteLine($" translating mod C++ with up to {maxThreads} worker thread(s)");
var cppRoot = Path.Combine(outDir, "cpp");
if (Directory.Exists(cppRoot))
@@ -2410,7 +2436,7 @@ int EmitModCpp(
// Some dynamically discovered module/continuation waves bypass
// ProcessBoundedWaves. Prewarm every parallel emission boundary as a
// final deterministic guard; phase-prewarmed entries are cheap hits.
ReportAbiPrewarm(inferredAbi.Prewarm(wave.Select(static work => work.Address)));
ReportAbiPrewarm(inferredAbi.Prewarm(wave.Select(static work => work.Address), maxThreads));
return ModTranslationWaveRunner.Translate(() => modTranslator, wave, parallelOptions);
}
@@ -2447,7 +2473,7 @@ int EmitModCpp(
// Populate the recursive ABI cache in a stable global order before any worker emits C++,
// so shared closure reuse and cyclic call graphs never depend on thread count, wave size,
// or worker order.
ReportAbiPrewarm(inferredAbi.Prewarm(ordered.Select(static work => work.Address)));
ReportAbiPrewarm(inferredAbi.Prewarm(ordered.Select(static work => work.Address), maxThreads));
// Commit callbacks here only append to the discovered-continuation queue that a later
// phase drains, so nothing in this phase reads back its own commits; wave barriers would
// only cost idle workers, so it runs as one parallel batch and the runner re-sorts by the
@@ -2592,7 +2618,7 @@ int EmitModCpp(
{
while (discoveredContinuationQueue.Count > 0)
{
var wave = QueueBatch.Dequeue(discoveredContinuationQueue, waveSize)
var wave = QueueBatch.Dequeue(discoveredContinuationQueue, discoveredContinuationQueue.Count)
.Select(CreateContinuationWork)
.ToArray();
CommitWave(TranslateWave(wave), (result, work) =>
@@ -2611,10 +2637,10 @@ int EmitModCpp(
var emittedModuleStarts = new HashSet<uint>();
while (moduleQueue.Count > 0)
{
var wave = new List<ModTranslationWork>(waveSize);
var wave = new List<ModTranslationWork>(moduleQueue.Count);
foreach (var start in QueueBatch.Dequeue(
moduleQueue,
waveSize,
moduleQueue.Count,
start => !emittedModuleStarts.Add(start.Address)))
{
var name = ModuleFunctionName(start.Address);
@@ -8,6 +8,7 @@
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<ServerGarbageCollection>true</ServerGarbageCollection>
<ConcurrentGarbageCollection>true</ConcurrentGarbageCollection>
<TieredPGO>false</TieredPGO>
<VersionPrefix>0.5.0</VersionPrefix>
<Company>WiiCompiled</Company>
<Product>Mario Kart Wii Static Translator</Product>
@@ -102,12 +102,7 @@ public static class GuestMemoryRangeLowering
.Any(group => group.Count() >= minimumAccesses))
return function;
var cfg = IrCfg.Build(function);
var dominators = ComputeDominators(cfg);
var reachability = ComputeReachability(cfg);
var cyclicBlocks = cfg.Blocks.Keys.Where(label =>
cfg.Successors(label).Any(successor => reachability[successor].Contains(label)))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var graph = BlockGraph.Build(function);
var plans = new Dictionary<(int Block, int Instruction), Plan>();
var resolves = new Dictionary<int, List<PlacedResolve>>();
@@ -129,24 +124,35 @@ public static class GuestMemoryRangeLowering
if (definitionSites.Count != 1) continue;
definition = definitionSites[0];
var defined = definitionSites[0];
var definitionBlock = function.Blocks[defined.BlockIndex].Label;
if (remaining.Any(candidate =>
!dominators[candidate.AccessBlockLabel(function)].Contains(definitionBlock) ||
!graph.Dominates(defined.BlockIndex, candidate.BlockIndex) ||
candidate.BlockIndex == defined.BlockIndex &&
candidate.Access.Index <= defined.InstructionIndex))
continue;
}
// The pair predicate is invariant across rounds, so memoize it.
var groupIndex = new Dictionary<FunctionAccess, int>(ReferenceEqualityComparer.Instance);
foreach (var access in remaining) groupIndex.Add(access, groupIndex.Count);
var compatible = remaining.Count <= 4096 ? new sbyte[remaining.Count * remaining.Count] : null;
bool Compatible(FunctionAccess anchor, FunctionAccess candidate)
{
if (compatible is null)
return DominatesAccess(graph, anchor, candidate) &&
!HasInvalidatingBoundaryOnPath(graph, invalidators, anchor, candidate);
var slot = groupIndex[anchor] * groupIndex.Count + groupIndex[candidate];
if (compatible[slot] != 0) return compatible[slot] > 0;
var value = DominatesAccess(graph, anchor, candidate) &&
!HasInvalidatingBoundaryOnPath(graph, invalidators, anchor, candidate);
compatible[slot] = value ? (sbyte)1 : (sbyte)-1;
return value;
}
while (remaining.Count >= minimumAccesses)
{
FunctionAccess? bestAnchor = null;
FunctionAccess[] bestMembers = [];
foreach (var anchor in remaining)
{
var members = remaining.Where(candidate =>
DominatesAccess(function, dominators, anchor, candidate) &&
!HasInvalidatingBoundaryOnPath(
function, reachability, cyclicBlocks,
invalidators, anchor, candidate))
var members = remaining.Where(candidate => Compatible(anchor, candidate))
.OrderBy(candidate => candidate.Access.Index)
.ToList();
// Keep a bounded affine window around the real anchor. A
@@ -248,15 +254,13 @@ public static class GuestMemoryRangeLowering
}
private static bool DominatesAccess(
IrFunction function,
IReadOnlyDictionary<string, HashSet<string>> dominators,
BlockGraph graph,
FunctionAccess anchor,
FunctionAccess consumer)
{
if (anchor.BlockIndex == consumer.BlockIndex)
return anchor.Access.Index <= consumer.Access.Index;
return dominators[consumer.AccessBlockLabel(function)].Contains(
anchor.AccessBlockLabel(function));
return graph.Dominates(anchor.BlockIndex, consumer.BlockIndex);
}
private static bool HasOverlappingOppositeAccess(
@@ -273,27 +277,25 @@ public static class GuestMemoryRangeLowering
}
private static bool HasInvalidatingBoundaryOnPath(
IrFunction function,
IReadOnlyDictionary<string, HashSet<string>> reachability,
IReadOnlySet<string> cyclicBlocks,
BlockGraph graph,
IReadOnlyList<InstructionSite> invalidators,
FunctionAccess anchor,
FunctionAccess consumer)
{
var anchorLabel = anchor.AccessBlockLabel(function);
var consumerLabel = consumer.AccessBlockLabel(function);
var anchorBlock = anchor.BlockIndex;
var consumerBlock = consumer.BlockIndex;
var consumerReentered = anchorBlock != consumerBlock && graph.Cyclic[consumerBlock];
foreach (var invalidator in invalidators)
{
var boundaryLabel = function.Blocks[invalidator.BlockIndex].Label;
var afterAnchor = invalidator.BlockIndex == anchor.BlockIndex
var boundaryBlock = invalidator.BlockIndex;
var afterAnchor = boundaryBlock == anchorBlock
? invalidator.InstructionIndex > anchor.Access.Index
: reachability[anchorLabel].Contains(boundaryLabel);
var beforeConsumer = invalidator.BlockIndex == consumer.BlockIndex
? invalidator.InstructionIndex < consumer.Access.Index ||
anchor.BlockIndex != consumer.BlockIndex &&
cyclicBlocks.Contains(consumerLabel)
: reachability[boundaryLabel].Contains(consumerLabel);
if (afterAnchor && beforeConsumer) return true;
: graph.Reaches(anchorBlock, boundaryBlock);
if (!afterAnchor) continue;
var beforeConsumer = boundaryBlock == consumerBlock
? invalidator.InstructionIndex < consumer.Access.Index || consumerReentered
: graph.Reaches(boundaryBlock, consumerBlock);
if (beforeConsumer) return true;
}
return false;
}
@@ -315,54 +317,117 @@ public static class GuestMemoryRangeLowering
return result;
}
private static Dictionary<string, HashSet<string>> ComputeDominators(IrCfg cfg)
/// <summary>Dominator and reachability sets as bitsets over block ordinals.</summary>
private sealed class BlockGraph
{
var comparer = StringComparer.OrdinalIgnoreCase;
var labels = cfg.Blocks.Keys.ToArray();
var all = labels.ToHashSet(comparer);
var result = labels.ToDictionary(label => label,
label => label.Equals(cfg.Entry, StringComparison.OrdinalIgnoreCase)
? new HashSet<string>([label], comparer)
: new HashSet<string>(all, comparer), comparer);
bool changed;
do
{
changed = false;
foreach (var label in labels)
{
if (label.Equals(cfg.Entry, StringComparison.OrdinalIgnoreCase)) continue;
var predecessors = cfg.Predecessors(label);
var next = predecessors.Count == 0
? new HashSet<string>(comparer)
: new HashSet<string>(result[predecessors[0]], comparer);
foreach (var predecessor in predecessors.Skip(1)) next.IntersectWith(result[predecessor]);
next.Add(label);
if (result[label].SetEquals(next)) continue;
result[label] = next;
changed = true;
}
} while (changed);
return result;
}
private readonly ulong[][] _dominators;
private readonly ulong[][] _reach;
private static Dictionary<string, HashSet<string>> ComputeReachability(IrCfg cfg)
{
var comparer = StringComparer.OrdinalIgnoreCase;
var result = new Dictionary<string, HashSet<string>>(comparer);
foreach (var start in cfg.Blocks.Keys)
private BlockGraph(ulong[][] dominators, ulong[][] reach, bool[] cyclic)
{
var reachable = new HashSet<string>(comparer);
var pending = new Stack<string>();
pending.Push(start);
while (pending.Count != 0)
{
var current = pending.Pop();
if (!reachable.Add(current)) continue;
foreach (var successor in cfg.Successors(current)) pending.Push(successor);
}
result[start] = reachable;
_dominators = dominators;
_reach = reach;
Cyclic = cyclic;
}
/// <summary>Blocks that can reach themselves again through a successor.</summary>
public bool[] Cyclic { get; }
public bool Dominates(int dominator, int block) =>
(_dominators[block][dominator >> 6] & (1UL << (dominator & 63))) != 0;
/// <summary>Reflexive: every block reaches itself.</summary>
public bool Reaches(int from, int to) =>
(_reach[from][to >> 6] & (1UL << (to & 63))) != 0;
public static BlockGraph Build(IrFunction function)
{
var cfg = IrCfg.Build(function);
var count = function.Blocks.Count;
var words = (count + 63) >> 6;
var index = new Dictionary<string, int>(count, StringComparer.OrdinalIgnoreCase);
for (var i = 0; i < count; i++) index.Add(function.Blocks[i].Label, i);
var successors = new int[count][];
var predecessors = new int[count][];
for (var i = 0; i < count; i++)
{
var label = function.Blocks[i].Label;
successors[i] = cfg.Successors(label).Select(successor => index[successor]).ToArray();
predecessors[i] = cfg.Predecessors(label).Select(predecessor => index[predecessor]).ToArray();
}
var reach = new ulong[count][];
var pending = new Stack<int>();
for (var start = 0; start < count; start++)
{
var row = new ulong[words];
pending.Push(start);
while (pending.Count != 0)
{
var current = pending.Pop();
ref var word = ref row[current >> 6];
var bit = 1UL << (current & 63);
if ((word & bit) != 0) continue;
word |= bit;
foreach (var successor in successors[current]) pending.Push(successor);
}
reach[start] = row;
}
var cyclic = new bool[count];
for (var i = 0; i < count; i++)
{
foreach (var successor in successors[i])
{
if ((reach[successor][i >> 6] & (1UL << (i & 63))) == 0) continue;
cyclic[i] = true;
break;
}
}
var entry = index[cfg.Entry];
var dominators = new ulong[count][];
for (var i = 0; i < count; i++)
{
dominators[i] = new ulong[words];
if (i == entry)
{
dominators[i][i >> 6] = 1UL << (i & 63);
continue;
}
Array.Fill(dominators[i], ulong.MaxValue);
if ((count & 63) != 0) dominators[i][words - 1] = (1UL << (count & 63)) - 1;
}
var next = new ulong[words];
bool changed;
do
{
changed = false;
for (var i = 0; i < count; i++)
{
if (i == entry) continue;
var preds = predecessors[i];
if (preds.Length == 0)
{
Array.Clear(next);
}
else
{
Array.Copy(dominators[preds[0]], next, words);
for (var p = 1; p < preds.Length; p++)
{
var other = dominators[preds[p]];
for (var w = 0; w < words; w++) next[w] &= other[w];
}
}
next[i >> 6] |= 1UL << (i & 63);
if (next.AsSpan().SequenceEqual(dominators[i])) continue;
next.AsSpan().CopyTo(dominators[i]);
changed = true;
}
} while (changed);
return new BlockGraph(dominators, reach, cyclic);
}
return result;
}
private static IrInstruction ToResolved(IrInstruction instruction, Access access, Plan plan)
@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Numerics;
using System.Threading;
using System.Threading.Tasks;
using Translator.Core.Analysis;
@@ -72,6 +73,9 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
/// <summary>Reusable backing list for <see cref="FrameTaint"/>; phase A is never reentrant.</summary>
public readonly List<bool> FrameTaintBuffer = new();
/// <summary>Address per <see cref="FrameTaint"/> entry, innermost last.</summary>
public readonly List<uint> FrameAddresses = new();
/// <summary>Addresses already known tainted, shared by every phase-A worker of the current Prewarm
/// call and reset each invocation. Without this memoization an unpublishable address gets
/// re-analyzed, subtree and all, by every caller that reaches it, which compounds combinatorially
@@ -95,14 +99,21 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
return false;
}
var state = _threadState.Value!;
var frameTaint = state.FrameTaint;
if (frameTaint is not null && state.FrameAddresses.Count > 0 && state.FrameAddresses[^1] == address)
{
// Direct self-recursion sees what a sequential run would, even once another thread has published it.
abi = GuestFunctionAbi.Empty;
return false;
}
if (_cache.TryGetValue(address, out abi!))
{
return true;
}
var state = _threadState.Value!;
var inProgress = state.InProgress;
var frameTaint = state.FrameTaint;
if (frameTaint is not null && state.TaintedAddresses!.ContainsKey(address))
{
@@ -151,6 +162,7 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
// Prewarm phase A: track taint for this frame and publish only if it stayed clean.
frameTaint.Add(false);
state.FrameAddresses.Add(address);
var frameIndex = frameTaint.Count - 1;
bool succeeded;
bool tainted;
@@ -172,6 +184,7 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
finally
{
inProgress.Remove(address);
state.FrameAddresses.RemoveAt(frameIndex);
tainted = frameTaint[frameIndex];
frameTaint.RemoveAt(frameIndex);
if (tainted && frameIndex > 0)
@@ -274,6 +287,7 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
var state = _threadState.Value!;
var frames = state.FrameTaintBuffer;
frames.Clear();
state.FrameAddresses.Clear();
state.TaintedAddresses = taintedAddresses;
state.FrameTaint = frames;
try
@@ -418,82 +432,102 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
private HashSet<string> InferArgumentRegisters(IrFunction function, IrCfg cfg)
{
var comparer = StringComparer.OrdinalIgnoreCase;
var blockUses = new Dictionary<string, HashSet<string>>(comparer);
var blockDefs = new Dictionary<string, HashSet<string>>(comparer);
foreach (var block in function.Blocks)
// Interned register ids so the liveness fixpoint runs over bitsets.
var ids = new Dictionary<string, int>(comparer);
var names = new List<string>();
int Id(string name)
{
var uses = new HashSet<string>(comparer);
var defs = new HashSet<string>(comparer);
if (!ids.TryGetValue(name, out var id))
{
id = names.Count;
ids.Add(name, id);
names.Add(name);
}
return id;
}
var blockCount = function.Blocks.Count;
var useLists = new List<int>[blockCount];
var defLists = new List<int>[blockCount];
var labelIndex = new Dictionary<string, int>(blockCount, comparer);
for (var b = 0; b < blockCount; b++)
{
var block = function.Blocks[b];
var uses = new List<int>();
var defs = new List<int>();
var defined = new HashSet<int>();
foreach (var instruction in block.Instructions)
{
foreach (var use in InstructionRegisterUsesForArgumentInference(instruction))
{
if (!defs.Contains(use))
{
uses.Add(use);
}
var id = Id(use);
if (!defined.Contains(id))
uses.Add(id);
}
foreach (var def in InstructionRegisterDefsForArgumentInference(instruction))
{
defs.Add(def);
var id = Id(def);
if (defined.Add(id))
defs.Add(id);
}
}
blockUses[block.Label] = uses;
blockDefs[block.Label] = defs;
useLists[b] = uses;
defLists[b] = defs;
}
var liveIn = function.Blocks.ToDictionary(
static block => block.Label,
_ => new HashSet<string>(comparer),
comparer);
var liveOut = function.Blocks.ToDictionary(
static block => block.Label,
_ => new HashSet<string>(comparer),
comparer);
for (var b = 0; b < blockCount; b++)
labelIndex.Add(function.Blocks[b].Label, b);
var words = Math.Max(1, (names.Count + 63) >> 6);
var useBits = new ulong[blockCount * words];
var keepBits = new ulong[blockCount * words];
Array.Fill(keepBits, ulong.MaxValue);
var successors = new int[blockCount][];
for (var b = 0; b < blockCount; b++)
{
foreach (var id in useLists[b]) useBits[b * words + (id >> 6)] |= 1UL << (id & 63);
foreach (var id in defLists[b]) keepBits[b * words + (id >> 6)] &= ~(1UL << (id & 63));
var targets = new List<int>();
foreach (var successor in cfg.Successors(function.Blocks[b].Label))
if (labelIndex.TryGetValue(successor, out var index))
targets.Add(index);
successors[b] = targets.ToArray();
}
var liveIn = new ulong[blockCount * words];
var changed = true;
while (changed)
{
changed = false;
for (var blockIndex = function.Blocks.Count - 1; blockIndex >= 0; blockIndex--)
for (var b = blockCount - 1; b >= 0; b--)
{
var block = function.Blocks[blockIndex];
var newOut = new HashSet<string>(comparer);
foreach (var successor in cfg.Successors(block.Label))
var row = b * words;
for (var w = 0; w < words; w++)
{
if (liveIn.TryGetValue(successor, out var successorLiveIn))
ulong liveOut = 0;
foreach (var successor in successors[b])
liveOut |= liveIn[successor * words + w];
var value = useBits[row + w] | (liveOut & keepBits[row + w]);
if (value != liveIn[row + w])
{
newOut.UnionWith(successorLiveIn);
liveIn[row + w] = value;
changed = true;
}
}
var newIn = new HashSet<string>(blockUses[block.Label], comparer);
var outMinusDefs = new HashSet<string>(newOut, comparer);
outMinusDefs.ExceptWith(blockDefs[block.Label]);
newIn.UnionWith(outMinusDefs);
if (!newOut.SetEquals(liveOut[block.Label]))
{
liveOut[block.Label] = newOut;
changed = true;
}
if (!newIn.SetEquals(liveIn[block.Label]))
{
liveIn[block.Label] = newIn;
changed = true;
}
}
}
return liveIn.TryGetValue(function.EntryLabel, out var entryLive)
? entryLive
.Where(IsGprOrFprRegister)
.ToHashSet(comparer)
: new HashSet<string>(comparer);
var result = new HashSet<string>(comparer);
if (labelIndex.TryGetValue(function.EntryLabel, out var entry))
{
for (var id = 0; id < names.Count; id++)
{
if ((liveIn[entry * words + (id >> 6)] & (1UL << (id & 63))) != 0 && IsGprOrFprRegister(names[id]))
result.Add(names[id]);
}
}
return result;
}
private IEnumerable<string> InstructionRegisterUsesForArgumentInference(IrInstruction instruction)
@@ -755,45 +789,74 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
private HashSet<string> InferScalarFloatArguments(IrFunction function, IrCfg cfg)
{
var scalarArgs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var comparer = StringComparer.OrdinalIgnoreCase;
var inMap = function.Blocks.ToDictionary(
static b => b.Label,
b => b.Label.Equals(function.EntryLabel, StringComparison.OrdinalIgnoreCase)
? CreateInitialArgumentAliasState()
: new Dictionary<string, HashSet<string>>(comparer),
comparer);
var outMap = function.Blocks.ToDictionary(
static b => b.Label,
b => b.Label.Equals(function.EntryLabel, StringComparison.OrdinalIgnoreCase)
? CreateInitialArgumentAliasState()
: new Dictionary<string, HashSet<string>>(comparer),
comparer);
var keys = new AliasKeys(function);
var blockCount = function.Blocks.Count;
var blockIndex = new Dictionary<string, int>(blockCount, StringComparer.OrdinalIgnoreCase);
for (var b = 0; b < blockCount; b++)
blockIndex[function.Blocks[b].Label] = b;
var predecessors = new int[blockCount][];
var isEntry = new bool[blockCount];
for (var b = 0; b < blockCount; b++)
{
var block = function.Blocks[b];
isEntry[b] = block.Label.Equals(function.EntryLabel, StringComparison.OrdinalIgnoreCase);
var list = new List<int>();
foreach (var predecessor in cfg.Predecessors(block.Label))
if (blockIndex.TryGetValue(predecessor, out var index))
list.Add(index);
predecessors[b] = list.ToArray();
}
// Per alias key: -1 when absent, otherwise the mask of ABI float argument sources.
var initial = keys.InitialState();
var inMap = new int[blockCount][];
var outMap = new int[blockCount][];
for (var b = 0; b < blockCount; b++)
{
inMap[b] = isEntry[b] ? (int[])initial.Clone() : keys.EmptyState();
outMap[b] = isEntry[b] ? (int[])initial.Clone() : keys.EmptyState();
}
var changed = true;
while (changed)
{
changed = false;
foreach (var block in function.Blocks)
for (var b = 0; b < blockCount; b++)
{
var newIn = block.Label.Equals(function.EntryLabel, StringComparison.OrdinalIgnoreCase)
? CreateInitialArgumentAliasState()
: UnionPredecessorStates(cfg.Predecessors(block.Label), outMap);
var newOut = CopyAliasState(newIn);
foreach (var ins in block.Instructions)
int[] newIn;
if (isEntry[b])
{
ProcessScalarArgumentInstruction(ins, newOut, scalarArgs);
newIn = (int[])initial.Clone();
}
else
{
newIn = keys.EmptyState();
foreach (var predecessor in predecessors[b])
{
var predState = outMap[predecessor];
for (var key = 0; key < newIn.Length; key++)
{
if (predState[key] < 0) continue;
newIn[key] = newIn[key] < 0 ? predState[key] : newIn[key] | predState[key];
}
}
}
if (!AliasStatesEqual(newIn, inMap[block.Label]))
var newOut = (int[])newIn.Clone();
foreach (var ins in function.Blocks[b].Instructions)
{
inMap[block.Label] = newIn;
ProcessScalarArgumentInstruction(ins, keys, newOut, scalarArgs);
}
if (!newIn.AsSpan().SequenceEqual(inMap[b]))
{
inMap[b] = newIn;
changed = true;
}
if (!AliasStatesEqual(newOut, outMap[block.Label]))
if (!newOut.AsSpan().SequenceEqual(outMap[b]))
{
outMap[block.Label] = newOut;
outMap[b] = newOut;
changed = true;
}
}
@@ -802,63 +865,153 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
return scalarArgs;
}
/// <summary>Interned float register names the scalar-argument alias analysis can track.
/// Ids 0..12 are the ABI float argument registers, so a source set is a 13-bit mask.</summary>
private sealed class AliasKeys
{
private readonly Dictionary<string, int> _ids = new(StringComparer.OrdinalIgnoreCase);
public AliasKeys(IrFunction function)
{
foreach (var register in AbiFloatArgumentRegisters)
_ids.Add(register, _ids.Count);
foreach (var block in function.Blocks)
{
foreach (var instruction in block.Instructions)
{
switch (instruction)
{
case IrAssign assign:
AddName(assign.Destination);
AddValue(assign.Value);
break;
case IrBinary binary:
AddValue(binary.Left);
AddValue(binary.Right);
AddName(binary.Destination);
break;
case IrLoad load:
AddName(load.Destination);
break;
case IrStore store:
AddValue(store.Source);
break;
case IrCall call:
foreach (var argument in call.Arguments) AddValue(argument);
AddName(call.Destination);
break;
case IrIndirectCall indirectCall:
foreach (var argument in indirectCall.Arguments) AddValue(argument);
AddName(indirectCall.Destination);
break;
case IrSetCrField setCr:
AddValue(setCr.Left);
AddValue(setCr.Right);
break;
case IrPhi phi:
AddName(phi.Destination);
break;
}
}
}
}
public int Count => _ids.Count;
public int[] EmptyState()
{
var state = new int[_ids.Count];
Array.Fill(state, -1);
return state;
}
public int[] InitialState()
{
var state = EmptyState();
for (var i = 0; i < AbiFloatArgumentRegisters.Length; i++)
state[i] = 1 << i;
return state;
}
public bool TryGet(string baseName, out int id) => _ids.TryGetValue(baseName, out id);
private void AddValue(IrValue? value)
{
if (value is { Kind: "register", RegisterName: { } name })
AddName(name);
}
private void AddName(string? name)
{
if (string.IsNullOrEmpty(name)) return;
var baseName = Base(name);
if (IsFloatRegister(baseName))
_ids.TryAdd(baseName, _ids.Count);
}
}
private void ProcessScalarArgumentInstruction(
IrInstruction ins,
Dictionary<string, HashSet<string>> aliases,
AliasKeys keys,
int[] aliases,
HashSet<string> scalarArgs)
{
switch (ins)
{
case IrAssign assign:
PropagateFloatAlias(assign.Destination, assign.Value, aliases);
PropagateFloatAlias(assign.Destination, assign.Value, keys, aliases);
break;
case IrBinary bin:
MarkScalarUse(bin.Left, aliases, scalarArgs);
MarkScalarUse(bin.Right, aliases, scalarArgs);
KillFloatAlias(bin.Destination, aliases);
MarkScalarUse(bin.Left, keys, aliases, scalarArgs);
MarkScalarUse(bin.Right, keys, aliases, scalarArgs);
KillFloatAlias(bin.Destination, keys, aliases);
break;
case IrLoad load:
KillFloatAlias(load.Destination, aliases);
KillFloatAlias(load.Destination, keys, aliases);
break;
case IrStore store:
if (store.SizeBytes is 4 or 8)
{
MarkScalarUse(store.Source, aliases, scalarArgs);
MarkScalarUse(store.Source, keys, aliases, scalarArgs);
}
break;
case IrCall call:
ProcessScalarArgumentCall(call.Target, call.Arguments, aliases, scalarArgs);
KillFloatAlias(call.Destination, aliases);
ProcessScalarArgumentCall(call.Target, call.Arguments, keys, aliases, scalarArgs);
KillFloatAlias(call.Destination, keys, aliases);
break;
case IrIndirectCall icall:
foreach (var arg in icall.Arguments)
{
MarkScalarUse(arg, aliases, scalarArgs);
MarkScalarUse(arg, keys, aliases, scalarArgs);
}
aliases.Remove("f1");
KillFloatAlias(icall.Destination, aliases);
aliases[F1AliasKey] = -1;
KillFloatAlias(icall.Destination, keys, aliases);
break;
case IrSetCrField setCr:
MarkScalarUse(setCr.Left, aliases, scalarArgs);
MarkScalarUse(setCr.Right, aliases, scalarArgs);
MarkScalarUse(setCr.Left, keys, aliases, scalarArgs);
MarkScalarUse(setCr.Right, keys, aliases, scalarArgs);
break;
case IrPhi phi:
KillFloatAlias(phi.Destination, aliases);
KillFloatAlias(phi.Destination, keys, aliases);
break;
}
}
/// <summary>Alias key of f1, the first ABI float argument register.</summary>
private const int F1AliasKey = 0;
private void ProcessScalarArgumentCall(
string target,
IReadOnlyList<IrValue> arguments,
Dictionary<string, HashSet<string>> aliases,
AliasKeys keys,
int[] aliases,
HashSet<string> scalarArgs)
{
if (IsGuestCallTarget(target))
@@ -873,14 +1026,14 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
arg.RegisterName != null &&
abi.HasScalarFloatArgument(Base(arg.RegisterName)))
{
MarkScalarUse(arg, aliases, scalarArgs);
MarkScalarUse(arg, keys, aliases, scalarArgs);
}
}
}
if (writesFloatReturnRegister)
{
aliases.Remove("f1");
aliases[F1AliasKey] = -1;
}
return;
}
@@ -892,7 +1045,7 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
foreach (var arg in arguments)
{
MarkScalarUse(arg, aliases, scalarArgs);
MarkScalarUse(arg, keys, aliases, scalarArgs);
}
}
@@ -1083,7 +1236,8 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
private static void MarkScalarUse(
IrValue value,
IReadOnlyDictionary<string, HashSet<string>> aliases,
AliasKeys keys,
int[] aliases,
HashSet<string> scalarArgs)
{
if (value.Kind != "register" || value.RegisterName == null)
@@ -1092,24 +1246,22 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
}
var baseName = Base(value.RegisterName);
if (!IsFloatRegister(baseName) || !aliases.TryGetValue(baseName, out var sources))
if (!IsFloatRegister(baseName) || !keys.TryGet(baseName, out var key) || aliases[key] < 0)
{
return;
}
foreach (var source in sources)
for (var sources = aliases[key]; sources != 0; sources &= sources - 1)
{
if (AbiFloatArgumentSet.Contains(source))
{
scalarArgs.Add(source);
}
scalarArgs.Add(AbiFloatArgumentRegisters[BitOperations.TrailingZeroCount(sources)]);
}
}
private static void PropagateFloatAlias(
string destination,
IrValue value,
Dictionary<string, HashSet<string>> aliases)
AliasKeys keys,
int[] aliases)
{
var destBase = Base(destination);
if (!IsFloatRegister(destBase))
@@ -1117,24 +1269,27 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
return;
}
keys.TryGet(destBase, out var destKey);
if (value.Kind == "register" &&
value.RegisterName != null &&
IsFloatRegister(Base(value.RegisterName)) &&
aliases.TryGetValue(Base(value.RegisterName), out var srcAliases))
keys.TryGet(Base(value.RegisterName), out var sourceKey) &&
aliases[sourceKey] >= 0)
{
aliases[destBase] = new HashSet<string>(srcAliases, StringComparer.OrdinalIgnoreCase);
aliases[destKey] = aliases[sourceKey];
}
else
{
aliases.Remove(destBase);
aliases[destKey] = -1;
}
}
private static void KillFloatAlias(string destination, Dictionary<string, HashSet<string>> aliases)
private static void KillFloatAlias(string destination, AliasKeys keys, int[] aliases)
{
if (!string.IsNullOrWhiteSpace(destination) && IsFloatRegister(Base(destination)))
if (!string.IsNullOrWhiteSpace(destination) && IsFloatRegister(Base(destination)) &&
keys.TryGet(Base(destination), out var key))
{
aliases.Remove(Base(destination));
aliases[key] = -1;
}
}
@@ -1146,70 +1301,6 @@ public sealed class InferredGuestFunctionAbiProvider : IGuestFunctionAbiProvider
}
}
private static Dictionary<string, HashSet<string>> CreateInitialArgumentAliasState()
{
return AbiFloatArgumentRegisters.ToDictionary(
static r => r,
static r => new HashSet<string>(StringComparer.OrdinalIgnoreCase) { r },
StringComparer.OrdinalIgnoreCase);
}
private static Dictionary<string, HashSet<string>> UnionPredecessorStates(
IReadOnlyList<string> predecessors,
IReadOnlyDictionary<string, Dictionary<string, HashSet<string>>> outMap)
{
var result = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
foreach (var pred in predecessors)
{
if (!outMap.TryGetValue(pred, out var predState))
{
continue;
}
foreach (var (register, sources) in predState)
{
if (!result.TryGetValue(register, out var destSources))
{
destSources = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
result[register] = destSources;
}
destSources.UnionWith(sources);
}
}
return result;
}
private static Dictionary<string, HashSet<string>> CopyAliasState(
IReadOnlyDictionary<string, HashSet<string>> state)
{
return state.ToDictionary(
static kv => kv.Key,
static kv => new HashSet<string>(kv.Value, StringComparer.OrdinalIgnoreCase),
StringComparer.OrdinalIgnoreCase);
}
private static bool AliasStatesEqual(
IReadOnlyDictionary<string, HashSet<string>> left,
IReadOnlyDictionary<string, HashSet<string>> right)
{
if (left.Count != right.Count)
{
return false;
}
foreach (var (key, leftSet) in left)
{
if (!right.TryGetValue(key, out var rightSet) || !leftSet.SetEquals(rightSet))
{
return false;
}
}
return true;
}
private static bool IsGuestCallTarget(string target)
{
if (string.IsNullOrWhiteSpace(target))
@@ -83,9 +83,11 @@ public static partial class GeneratedMarkers
/// <summary>
/// <c>REGISTER_NATIVE_FUNCTION(0x…, symbol)</c> and its <c>_AS</c> alias form;
/// the <c>as</c> group tells the two apart.
/// the <c>as</c> group tells the two apart. <c>REGISTER_NATIVE_WRAPPER_AS</c> and
/// <c>REGISTER_NATIVE_REPLACEMENT_AS</c> are alias forms too, named by the
/// <c>modOverlay</c> group.
/// </summary>
[GeneratedRegex(@"REGISTER_NATIVE_FUNCTION(?<as>_AS)?\s*\(\s*0x(?<address>[0-9A-Fa-f]{8})\s*,\s*(?<symbol>[A-Za-z_][A-Za-z0-9_]*)", RegexOptions.CultureInvariant)]
[GeneratedRegex(@"REGISTER_NATIVE_(?:FUNCTION(?<as>_AS)?|(?<modOverlay>WRAPPER|REPLACEMENT)_AS)\s*\(\s*0x(?<address>[0-9A-Fa-f]{8})\s*,\s*(?<symbol>[A-Za-z_][A-Za-z0-9_]*)", RegexOptions.CultureInvariant)]
public static partial Regex NativeFunctionRegistrationPattern();
[GeneratedRegex(@"REGISTER_TRANSLATED_FUNCTION\s*\(\s*0x(?<address>[0-9A-Fa-f]{8})\s*,\s*(?<symbol>[A-Za-z_][A-Za-z0-9_]*)", RegexOptions.CultureInvariant)]
@@ -11,7 +11,19 @@ public sealed record RuntimeNativeRegistration(
string Symbol,
string SourceFile,
bool IsTranslatedOverride,
bool ExcludesBaseTranslation);
bool ExcludesBaseTranslation,
NativeModOverlay ModOverlay = NativeModOverlay.Unspecified);
/// <summary>What happens to a mod overlay at a native's address.</summary>
public enum NativeModOverlay
{
/// <summary>Plain registration: the overlay is lost, and reported as such.</summary>
Unspecified,
/// <summary>The native calls through to the overlay (<c>MKW_GUEST_NEXT</c>).</summary>
Wrapper,
/// <summary>The native deliberately takes over the overlay's address.</summary>
Replacement
}
public sealed record RuntimeNativeAbiEntry(
uint Address,
@@ -119,7 +131,17 @@ public static class RuntimeNativeIndexBuilder
{
var source = sourceFile.Content;
foreach (Match match in GeneratedMarkers.NativeFunctionRegistrationPattern().Matches(source))
Add(match, match.Groups["symbol"].Value, false, !match.Groups["as"].Success);
{
var modOverlay = match.Groups["modOverlay"];
Add(match, match.Groups["symbol"].Value, false,
!match.Groups["as"].Success && !modOverlay.Success,
modOverlay.Value switch
{
"WRAPPER" => NativeModOverlay.Wrapper,
"REPLACEMENT" => NativeModOverlay.Replacement,
_ => NativeModOverlay.Unspecified
});
}
foreach (Match match in GeneratedMarkers.TranslatedFunctionRegistrationPattern().Matches(source))
Add(match, match.Groups["symbol"].Value, true, true);
foreach (Match match in GeneratedMarkers.NativeOverridePattern().Matches(source))
@@ -127,10 +149,11 @@ public static class RuntimeNativeIndexBuilder
foreach (Match match in GeneratedMarkers.FatalStubPattern().Matches(source))
Add(match, $"GX_FATAL_STUB_{match.Groups["address"].Value}", false, true);
void Add(Match match, string symbol, bool translated, bool excludesBase) =>
void Add(Match match, string symbol, bool translated, bool excludesBase,
NativeModOverlay modOverlay = NativeModOverlay.Unspecified) =>
registrations.Add(new RuntimeNativeRegistration(
ParseAddress(match.Groups["address"].Value), symbol, sourceFile.RelativePath,
translated, excludesBase));
translated, excludesBase, modOverlay));
}
return registrations
@@ -139,7 +162,8 @@ public static class RuntimeNativeIndexBuilder
.ThenBy(static registration => registration.Symbol, StringComparer.Ordinal)
.ThenBy(static registration => registration.SourceFile, StringComparer.Ordinal)
.ThenBy(static registration => registration.IsTranslatedOverride)
.ThenBy(static registration => registration.ExcludesBaseTranslation);
.ThenBy(static registration => registration.ExcludesBaseTranslation)
.ThenBy(static registration => registration.ModOverlay);
}
private static uint ParseAddress(string value) => GuestTargetParser.ParseHexAddress(value);
@@ -14,9 +14,9 @@ namespace Translator.Core;
public static class RuntimeNativeIndexFile
{
public const string Format = "wiicompiled-runtime-native-index";
public const int FormatVersion = 1;
public const int FormatVersion = 2;
// Bump whenever the scanner or the ABI/effect analyzers can produce different output for the same sources.
public const int ExtractorVersion = 1;
public const int ExtractorVersion = 2;
public const string DefaultFileName = "native-index.json";
private static readonly JsonSerializerOptions Options = new()
@@ -59,13 +59,15 @@ public static class RuntimeNativeIndexFile
.ThenBy(static registration => registration.SourceFile, StringComparer.Ordinal)
.ThenBy(static registration => registration.IsTranslatedOverride)
.ThenBy(static registration => registration.ExcludesBaseTranslation)
.ThenBy(static registration => registration.ModOverlay)
.Select(static registration => new RegistrationDto
{
Address = FormatAddress(registration.Address),
Symbol = registration.Symbol,
SourceFile = registration.SourceFile,
IsTranslatedOverride = registration.IsTranslatedOverride,
ExcludesBaseTranslation = registration.ExcludesBaseTranslation
ExcludesBaseTranslation = registration.ExcludesBaseTranslation,
ModOverlay = registration.ModOverlay.ToString()
})
.ToList(),
VoidStubAbis = index.VoidStubAbis
@@ -160,7 +162,8 @@ public static class RuntimeNativeIndexFile
Present(registration.Symbol, "symbol"),
Present(registration.SourceFile, "sourceFile"),
registration.IsTranslatedOverride,
registration.ExcludesBaseTranslation))
registration.ExcludesBaseTranslation,
ParseModOverlay(Present(registration.ModOverlay, "modOverlay"))))
.ToArray(),
Present(document.VoidStubAbis, "voidStubAbis")
.Select(static entry => new RuntimeNativeAbiEntry(
@@ -185,6 +188,11 @@ public static class RuntimeNativeIndexFile
private static string FormatAddress(uint address) => $"0x{address:X8}";
private static NativeModOverlay ParseModOverlay(string text) =>
Enum.TryParse<NativeModOverlay>(text, ignoreCase: false, out var value) && Enum.IsDefined(value)
? value
: throw new FormatException($"Unknown modOverlay '{text}'.");
private static uint ParseAddress(string text) => GuestTargetParser.ParseHexAddress(text);
private static T Present<T>(T? value, string name) where T : class =>
@@ -244,6 +252,7 @@ public static class RuntimeNativeIndexFile
public required string SourceFile { get; init; }
public required bool IsTranslatedOverride { get; init; }
public required bool ExcludesBaseTranslation { get; init; }
public required string ModOverlay { get; init; }
}
private sealed class AbiDto
@@ -30,6 +30,14 @@ public sealed class CanonicalIrStore : ICanonicalIrProvider
}
}
public void Remove(uint entryPoint)
{
lock (_gate)
{
_entries.Remove(entryPoint);
}
}
public bool TryGet(uint entryPoint, out IrFunction function)
{
byte[]? encoded;
@@ -55,12 +63,16 @@ internal static class CanonicalIrBinaryCodec
IndirectJump, JumpTable, Return, Comment, TracePpc, Undefined
}
// Reused per thread so encoding doesn't regrow (and re-fault) a fresh buffer per function.
[ThreadStatic] private static MemoryStream? _encodeStream;
public static byte[] Encode(IrFunction function)
{
var strings = BuildStringTable(function);
var ids = strings.Select((value, index) => (value, index))
.ToDictionary(static pair => pair.value, static pair => pair.index, StringComparer.Ordinal);
using var stream = new MemoryStream();
var stream = _encodeStream ??= new MemoryStream();
stream.SetLength(0);
using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true);
writer.Write(Magic);
WriteVarUInt(writer, (uint)CanonicalIrStore.FormatVersion);
@@ -85,9 +97,11 @@ internal static class CanonicalIrBinaryCodec
return stream.ToArray();
}
public static IrFunction Decode(ReadOnlySpan<byte> encoded)
public static IrFunction Decode(ReadOnlySpan<byte> encoded) => Decode(encoded.ToArray());
public static IrFunction Decode(byte[] encoded)
{
using var stream = new MemoryStream(encoded.ToArray(), writable: false);
using var stream = new MemoryStream(encoded, writable: false);
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true);
if (!reader.ReadBytes(Magic.Length).SequenceEqual(Magic) ||
ReadVarUInt(reader) != CanonicalIrStore.FormatVersion)
@@ -497,12 +497,52 @@ public static class GqrInterproceduralAnalysis
return masks;
}
/// <summary>
/// Unknown-entry roots that depend only on the final call graph, never on discovery order:
/// <paramref name="fixedRoots"/>, every candidate without a direct caller, then (ascending) any
/// candidate the roots still don't reach, so no candidate is left without an entry fact source.
/// </summary>
public static HashSet<uint> SelectUnknownEntryRoots(
IReadOnlyDictionary<uint, GqrFunctionSummary> functions,
IEnumerable<uint> fixedRoots,
IEnumerable<uint> candidates)
{
var called = new HashSet<uint>();
foreach (var summary in functions.Values)
foreach (var target in summary.DirectGuestTargets)
called.Add(target);
var ordered = candidates.Where(functions.ContainsKey).Distinct().OrderBy(static a => a).ToArray();
var roots = fixedRoots.Where(functions.ContainsKey).ToHashSet();
foreach (var address in ordered)
if (!called.Contains(address))
roots.Add(address);
var reachable = ComputeCompactReachable(functions, roots);
foreach (var address in ordered)
{
if (reachable.Contains(address)) continue;
roots.Add(address);
ExtendCompactReachable(functions, reachable, address);
}
return roots;
}
private static HashSet<uint> ComputeCompactReachable(
IReadOnlyDictionary<uint, GqrFunctionSummary> functions,
IReadOnlySet<uint> roots)
{
var reachable = new HashSet<uint>();
var queue = new Queue<uint>(roots);
foreach (var root in roots)
ExtendCompactReachable(functions, reachable, root);
return reachable;
}
private static void ExtendCompactReachable(
IReadOnlyDictionary<uint, GqrFunctionSummary> functions,
HashSet<uint> reachable,
uint root)
{
var queue = new Queue<uint>();
queue.Enqueue(root);
while (queue.TryDequeue(out var address))
{
if (!reachable.Add(address)) continue;
@@ -510,7 +550,6 @@ public static class GqrInterproceduralAnalysis
if (functions.ContainsKey(target) && !reachable.Contains(target))
queue.Enqueue(target);
}
return reachable;
}
private static void TransferCompact(
@@ -167,6 +167,31 @@ public sealed class GqrConstantPropagationTests
instruction => instruction is IrCall { Target: "PPC_PsqLKnown_00070007" });
}
[Fact]
public void UnknownEntryRootsDependOnlyOnTheCallGraph()
{
IrFunction Calls(params uint[] targets) => Function(targets
.Select(target => (IrInstruction)new IrCall(string.Empty, $"0x{target:X8}", Array.Empty<IrValue>()))
.Append(new IrReturn(null))
.ToArray());
var functions = new Dictionary<uint, IrFunction>
{
[0x80001000] = Calls(),
[0x80002000] = Calls(0x80003000),
[0x80003000] = Calls(),
[0x80004000] = Calls(0x80005000),
[0x80005000] = Calls(0x80004000)
};
var summaries = functions.ToDictionary(pair => pair.Key, pair => GqrFunctionSummary.Create(pair.Value));
uint[] candidates = [0x80002000, 0x80003000, 0x80004000, 0x80005000];
var roots = GqrInterproceduralAnalysis.SelectUnknownEntryRoots(summaries, [0x80001000u], candidates);
var reversed = GqrInterproceduralAnalysis.SelectUnknownEntryRoots(summaries, [0x80001000u], candidates.Reverse());
Assert.Equal(new uint[] { 0x80001000, 0x80002000, 0x80004000 }, roots.Order());
Assert.Equal(roots.Order(), reversed.Order());
}
[Fact]
public void InterproceduralAnalysisIntersectsConflictingCallers()
{
@@ -129,7 +129,7 @@ public sealed class GuestAddressTableTests
var calls = Directory.EnumerateFiles(Path.Combine(root, "runtime"), "*.*", SearchOption.AllDirectories)
.Where(path => (path.EndsWith(".cpp", StringComparison.Ordinal) || path.EndsWith(".h", StringComparison.Ordinal)) &&
!path.Contains(thirdParty, StringComparison.Ordinal))
.SelectMany(static path => Regex.Matches(File.ReadAllText(path), @"MKW_GUEST_FUNC\(\s*([0-9A-Fa-f]{8})\s*\)")
.SelectMany(static path => Regex.Matches(File.ReadAllText(path), @"MKW_GUEST_(?:FUNC|NEXT)\(\s*([0-9A-Fa-f]{8})\s*\)")
.Select(static match => Convert.ToUInt32(match.Groups[1].Value, 16)))
.Distinct()
.ToArray();
@@ -145,7 +145,7 @@ public sealed class GuestAddressTableTests
foreach (var (region, starts) in entries)
{
var address = table.Map(pal, region);
Assert.True(starts.Contains(address), $"MKW_GUEST_FUNC({pal:X8}) is not a {region} function entry (0x{address:X8})");
Assert.True(starts.Contains(address), $"MKW_GUEST_FUNC/NEXT({pal:X8}) is not a {region} function entry (0x{address:X8})");
}
}
}
@@ -1,5 +1,6 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using Translator.Core;
using Translator.Core.CodeGen;
using Xunit;
@@ -40,14 +41,24 @@ public sealed class RuntimeNativeIndexTests
PPC_NATIVE_OVERRIDE_VOID(80000040, Stub, (void), ());
GX_FATAL_STUB(80000050, "Fatal")
// REGISTER_NATIVE_FUNCTION(0x80000060, CommentedOut);
REGISTER_NATIVE_WRAPPER_AS(0x80000070, Wrapper, "wrapper");
REGISTER_NATIVE_REPLACEMENT_AS(0x80000080, Replacement, "replacement");
""");
var registrations = RuntimeNativeIndexBuilder.Build(directory).Registrations;
Assert.Equal(5, registrations.Length);
Assert.Equal(7, registrations.Length);
Assert.Contains(registrations, static entry =>
entry.Address == 0x80000010u && entry.ExcludesBaseTranslation);
entry.Address == 0x80000010u && entry.ExcludesBaseTranslation &&
entry.ModOverlay == NativeModOverlay.Unspecified);
Assert.Contains(registrations, static entry =>
entry.Address == 0x80000020u && !entry.ExcludesBaseTranslation);
entry.Address == 0x80000020u && !entry.ExcludesBaseTranslation &&
entry.ModOverlay == NativeModOverlay.Unspecified);
Assert.Contains(registrations, static entry =>
entry.Address == 0x80000070u && entry.Symbol == "Wrapper" && !entry.ExcludesBaseTranslation &&
entry.ModOverlay == NativeModOverlay.Wrapper);
Assert.Contains(registrations, static entry =>
entry.Address == 0x80000080u && entry.Symbol == "Replacement" && !entry.ExcludesBaseTranslation &&
entry.ModOverlay == NativeModOverlay.Replacement);
Assert.Contains(registrations, static entry =>
entry.Address == 0x80000030u && entry.IsTranslatedOverride);
Assert.Contains(registrations, static entry =>
@@ -108,6 +119,8 @@ public sealed class RuntimeNativeIndexTests
var sources = Path.Combine(ProjectPathsForTests.FindRepositoryRoot(), "runtime", "src");
var built = RuntimeNativeIndexBuilder.Build(sources);
Assert.NotEmpty(built.Registrations);
Assert.Contains(built.Registrations, static entry => entry.ModOverlay == NativeModOverlay.Wrapper);
Assert.Contains(built.Registrations, static entry => entry.ModOverlay == NativeModOverlay.Replacement);
Assert.NotEmpty(built.VoidStubAbis);
Assert.NotEmpty(built.Effects);
@@ -126,6 +139,26 @@ public sealed class RuntimeNativeIndexTests
}
}
// A wrapper that calls through with MKW_GUEST_FUNC would silently drop the mod overlay it promises to run.
[Fact]
public void RepositoryWrapperNativesCallThroughToTheNextTranslation()
{
var sources = Path.Combine(ProjectPathsForTests.FindRepositoryRoot(), "runtime", "src");
var wrappers = RuntimeNativeIndexBuilder.Build(sources).Registrations
.Where(static entry => entry.ModOverlay == NativeModOverlay.Wrapper)
.ToArray();
Assert.NotEmpty(wrappers);
foreach (var wrapper in wrappers)
{
var source = File.ReadAllText(Path.Combine(sources, wrapper.SourceFile));
var pal = $"{wrapper.Address:X8}";
Assert.True(Regex.IsMatch(source, $@"MKW_GUEST_NEXT\(\s*{pal}\s*\)", RegexOptions.IgnoreCase),
$"{wrapper.Symbol} is a wrapper but never calls MKW_GUEST_NEXT({pal})");
Assert.False(Regex.IsMatch(source, $@"MKW_GUEST_FUNC\(\s*{pal}\s*\)", RegexOptions.IgnoreCase),
$"{wrapper.Symbol} is a wrapper but calls MKW_GUEST_FUNC({pal})");
}
}
[Fact]
public void ResolveUsesAMatchingIndexAndKeepsWorkingWithoutSources()
{