diff --git a/.gitignore b/.gitignore index 1c69c0a..855cfed 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ Code.pul # Build output /build/ /build-*/ +/native-build/ /dist/ /out/ [Bb]in/ diff --git a/Launcher/WiiCompiled.Setup.Linux/BuildRunner.cs b/Launcher/WiiCompiled.Setup.Linux/BuildRunner.cs index 677b757..81007b0 100644 --- a/Launcher/WiiCompiled.Setup.Linux/BuildRunner.cs +++ b/Launcher/WiiCompiled.Setup.Linux/BuildRunner.cs @@ -12,7 +12,8 @@ internal static class BuildRunner public static async Task RunAsync( string workspace, string profile, string outputDir, string? baseOutputDir, string? retroDir, string? retroWfcOfflineDir, bool skipRetroWfcPayload, - bool forceCleanBuild, string? translatorBin, IInstallReporter reporter, CancellationToken cancellationToken) + bool forceCleanBuild, string? translatorBin, string? ccBin, string? cxxBin, string? fuseLd, + string? cmakeBin, string? ninjaBin, IInstallReporter reporter, CancellationToken cancellationToken) { var script = Path.Combine(workspace, "Launcher", "local-build.sh"); if (!File.Exists(script)) throw new FileNotFoundException("local-build.sh is missing", script); @@ -47,6 +48,28 @@ internal static class BuildRunner { startInfo.ArgumentList.Add("--translator-bin"); startInfo.ArgumentList.Add(translatorBin); } + // Forwarded by AppRun so the AppImage's bundled clang/lld (see prepare-portable-clang.sh) + // is used instead of local-build.sh's own default of whatever clang is on $PATH. + if (!string.IsNullOrEmpty(ccBin)) + { + startInfo.ArgumentList.Add("--cc"); startInfo.ArgumentList.Add(ccBin); + } + if (!string.IsNullOrEmpty(cxxBin)) + { + startInfo.ArgumentList.Add("--cxx"); startInfo.ArgumentList.Add(cxxBin); + } + if (!string.IsNullOrEmpty(fuseLd)) + { + startInfo.ArgumentList.Add("--fuse-ld"); startInfo.ArgumentList.Add(fuseLd); + } + if (!string.IsNullOrEmpty(cmakeBin)) + { + startInfo.ArgumentList.Add("--cmake"); startInfo.ArgumentList.Add(cmakeBin); + } + if (!string.IsNullOrEmpty(ninjaBin)) + { + startInfo.ArgumentList.Add("--ninja"); startInfo.ArgumentList.Add(ninjaBin); + } using var process = new Process { StartInfo = startInfo }; var window = new BuildProgressWindow(reporter, InstallStages.Build, start: 6, end: 96); diff --git a/Launcher/WiiCompiled.Setup.Linux/Program.cs b/Launcher/WiiCompiled.Setup.Linux/Program.cs index dd68523..b71bc27 100644 --- a/Launcher/WiiCompiled.Setup.Linux/Program.cs +++ b/Launcher/WiiCompiled.Setup.Linux/Program.cs @@ -150,6 +150,11 @@ internal static class Program skipPayload, flags.ContainsKey("force-clean-build"), flags.GetValueOrDefault("translator-bin"), + flags.GetValueOrDefault("cc"), + flags.GetValueOrDefault("cxx"), + flags.GetValueOrDefault("fuse-ld"), + flags.GetValueOrDefault("cmake"), + flags.GetValueOrDefault("ninja"), reporter, token); reporter.Progress(InstallStages.Shortcuts, "Creating shortcuts", 98); @@ -317,6 +322,7 @@ internal static class Program install [--game ISO_PATH] [--install-dir DIR] [--retro-dir DIR {--download-retro-wfc-payload | --skip-retro-wfc-payload}] [--force-clean-build] [--translator-bin PATH] [--disc-tool-bin PATH] + [--cc PATH] [--cxx PATH] [--fuse-ld NAME_OR_PATH] [--cmake PATH] [--ninja PATH] [--progress-json] [--workspace DIR] uninstall launch-base diff --git a/Launcher/build-appimage.sh b/Launcher/build-appimage.sh index 0fb3311..29999d5 100755 --- a/Launcher/build-appimage.sh +++ b/Launcher/build-appimage.sh @@ -5,8 +5,10 @@ # self-contained binaries, and `nodtool` (a prebuilt MIT/Apache-2.0 CLI from encounter/nod, see # NodToolProvider.cs) is downloaded and bundled too - AppRun passes --translator-bin and # --disc-tool-bin so local-build.sh/DiscTool.cs skip their from-source/download fallbacks entirely. -# It still shells out to system clang/cmake/ninja - no C/C++ toolchain is bundled, matching -# Launcher/local-build.sh's own remaining prerequisites. +# A pruned native clang/lld/cmake/ninja toolchain (see prepare-portable-tools.sh) is bundled the +# same way - AppRun passes --cc/--cxx/--fuse-ld/--cmake/--ninja so local-build.sh never has to find +# a system compiler, CMake, or Ninja. It still shells out to system pkg-config and Vulkan headers, +# matching Launcher/local-build.sh's own remaining prerequisites. # # An AppImage mounts read-only, but local-build.sh writes generated/, native-build/, Assets/, etc. # into the workspace it's given. So AppRun (written below) copies the bundled workspace snapshot @@ -107,6 +109,11 @@ nodtool_path=$(dotnet run --project "$workspace/Launcher/WiiCompiled.Setup.Commo cp "$nodtool_path" "$appdir/usr/bin/nodtool" chmod +x "$appdir/usr/bin/nodtool" +echo "Preparing the portable clang/lld/cmake/ninja toolchain ($appimagetool_arch)..." +bash "$script_dir/prepare-portable-tools.sh" --arch "$appimagetool_arch" +mkdir -p "$appdir/usr/toolchain" +cp -a "$workspace/Launcher/artifacts/portable-tools/toolchain-$appimagetool_arch"/. "$appdir/usr/toolchain/" + echo "Staging the bundled workspace snapshot..." for dir in runtime aurora-main projects; do cp -r "$workspace/$dir" "$appdir/workspace/$dir" @@ -144,7 +151,12 @@ if [ ! -f "$CACHE/.bundle-version" ] || \ fi exec "$HERE/usr/bin/wiicompiled-setup" --workspace "$CACHE" \ --translator-bin "$HERE/usr/bin/translator-cli" \ - --disc-tool-bin "$HERE/usr/bin/nodtool" "$@" + --disc-tool-bin "$HERE/usr/bin/nodtool" \ + --cc "$HERE/usr/toolchain/bin/clang" \ + --cxx "$HERE/usr/toolchain/bin/clang++" \ + --fuse-ld lld \ + --cmake "$HERE/usr/toolchain/bin/cmake" \ + --ninja "$HERE/usr/toolchain/bin/ninja" "$@" APPRUN chmod +x "$appdir/AppRun" diff --git a/Launcher/local-build.sh b/Launcher/local-build.sh index ac869c9..86bcffe 100755 --- a/Launcher/local-build.sh +++ b/Launcher/local-build.sh @@ -62,6 +62,7 @@ ninja_override="" dotnet_override="" translator_dll_override="" translator_bin_override="" +fuse_ld_override="" usage() { cat <<'EOF' @@ -77,6 +78,7 @@ Usage: local-build.sh --output-dir DIR [options] --force-clean-build Discard every translation/build cache first --parallel N Pin translator threads, translated-shard job pool, and Ninja parallelism to N --cc PATH / --cxx PATH C/C++ compiler (default: cc/c++ on PATH) + --fuse-ld NAME_OR_PATH Linker passed to clang as -fuse-ld=NAME_OR_PATH (default: clang's own default linker) --cmake PATH / --ninja PATH Build tools (default: on PATH) --dotnet PATH dotnet executable (default: on PATH) --translator-dll PATH Pre-built Translator.Cli.dll (skips building the translator; still needs --dotnet to run it) @@ -97,6 +99,7 @@ while [[ $# -gt 0 ]]; do --parallel) parallel_override=$2; shift 2 ;; --cc) cc_override=$2; shift 2 ;; --cxx) cxx_override=$2; shift 2 ;; + --fuse-ld) fuse_ld_override=$2; shift 2 ;; --cmake) cmake_override=$2; shift 2 ;; --ninja) ninja_override=$2; shift 2 ;; --dotnet) dotnet_override=$2; shift 2 ;; @@ -350,12 +353,17 @@ elif [[ "$keep_native_build" -eq 1 ]]; then echo "MKWCBUILD: Reusing the incremental native build directory" fi +configure_args=(-S "$workspace/runtime" -B "$build" -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER="$cc_bin" -DCMAKE_CXX_COMPILER="$cxx_bin" + -DCMAKE_MAKE_PROGRAM="$ninja_bin" + -DMKW_TRANSLATED_COMPILE_JOBS="$translated_jobs") +if [[ -n "$fuse_ld_override" ]]; then + configure_args+=(-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=$fuse_ld_override") +fi + log_step configure-native "Configuring the native toolchain" -"$cmake_bin" -S "$workspace/runtime" -B "$build" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER="$cc_bin" -DCMAKE_CXX_COMPILER="$cxx_bin" \ - -DCMAKE_MAKE_PROGRAM="$ninja_bin" \ - -DMKW_TRANSLATED_COMPILE_JOBS="$translated_jobs" +"$cmake_bin" "${configure_args[@]}" case "$profile" in base) targets=(WiiCompiled) ;; diff --git a/Launcher/prepare-portable-tools.sh b/Launcher/prepare-portable-tools.sh new file mode 100755 index 0000000..8e812ff --- /dev/null +++ b/Launcher/prepare-portable-tools.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# Prepares a self-contained native-Linux build toolchain bundled into the AppImage by +# build-appimage.sh, so a user needs no `clang`/`cmake`/`ninja` of their own to build the +# translated game (mirrors why Windows bundles llvm-mingw + CMake + Ninja via +# Prepare-PortableTools.ps1 - this is that script's Linux counterpart, for the same reason). +# +# clang/lld/llvm-ar: pruned from the official llvm.org GitHub release tarball (NOT llvm-mingw - +# that project targets Windows/mingw, never native Linux) down to just what's needed to compile +# and link: clang, lld, llvm-ar, the clang resource dir (builtin headers + compiler-rt), and +# libc++/libc++abi/libunwind (so the toolchain never has to fall back to the host's system +# libstdc++ headers). The raw release is ~1.9 GiB per arch (every LLVM backend, mlir, flang, lldb, +# docs, tests); pruned it is ~500 MiB uncompressed / ~100 MiB compressed, verified against a real +# build of this project. +# +# cmake: pruned from the official Kitware GitHub release tarball down to bin/cmake (not +# ccmake/cmake-gui/cpack/ctest, which local-build.sh never invokes) plus the Modules/Templates +# CMake needs at runtime (found relative to bin/cmake via CMAKE_ROOT auto-detection - Help/doc/man/ +# the desktop-integration files under share/ are documentation/GUI-only and dropped). Verified with +# a real configure+build using the pruned cmake+ninja+clang together. +# +# ninja: the official ninja-build GitHub release zip, used as-is - it is already a single small +# (~130 KiB compressed) static-ish binary with nothing to prune. +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +workspace=$(cd "$script_dir/.." && pwd) + +llvm_version=22.1.8 +cmake_version=4.3.3 +ninja_version=1.13.2 +destination="$script_dir/artifacts/portable-tools" +arch="" + +usage() { + cat <<'EOF' +Usage: prepare-portable-tools.sh --arch {x86_64|aarch64} [--destination DIR] + + --arch ARCH Target architecture (required) + --destination DIR Where the toolchain is written, as DIR/toolchain-ARCH + (default: Launcher/artifacts/portable-tools) +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) arch=$2; shift 2 ;; + --destination) destination=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "prepare-portable-tools.sh: unknown argument: $1" >&2; exit 1 ;; + esac +done + +case "$arch" in + x86_64) llvm_release_arch=X64; target_triple=x86_64-unknown-linux-gnu + llvm_release_sha256=df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384 + cmake_release_arch=x86_64 + cmake_sha256=927b2368a946c37269c3a66225ab00544e756459cdd0b5d0da438694fb9ff802 + ninja_asset=ninja-linux.zip + ninja_sha256=5749cbc4e668273514150a80e387a957f933c6ed3f5f11e03fb30955e2bbead6 ;; + aarch64) llvm_release_arch=ARM64; target_triple=aarch64-unknown-linux-gnu + llvm_release_sha256=805efad2bb91cb4967fa569e0881d10c0f69c04461cf671cccbae19f547acc34 + cmake_release_arch=aarch64 + cmake_sha256=9ea38356dbd3e32e51029a3e09a0f2f8e117ef4fbcaad7a21ffb36409bbd5cb4 + ninja_asset=ninja-linux-aarch64.zip + ninja_sha256=fd2cacc8050a7f12a16a2e48f9e06fca5c14fc4c2bee2babb67b58be17a607fc ;; + *) echo "prepare-portable-tools.sh: --arch must be x86_64 or aarch64" >&2; usage; exit 1 ;; +esac + +destination=$(mkdir -p "$destination" && cd "$destination" && pwd) +toolchain_dir="$destination/toolchain-$arch" +downloads="$script_dir/artifacts/downloads" +mkdir -p "$downloads" + +sha256_of() { sha256sum "$1" | awk '{print $1}'; } + +download_verified() { + # $1 = destination path, $2 = URL, $3 = expected sha256 + local dest=$1 url=$2 expected=$3 + if [[ -f "$dest" ]] && [[ "$(sha256_of "$dest")" == "$expected" ]]; then return; fi + echo "prepare-portable-tools.sh: downloading $(basename "$dest")..." + local tmp="$dest.partial" + rm -f "$tmp" + curl -fL --progress-bar -o "$tmp" "$url" + local actual + actual=$(sha256_of "$tmp") + if [[ "$actual" != "$expected" ]]; then + echo "prepare-portable-tools.sh: $(basename "$dest") hash mismatch: expected $expected, got $actual" >&2 + rm -f "$tmp" + exit 1 + fi + mv "$tmp" "$dest" +} + +if [[ -x "$toolchain_dir/bin/clang" && -x "$toolchain_dir/bin/ninja" && -x "$toolchain_dir/bin/cmake" ]]; then + echo "prepare-portable-tools.sh: reusing existing toolchain at $toolchain_dir" + exit 0 +fi + +work="$destination/.building-toolchain-$arch" +rm -rf "$work" +mkdir -p "$work/bin" "$work/lib/$target_triple" "$work/include/$target_triple/c++/v1" + +# --- clang/lld/llvm-ar, pruned from the official LLVM release --- + +llvm_archive_name="LLVM-$llvm_version-Linux-$llvm_release_arch.tar.xz" +llvm_archive="$downloads/$llvm_archive_name" +download_verified "$llvm_archive" \ + "https://github.com/llvm/llvm-project/releases/download/llvmorg-$llvm_version/$llvm_archive_name" \ + "$llvm_release_sha256" + +extract_root="$script_dir/artifacts/.extract-clang-$arch" +rm -rf "$extract_root" +mkdir -p "$extract_root" +echo "prepare-portable-tools.sh: extracting $llvm_archive_name (this is the full ~1.9 GiB release; only a fraction is kept)..." +tar -xf "$llvm_archive" -C "$extract_root" +src="$extract_root/LLVM-$llvm_version-Linux-$llvm_release_arch" +[[ -d "$src" ]] || { echo "prepare-portable-tools.sh: unexpected archive layout, expected $src" >&2; exit 1; } + +echo "prepare-portable-tools.sh: pruning to the minimal compile+link toolchain..." + +# clang: the real driver executable plus the clang/clang++ symlinks CMake/local-build.sh invoke. +# Stripped: debug symbols are dead weight for a bundled compiler nobody will debug. +cp -a "$src/bin/clang-22" "$work/bin/" +strip "$work/bin/clang-22" +ln -s clang-22 "$work/bin/clang" +ln -s clang "$work/bin/clang++" + +# lld: linked via -fuse-ld=lld, which clang resolves by looking for ld.lld next to itself first - +# see local-build.sh's --fuse-ld option. +cp -a "$src/bin/lld" "$work/bin/" +strip "$work/bin/lld" +ln -s lld "$work/bin/ld.lld" + +# llvm-ar/llvm-ranlib: CMake's archiver for the many static libraries this project builds +# (aurora, Crypto++, SDL3, Dawn's dependency closure, the translated game shards). +cp -a "$src/bin/llvm-ar" "$work/bin/" +strip "$work/bin/llvm-ar" +ln -s llvm-ar "$work/bin/llvm-ranlib" + +# Clang's resource directory: builtin headers (stddef.h, immintrin.h, ...) and compiler-rt +# (builtins, sanitizer runtimes). `clang -print-resource-dir` must find this at lib/clang//. +cp -a "$src/lib/clang" "$work/lib/" + +# libc++/libc++abi/libunwind: so this toolchain never has to fall back to whatever libstdc++ the +# host distro happens to have installed. Not the default yet (local-build.sh still resolves the +# system libstdc++ unless -stdlib=libc++ is passed), but bundled so that option exists. +cp -a "$src/include/c++" "$work/include/" +cp -a "$src/include/$target_triple/c++/v1/__config_site" "$work/include/$target_triple/c++/v1/" +cp -a "$src/lib/$target_triple"/libc++.a "$src/lib/$target_triple"/libc++abi.a "$src/lib/$target_triple"/libunwind.a "$work/lib/$target_triple/" +cp -a "$src/lib/$target_triple"/libc++.so* "$src/lib/$target_triple"/libc++abi.so* "$src/lib/$target_triple"/libunwind.so* "$work/lib/$target_triple/" + +rm -rf "$extract_root" + +# --- cmake, pruned from the official Kitware release --- + +cmake_share_version=${cmake_version%.*} +cmake_archive_name="cmake-$cmake_version-linux-$cmake_release_arch.tar.gz" +cmake_archive="$downloads/$cmake_archive_name" +download_verified "$cmake_archive" \ + "https://github.com/Kitware/CMake/releases/download/v$cmake_version/$cmake_archive_name" \ + "$cmake_sha256" + +cmake_extract_root="$script_dir/artifacts/.extract-cmake-$arch" +rm -rf "$cmake_extract_root" +mkdir -p "$cmake_extract_root" +echo "prepare-portable-tools.sh: extracting $cmake_archive_name..." +tar -xzf "$cmake_archive" -C "$cmake_extract_root" +cmake_src="$cmake_extract_root/cmake-$cmake_version-linux-$cmake_release_arch" +[[ -d "$cmake_src" ]] || { echo "prepare-portable-tools.sh: unexpected archive layout, expected $cmake_src" >&2; exit 1; } + +mkdir -p "$work/share/cmake-$cmake_share_version" +cp -a "$cmake_src/bin/cmake" "$work/bin/" +cp -a "$cmake_src/share/cmake-$cmake_share_version/Modules" "$cmake_src/share/cmake-$cmake_share_version/Templates" \ + "$work/share/cmake-$cmake_share_version/" +rm -rf "$cmake_extract_root" + +# --- ninja, used as-is --- + +ninja_archive="$downloads/ninja-$ninja_version-$arch.zip" +download_verified "$ninja_archive" \ + "https://github.com/ninja-build/ninja/releases/download/v$ninja_version/$ninja_asset" \ + "$ninja_sha256" +echo "prepare-portable-tools.sh: staging ninja $ninja_version..." +unzip -oq "$ninja_archive" -d "$work/bin" +chmod +x "$work/bin/ninja" + +cat > "$work/LICENSE.txt" < "$smoke_dir/t.cpp" <<'EOF' +#include +#include +int main() { + std::vector v{1, 2, 3}; + int sum = 0; + for (int x : v) sum += x; + return sum == 6 ? 0 : 1; +} +EOF +"$work/bin/clang++" -std=c++20 -fuse-ld=lld "$smoke_dir/t.cpp" -o "$smoke_dir/t" +"$smoke_dir/t" + +# Also exercised together through CMake+Ninja, exactly how local-build.sh drives them - a plain +# clang++ invocation above would not catch a broken CMAKE_ROOT (Modules/Templates) or a Ninja that +# can't find the compiler. +cat > "$smoke_dir/CMakeLists.txt" <<'EOF' +cmake_minimum_required(VERSION 3.16) +project(smoke CXX) +add_executable(smoke t.cpp) +EOF +"$work/bin/cmake" -S "$smoke_dir" -B "$smoke_dir/build" -G Ninja \ + -DCMAKE_MAKE_PROGRAM="$work/bin/ninja" -DCMAKE_CXX_COMPILER="$work/bin/clang++" >/dev/null +"$work/bin/cmake" --build "$smoke_dir/build" >/dev/null +"$smoke_dir/build/smoke" + +rm -rf "$smoke_dir" +trap - EXIT + +mv "$work" "$toolchain_dir" +echo "prepare-portable-tools.sh: toolchain ready at $toolchain_dir ($(du -sh "$toolchain_dir" | cut -f1))"