feat: Facilitate reproducible builds on non-Windows systems (#270)

* fix: Use `DOWNLOAD_EXTRACT_TIMESTAMP` consistently

* fix: Set all extracted dependency files' timestamps to extraction time

Launcher/Prepare-NativePrebuilt.sh

@@ -22,11 +22,17 @@ workspace=$(cd "$script_dir/.." && pwd)
 
 arch=""
 output_dir=""
+toolchain_dir=""
+ninja_bin=""
+cmake_bin=""
+llvm_dir=""
+sysroot=""
 stage_dir="$workspace/build/native-prebuilt-stage"
 keep_stage=0
 reuse_stage=0
 parallel=0
 print_fingerprint_only=0
+disconnected=0
 
 usage() {
     cat <<'EOF'
@@ -55,6 +61,12 @@ while [[ $# -gt 0 ]]; do
         --reuse-stage) reuse_stage=1; shift ;;
         --parallel) parallel=$2; shift 2 ;;
         --print-fingerprint-only) print_fingerprint_only=1; shift ;;
+        --toolchain-dir) toolchain_dir=$2; shift 2 ;;
+        --ninja-bin) ninja_bin=$2; shift 2 ;;
+        --cmake-bin) cmake_bin=$2; shift 2 ;;
+        --llvm-dir) llvm_dir=$2; shift 2 ;;
+        --sysroot) sysroot=$2; shift 2 ;;
+        --disconnected) disconnected=1; shift ;;
         -h|--help) usage; exit 0 ;;
         *) echo "Prepare-NativePrebuilt.sh: unknown argument: $1" >&2; exit 1 ;;
     esac
@@ -71,14 +83,22 @@ assert_dir() { [[ -d "$1" ]] || fail "$2 is missing: $1"; }
 sha256_of() { sha256sum "$1" | awk '{print $1}'; }
 normalize() { readlink -f "$1"; }
 
+# Environment check: Ensure the `SOURCE_DATE_EPOCH` value is not invalid
+# before trying to build anything (fail-fast)
+if [[ -n "${SOURCE_DATE_EPOCH:-}" ]] && ! python3 -c \
+    'import datetime, os; datetime.datetime.fromtimestamp(int(os.environ.get("SOURCE_DATE_EPOCH")), datetime.timezone.utc)'
+then
+    fail "SOURCE_DATE_EPOCH must be a valid UNIX timestamp"
+fi
+
 [[ -n "$output_dir" ]] || output_dir="$script_dir/artifacts/native-prebuilt-$arch"
 [[ "$stage_dir" = /* ]] || stage_dir="$workspace/$stage_dir"
 
-toolchain_dir="$script_dir/artifacts/portable-tools/toolchain-$arch"
-cc="$toolchain_dir/bin/clang"
-cxx="$toolchain_dir/bin/clang++"
-cmake_bin="$toolchain_dir/bin/cmake"
-ninja_bin="$toolchain_dir/bin/ninja"
+toolchain_dir="${toolchain_dir:-$script_dir/artifacts/portable-tools/toolchain-$arch}"
+cc="${llvm_dir:-$toolchain_dir}/bin/clang"
+cxx="${llvm_dir:-$toolchain_dir}/bin/clang++"
+cmake_bin="${cmake_bin:-$toolchain_dir/bin/cmake}"
+ninja_bin="${ninja_bin:-$toolchain_dir/bin/ninja}"
 runtime_source="$workspace/runtime"
 aurora_source="$workspace/aurora-main"
 
@@ -87,7 +107,7 @@ assert_file "$ninja_bin" "Portable Ninja"
 assert_file "$cc" "Portable C compiler"
 assert_file "$cxx" "Portable C++ compiler"
 assert_dir "$aurora_source" "aurora-main source tree"
-clang_binary=$(normalize "$toolchain_dir/bin/clang-22")
+clang_binary=$(normalize "${llvm_dir:-$toolchain_dir}/bin/clang-22")
 assert_file "$clang_binary" "Portable clang driver binary"
 
 (( parallel > 0 )) || parallel=$(nproc)
@@ -142,6 +162,16 @@ fixed_configure_flags=(
     -DFT_DISABLE_BZIP2=ON
     -DCMAKE_POLICY_DEFAULT_CMP0168=NEW
 )
+if [[ "$disconnected" -eq 1 ]]; then
+    fixed_configure_flags+=(
+        -DFETCHCONTENT_FULLY_DISCONNECTED=ON
+    )
+fi
+if [[ -n "$sysroot" ]]; then
+    fixed_configure_flags+=(
+        -DCMAKE_SYSROOT="$sysroot"
+    )
+fi
 flag_fingerprint=$(printf '%s\n' "${fixed_configure_flags[@]}" | sha256sum | awk '{print $1}')
 
 # extern/ is excluded because the payload ships that tree separately (aurora-main/extern is bundled
@@ -154,7 +184,12 @@ third_party_fingerprint=$(fingerprint_tree "$runtime_source/third_party")
 [[ -n "$third_party_fingerprint" ]] || fail "The vendored third-party tree could not be fingerprinted: $runtime_source/third_party"
 mbedtls_fingerprint=$(sha256_of "$runtime_source/cmake/MbedTLSPin.cmake")
 
-compiler_sha256=$(sha256_of "$clang_binary")
+compiler_sha256=$(sha256_of "$cc")
+cxx_sha256=$(sha256_of "$cxx")
+# Append the CXX SHA256 if it doesn't match the hash of CC
+if [[ "${cxx_sha256}" != "${compiler_sha256}" ]]; then
+    compiler_sha256="${compiler_sha256}:${cxx_sha256}"
+fi
 
 if [[ "$print_fingerprint_only" -eq 1 ]]; then
     printf 'compiler_sha256=%s\n' "$compiler_sha256"
@@ -372,7 +407,7 @@ while IFS='|' read -r name linkerfile; do
     [[ -n "$ref" ]] || fail "Mbed TLS archive was not harvested: $name ($linkerfile)"
     mbedtls_refs+=("$ref")
 done < "$mbedtls_txt"
-[[ ${#mbedtls_refs[@]} -eq 3 ]] || fail "Expected three Mbed TLS archives, got ${#mbedtls_refs[@]}"
+[[ ${#mbedtls_refs[@]} -ge 3 ]] || fail "Expected at least three Mbed TLS archives, got ${#mbedtls_refs[@]}"
 mbedtls_source_dir=$(get_meta mbedtls_source_dir)
 assert_dir "$mbedtls_source_dir/include/mbedtls" "Mbed TLS headers"
 mkdir -p "$output_dir/include/mbedtls"
@@ -582,9 +617,21 @@ for root, dirs, files in os.walk(output_dir):
         contents.append({"Path": rel, "Bytes": os.path.getsize(path), "Sha256": digest})
 contents.sort(key=lambda c: c["Path"])
 
+source_date_epoch = os.environ.get("SOURCE_DATE_EPOCH") or None
+utc = datetime.timezone.utc
+if source_date_epoch is not None:
+    built_utc = datetime.datetime.fromtimestamp(
+        int(source_date_epoch),
+        utc
+    )
+else:
+    built_utc = datetime.datetime.now(utc)
+
+built_utc = built_utc.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
 provenance = {
     "SchemaVersion": 1,
-    "BuiltUtc": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
+    "BuiltUtc": built_utc,
     "CompilerSha256": compiler_sha256,
     "CompilerVersion": compiler_version,
     "FlagFingerprint": flag_fingerprint,

Launcher/prepare-portable-tools.sh

@@ -225,7 +225,7 @@ EOF
 # clang++ invocation above would not catch a broken CMAKE_ROOT (Modules/Templates) or a Ninja that
 # can't find the compiler.
 cat > "$test_dir/CMakeLists.txt" <<'EOF'
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.25)
 project(test CXX)
 add_executable(test t.cpp)
 EOF

aurora-main/cmake/AuroraDawnProvider.cmake

@@ -101,7 +101,7 @@ if (_aurora_dawn_provider STREQUAL "vendor")
     include(FetchContent)
     FetchContent_Declare(dawn
       URL "https://github.com/google/dawn/archive/refs/tags/${AURORA_DAWN_VERSION}.tar.gz"
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       EXCLUDE_FROM_ALL
     )
     FetchContent_MakeAvailable(dawn)
@@ -194,7 +194,7 @@ elseif (_aurora_dawn_provider STREQUAL "package")
   FetchContent_Declare(dawn_prebuilt
     URL "${AURORA_DAWN_PACKAGE_URL}"
     ${_dawn_prebuilt_hash_argument}
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
   )
   FetchContent_MakeAvailable(dawn_prebuilt)
 

aurora-main/cmake/AuroraLibUSB.cmake

@@ -7,7 +7,7 @@
 include(FetchContent)
 FetchContent_Declare(libusb
   URL "https://github.com/libusb/libusb/releases/download/v${AURORA_LIBUSB_VERSION}/libusb-${AURORA_LIBUSB_VERSION}.tar.bz2"
-  DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+  DOWNLOAD_EXTRACT_TIMESTAMP FALSE
 )
 # Upstream ships no CMakeLists.txt, so this only populates the source tree.
 FetchContent_MakeAvailable(libusb)

aurora-main/cmake/AuroraNodProvider.cmake

@@ -120,7 +120,7 @@ elseif (_aurora_nod_provider STREQUAL "package")
   include(FetchContent)
   FetchContent_Declare(nod_prebuilt
     URL "${AURORA_NOD_PACKAGE_URL}"
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
     EXCLUDE_FROM_ALL
   )
   FetchContent_MakeAvailable(nod_prebuilt)

aurora-main/cmake/AuroraSDL3Provider.cmake

@@ -92,7 +92,7 @@ elseif (_aurora_sdl3_provider STREQUAL "package")
   include(FetchContent)
   FetchContent_Declare(sdl3_prebuilt
     URL "${AURORA_SDL3_PACKAGE_URL}"
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
   )
   FetchContent_MakeAvailable(sdl3_prebuilt)
 
@@ -145,7 +145,7 @@ elseif (_aurora_sdl3_provider STREQUAL "vendor")
     endif ()
     FetchContent_Declare(SDL
       URL "https://github.com/libsdl-org/SDL/releases/download/release-${AURORA_SDL3_VERSION}/SDL3-${AURORA_SDL3_VERSION}.tar.gz"
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       PATCH_COMMAND "${CMAKE_COMMAND}" -DSDL_SOURCE_DIR=<SOURCE_DIR> -P "${_aurora_sdl3_patches}"
       EXCLUDE_FROM_ALL
     )

aurora-main/extern/CMakeLists.txt

@@ -38,7 +38,7 @@ else ()
   set(ABSL_PROPAGATE_CXX_STD ON)
   FetchContent_Declare(abseil-cpp
     URL https://github.com/abseil/abseil-cpp/archive/refs/tags/20240722.0.tar.gz
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
     EXCLUDE_FROM_ALL
   )
   FetchContent_MakeAvailable(abseil-cpp)
@@ -55,7 +55,7 @@ if (NOT TARGET xxhash)
     URL https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.4.tar.gz
     URL_HASH SHA256=5738270935e7c3d38a79b3adf7c9692566ce7895a25f67de43ad52ab504acd32
     SOURCE_SUBDIR build/cmake
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
     EXCLUDE_FROM_ALL
   )
   set(XXHASH_BUILD_XXHSUM OFF CACHE INTERNAL "Build the xxhsum binary")
@@ -78,7 +78,7 @@ if (NOT TARGET fmt)
   FetchContent_Declare(fmt
     URL https://github.com/fmtlib/fmt/archive/refs/tags/11.1.4.tar.gz
     URL_HASH SHA256=ac366b7b4c2e9f0dde63a59b3feb5ee59b67974b14ee5dc9ea8ad78aa2c1ee1e
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
     EXCLUDE_FROM_ALL
   )
   FetchContent_MakeAvailable(fmt)
@@ -116,7 +116,7 @@ if (AURORA_ENABLE_GX)
       ZLIB
       URL https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz
       URL_HASH SHA256=bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       EXCLUDE_FROM_ALL
     )
     FetchContent_MakeAvailable(ZLIB)
@@ -152,7 +152,7 @@ if (AURORA_ENABLE_GX)
       PNG
       URL https://github.com/pnggroup/libpng/archive/refs/tags/v1.6.58.tar.gz
       URL_HASH SHA256=A9D4DF463D36A6E5F9C29BD6F4967312D17E996C1854F3511F833924EB1993CF
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       EXCLUDE_FROM_ALL
     )
     FetchContent_MakeAvailable(PNG)
@@ -176,7 +176,7 @@ if (AURORA_ENABLE_GX)
     FetchContent_Declare(Freetype
       URL https://files.twilitrealm.dev/freetype-2.14.3.tar.gz
       URL_HASH SHA256=e61b31ab26358b946e767ed7eb7f4bb2e507da1cfefeb7a8861ace7fd5c899a1
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       EXCLUDE_FROM_ALL
     )
     FetchContent_MakeAvailable(Freetype)
@@ -190,7 +190,7 @@ if (AURORA_ENABLE_GX)
     FetchContent_Declare(imgui
       URL https://github.com/ocornut/imgui/archive/refs/tags/v1.91.9b-docking.tar.gz
       URL_HASH SHA256=466fdef9b18de15f0bb6e288e3d00ffa3d82200ec458ce5e4f724a161d9528a5
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       EXCLUDE_FROM_ALL
     )
     FetchContent_MakeAvailable(imgui)
@@ -235,7 +235,7 @@ if (AURORA_ENABLE_GX)
     FetchContent_Declare(sqlite3
       URL https://sqlite.org/2026/sqlite-amalgamation-3510300.zip
       URL_HASH SHA256=acb1e6f5d832484bf6d32b681e858c38add8b2acdfd42ac5df24b8afb46552b4
-      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+      DOWNLOAD_EXTRACT_TIMESTAMP FALSE
       EXCLUDE_FROM_ALL
     )
     FetchContent_MakeAvailable(sqlite3)
@@ -278,7 +278,7 @@ if (AURORA_ENABLE_GX)
       FetchContent_Declare(zstd
         URL https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz
         URL_HASH SHA256=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3
-        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+        DOWNLOAD_EXTRACT_TIMESTAMP FALSE
         SOURCE_SUBDIR build/cmake
         EXCLUDE_FROM_ALL
       )
@@ -299,7 +299,7 @@ if (NOT TARGET TracyClient)
           tracy
           URL https://github.com/wolfpld/tracy/archive/a64b9a20294d59421a2f57aeca3c6383d8c48169.tar.gz
           URL_HASH SHA256=24d342b5127d7f659dc3cf94f24347b348cc736f25b17a691fd7f69541937658
-          DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+          DOWNLOAD_EXTRACT_TIMESTAMP FALSE
           EXCLUDE_FROM_ALL
   )
   FetchContent_MakeAvailable(tracy)

aurora-main/tests/CMakeLists.txt

@@ -5,7 +5,7 @@ if (NOT TARGET gtest)
   FetchContent_Declare(googletest
     URL https://github.com/google/googletest/archive/refs/tags/v1.17.0.tar.gz
     URL_HASH SHA256=65fab701d9829d38cb77c14acdc431d2108bfdbf8979e40eb8ae567edf10b27c
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    DOWNLOAD_EXTRACT_TIMESTAMP FALSE
     EXCLUDE_FROM_ALL
   )
   FetchContent_MakeAvailable(googletest)

runtime/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.25)
 project(mkw_recompiled)
 
 if(NOT CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang)$" OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
@@ -132,7 +132,9 @@ if(NOT MKW_PLATFORM_WINDOWS)
         include(FetchContent)
         FetchContent_Declare(mkw_mbedtls_upstream
             URL "https://github.com/Mbed-TLS/mbedtls/releases/download/mbedtls-${MKW_MBEDTLS_VERSION}/mbedtls-${MKW_MBEDTLS_VERSION}.tar.bz2"
-            URL_HASH SHA256=${MKW_MBEDTLS_SHA256})
+            URL_HASH SHA256=${MKW_MBEDTLS_SHA256}
+            DOWNLOAD_EXTRACT_TIMESTAMP FALSE
+        )
         # Subproject mode already disables testing and codegen; it still enables programs.
         set(ENABLE_PROGRAMS OFF CACHE BOOL "" FORCE)
         set(ENABLE_TESTING OFF CACHE BOOL "" FORCE)

runtime/cmake/NativePrebuiltExport.cmake

@@ -88,12 +88,21 @@ mkw_collect_buildsystem_targets("${_mkw_np_aurora_dir}" _mkw_np_all_targets)
 # checks below still apply to it.
 list(APPEND _mkw_np_all_targets "mkw_cryptopp")
 if(NOT MKW_PLATFORM_WINDOWS)
-    list(APPEND _mkw_np_all_targets mbedtls mbedx509 mbedcrypto)
-    # Keep the TLS archives separate from aurora's aggregate link interface.
-    string(CONCAT _mkw_np_mbedtls_lines
-        "mbedtls|$<TARGET_LINKER_FILE:MbedTLS::mbedtls>\n"
-        "mbedx509|$<TARGET_LINKER_FILE:MbedTLS::mbedx509>\n"
-        "mbedcrypto|$<TARGET_LINKER_FILE:MbedTLS::mbedcrypto>\n")
+    set(_mkw_np_mbedtls_targets mbedtls mbedx509 mbedcrypto)
+    # mbedcrypto links these third-party archives when upstream defines them.
+    # Keep dependencies after mbedcrypto for static linking on the consumer side.
+    foreach(_t everest p256m)
+        if(TARGET ${_t})
+            list(APPEND _mkw_np_mbedtls_targets ${_t})
+        endif()
+    endforeach()
+    list(APPEND _mkw_np_all_targets ${_mkw_np_mbedtls_targets})
+    # Keep the complete TLS archive set separate from aurora's aggregate link interface.
+    set(_mkw_np_mbedtls_lines "")
+    foreach(_t IN LISTS _mkw_np_mbedtls_targets)
+        string(APPEND _mkw_np_mbedtls_lines
+            "${_t}|$<TARGET_LINKER_FILE:${_t}>\n")
+    endforeach()
     file(GENERATE OUTPUT "${MKW_NATIVE_PREBUILT_EXPORT_DIR}/mbedtls.txt"
         CONTENT "${_mkw_np_mbedtls_lines}")
 endif()

translator/README.md

@@ -18,7 +18,7 @@ entry point, decodes every reachable function, and emits C++ (plus JSON metadata
 what was emitted).
 
 2.  **`generate-data-init --project <manifest>`** - writes the embedded `.data`/`.rodata`/`.sdata`
-section initializer and `RuntimeConfig.h`. 
+section initializer and `RuntimeConfig.h`.
 
 3.  **`emit-build-shards --project <manifest>`** - emits the CMake build graph (`shards.cmake`)
 covering both generated sources and `runtime/src`.
@@ -36,7 +36,7 @@ See `projects/examples/generic-dol.yml` for a minimal manifest driven by `RECOMP
 | Tool | Notes |
 | --- | --- |
 | .NET 8 SDK | Builds and runs the translator. |
-| CMake ≥ 3.16 and Ninja | Configures and drives the native build. |
+| CMake ≥ 3.25 and Ninja | Configures and drives the native build. |
 | Clang / LLVM | The shipped build uses LLVM-MinGW targeting `x86-64-v3`. MSVC is not the tested path. |
 
 Build the CLI once and invoke the assembly directly: