Add bootstrapped installation in Python for Windows (#1130)

A 1:1 port of the Bash script to Python for use on Windows.

Pulls some parts of #1068 but much more minimal. Avoids an additional
dependency on `requests`. Because we require `zstandard` to unzip the
distributions we unfortunately cannot be dependency free and cannot have
`bootstrap.sh` download the Python version needed to run this script
without it doing a non-trivial amount of work.

Retains the Bash script for now so you can bootstrap without Python
available. I may drop it in the future?

.github/workflows/ci.yml

@@ -80,16 +80,16 @@ jobs:
       - uses: actions/checkout@v4
       - name: "Install Rust toolchain"
         run: rustup component add clippy
-      - name: "Install Pythons"
+      - name: "Install Python for bootstrapping"
         uses: actions/setup-python@v4
         with:
-          python-version: |
+          python-version: 3.12
-            3.7
+      - name: "Install Python binaries"
-            3.8
+        run: |
-            3.9
+          pip install zstandard==0.22.0
-            3.10
+          python scripts/bootstrap/install.py
-            3.11
+          # ex) The path needs to be updated downstream
-            3.12
+          $env:Path = "$pwd\bin" + $env:Path
       - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
       - run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings

scripts/bootstrap/install.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+#
+# Download required Python versions and install to `bin`
+# Uses prebuilt Python distributions from indygreg/python-build-standalone
+#
+# This script can be run without Python installed via `install.sh`
+#
+# Requirements
+#
+#   pip install zstandard==0.22.0
+#
+# Usage
+#
+#   python scripts/bootstrap/install.py
+#
+# The Python versions are installed from `.python_versions`.
+# Python versions are linked in-order such that the _last_ defined version will be the default.
+#
+# Version metadata can be updated with `fetch-version-metadata.py`
+
+import hashlib
+import json
+import platform
+import shutil
+import sys
+import tarfile
+import tempfile
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+try:
+    import zstandard
+except ImportError:
+    print("ERROR: zstandard is required; install with `pip install zstandard==0.22.0`")
+    sys.exit(1)
+
+# Setup some file paths
+THIS_DIR = Path(__file__).parent
+ROOT_DIR = THIS_DIR.parent.parent
+BIN_DIR = ROOT_DIR / "bin"
+INSTALL_DIR = BIN_DIR / "versions"
+VERSIONS_FILE = ROOT_DIR / ".python-versions"
+VERSIONS_METADATA_FILE = THIS_DIR / "versions.json"
+
+# Map system information to those in the versions metadata
+ARCH_MAP = {"aarch64": "arm64", "amd64": "x86_64"}
+PLATFORM_MAP = {"win32": "windows"}
+PLATFORM = sys.platform
+ARCH = platform.machine().lower()
+INTERPRETER = "cpython"
+
+
+def decompress_file(archive_path: Path, output_path: Path):
+    if str(archive_path).endswith(".tar.zst"):
+        dctx = zstandard.ZstdDecompressor()
+
+        with tempfile.TemporaryFile(suffix=".tar") as ofh:
+            with archive_path.open("rb") as ifh:
+                dctx.copy_stream(ifh, ofh)
+            ofh.seek(0)
+            with tarfile.open(fileobj=ofh) as z:
+                z.extractall(output_path)
+    else:
+        raise ValueError(f"Unknown archive type {archive_path.suffix}")
+
+
+def sha256_file(path: Path):
+    h = hashlib.sha256()
+
+    with open(path, "rb") as file:
+        while True:
+            # Reading is buffered, so we can read smaller chunks.
+            chunk = file.read(h.block_size)
+            if not chunk:
+                break
+            h.update(chunk)
+
+    return h.hexdigest()
+
+
+versions_metadata = json.loads(VERSIONS_METADATA_FILE.read_text())
+versions = VERSIONS_FILE.read_text().splitlines()
+
+
+# Install each version
+for version in versions:
+    key = f"{INTERPRETER}-{version}-{PLATFORM_MAP.get(PLATFORM, PLATFORM)}-{ARCH_MAP.get(ARCH, ARCH)}"
+    install_dir = INSTALL_DIR / f"{INTERPRETER}@{version}"
+    already_exists = False
+    print(f"Installing {key}")
+
+    url = versions_metadata[key]["url"]
+
+    if not url:
+        print(f"No matching download for {key}")
+        sys.exit(1)
+
+    if not install_dir.exists():
+        filename = url.split("/")[-1]
+        print(f"Downloading {urllib.parse.unquote(filename)}")
+        download_path = THIS_DIR / filename
+        with urllib.request.urlopen(url) as response:
+            with download_path.open("wb") as download_file:
+                shutil.copyfileobj(response, download_file)
+
+        sha = versions_metadata[key]["sha256"]
+        if not sha:
+            print(f"WARNING: no checksum for {key}")
+        else:
+            print("Verifying checksum...", end="")
+            if sha256_file(download_path) != sha:
+                print(" FAILED!")
+                sys.exit(1)
+            print(" OK")
+
+        if install_dir.exists():
+            shutil.rmtree(install_dir)
+        print("Extracting to", install_dir)
+        install_dir.parent.mkdir(parents=True, exist_ok=True)
+
+        decompress_file(THIS_DIR / filename, install_dir.with_suffix(".tmp"))
+
+        # Setup the installation
+        (install_dir.with_suffix(".tmp") / "python").rename(install_dir)
+    else:
+        # We need to update executables even if the version is already downloaded and extracted
+        # to ensure that changes to the precedence of versions are respected
+        already_exists = True
+        print("Already available, skipping download")
+
+    # Use relative paths for links so if the bin is moved they don't break
+    executable = "." / install_dir.relative_to(BIN_DIR) / "install" / "bin" / "python3"
+    if PLATFORM == "win32":
+        executable = executable.with_suffix(".exe")
+
+    major = versions_metadata[key]["major"]
+    minor = versions_metadata[key]["minor"]
+
+    # Link as all version tuples, later versions in the file will take precedence
+    BIN_DIR.mkdir(parents=True, exist_ok=True)
+    targets = (
+        (BIN_DIR / f"python{version}"),
+        (BIN_DIR / f"python{major}.{minor}"),
+        (BIN_DIR / f"python{major}"),
+        (BIN_DIR / "python"),
+    )
+    for target in targets:
+        if PLATFORM == "win32":
+            target = target.with_suffix(".exe")
+
+        target.unlink(missing_ok=True)
+        target.symlink_to(executable)
+
+    if already_exists:
+        print(f"Updated executables for python{version}")
+    else:
+        print(f"Installed executables for python{version}")
+
+    # Cleanup
+    install_dir.with_suffix(".tmp").rmdir()
+    (THIS_DIR / filename).unlink()
+
+print("Done!")