From b3954f24495985e5de2b9a32a950aca110112835 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 19 Jan 2024 17:29:11 -0500 Subject: [PATCH] Enable PowerPC builds (#1017) Closes #1015. --- .github/workflows/build-binaries.yml | 146 ++++++++++++++++++++++++++- Cargo.lock | 1 + Cargo.toml | 5 +- crates/puffin/Cargo.toml | 10 ++ 4 files changed, 155 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index e3c0ae035..6c3c2e1fb 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -273,9 +273,7 @@ jobs: *.tar.gz *.sha256 - # TODO(charlie): Re-enable s390x-unknown-linux-gnu, powerpc64le-unknown-linux-gnu, and powerpc64-unknown-linux-gnu. - # s390x-unknown-linux-gnu works if we disable zlib-ng. The others fail in mysterious ways. - linux-cross: + linux-arm: runs-on: ubuntu-latest strategy: matrix: @@ -344,6 +342,148 @@ jobs: *.tar.gz *.sha256 + # Like `linux-arm`, but use `--no-default-features --features flate2-rust_backend` when + # building Puffin. + linux-s390x: + runs-on: ubuntu-latest + strategy: + matrix: + platform: + - target: s390x-unknown-linux-gnu + arch: s390x + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: "Prep README.md" + run: echo "# Puffin" > README.md + - name: "Build wheels" + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + manylinux: auto + docker-options: ${{ matrix.platform.maturin_docker_options }} + args: --release --locked --out dist --no-default-features --features flate2-rust_backend + - uses: uraimo/run-on-arch-action@v2 + if: matrix.platform.arch != 'ppc64' + name: Test wheel + with: + arch: ${{ matrix.platform.arch }} + distro: ubuntu20.04 + githubToken: ${{ github.token }} + install: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-pip + pip3 install -U pip + run: | + pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links dist/ --force-reinstall + ${{ env.MODULE_NAME }} --help + - name: "Upload wheels" + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + - name: "Archive binary" + shell: bash + run: | + set -euo pipefail + + TARGET=${{ matrix.platform.target }} + ARCHIVE_NAME=puffin-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/puffin $ARCHIVE_NAME/puffin + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME + shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 + - name: "Upload binary" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: | + *.tar.gz + *.sha256 + + + # Like `linux-arm`, but use `--no-default-features --features flate2-rust_backend` when + # building Puffin, and install the `gcc-powerpc64-linux-gnu` package. + linux-powerpc: + runs-on: ubuntu-latest + strategy: + matrix: + platform: + - target: powerpc64le-unknown-linux-gnu + arch: ppc64le + - target: powerpc64-unknown-linux-gnu + arch: ppc64 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: "Prep README.md" + run: echo "# Puffin" > README.md + - name: "Build wheels" + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + manylinux: auto + docker-options: ${{ matrix.platform.maturin_docker_options }} + args: --release --locked --out dist --no-default-features --features flate2-rust_backend + before-script-linux: | + if command -v yum &> /dev/null; then + yum update -y + yum -y install epel-release + yum repolist + yum install -y gcc-powerpc64-linux-gnu + fi + - uses: uraimo/run-on-arch-action@v2 + if: matrix.platform.arch != 'ppc64' + name: Test wheel + with: + arch: ${{ matrix.platform.arch }} + distro: ubuntu20.04 + githubToken: ${{ github.token }} + install: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-pip + pip3 install -U pip + run: | + pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links dist/ --force-reinstall + ${{ env.MODULE_NAME }} --help + - name: "Upload wheels" + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + - name: "Archive binary" + shell: bash + run: | + set -euo pipefail + + TARGET=${{ matrix.platform.target }} + ARCHIVE_NAME=puffin-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/puffin $ARCHIVE_NAME/puffin + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME + shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 + - name: "Upload binary" + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: | + *.tar.gz + *.sha256 + musllinux: runs-on: ubuntu-latest strategy: diff --git a/Cargo.lock b/Cargo.lock index 238b89389..516f679ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2266,6 +2266,7 @@ dependencies = [ "clap", "distribution-filename", "distribution-types", + "flate2", "fs-err", "futures", "gourgeist", diff --git a/Cargo.toml b/Cargo.toml index b847b2ed4..6af8963f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,10 +31,7 @@ data-encoding = { version = "2.5.0" } derivative = { version = "2.2.0" } directories = { version = "5.0.1" } dirs = { version = "5.0.1" } -# This tells flate2 (and all libraries that depend on it, including async_compression -# and async_zip) to use zlib-ng, which about 2x faster than the default flate2 backend -# at decompression. See https://github.com/rust-lang/flate2-rs#backends -flate2 = { version = "1.0.28", features = ["zlib-ng"], default-features = false } +flate2 = { version = "1.0.28", default-features = false } fs-err = { version = "2.11.0" } fs2 = { version = "0.4.3" } futures = { version = "0.3.30" } diff --git a/crates/puffin/Cargo.toml b/crates/puffin/Cargo.toml index 4d7f67ef1..1e6af229c 100644 --- a/crates/puffin/Cargo.toml +++ b/crates/puffin/Cargo.toml @@ -37,6 +37,11 @@ puffin-workspace = { path = "../puffin-workspace" } pypi-types = { path = "../pypi-types" } requirements-txt = { path = "../requirements-txt" } +# This tells flate2 (and all libraries that depend on it, including async_compression +# and async_zip) to use zlib-ng, which about 2x faster than the default flate2 backend +# at decompression. See https://github.com/rust-lang/flate2-rs#backends +flate2 = { workspace = true, default-features = false } + anstream = { workspace = true } anyhow = { workspace = true } bitflags = { workspace = true } @@ -80,6 +85,7 @@ predicates = { version = "3.0.4" } reqwest = { version = "0.11.23", features = ["blocking", "rustls"], default-features = false } [features] +default = ["flate2-zlib-ng"] # Introduces a dependency on a local Python installation. python = [] # Introduces a dependency on PyPI. @@ -88,3 +94,7 @@ pypi = [] git = [] # Introduces a dependency on Maturin. maturin = [] +# Adds the `libz-ng` feature to flate2. +flate2-zlib-ng = ["flate2/zlib-ng"] +# Adds the `rust_backend` feature to flate2. +flate2-rust_backend = ["flate2/rust_backend"]