From c3d809d276dc0e0c0ce0dae2abd50be72ab11f8a Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Mar 2025 09:29:31 -0800 Subject: [PATCH] Migrate to `zlib-rs` (again) (#11894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary I believe `zlib-rs` is now a better choice on ARM and x86, so I'm just going to assume it's a better choice everywhere. It's much easier to build (removes our CMake dependency), and in my benchmarking, it's substantially faster on ARM and faster or ~exactly even on my x86 Windows machine. We migrated to `zlib-rs` once before (#9184); however, I later reverted it as I learned that they were only doing compile-time feature detection, and so `zlib-rs` was meaningfully slower on x86. They now perform runtime feature detection: https://trifectatech.org/blog/zlib-rs-is-faster-than-c/. To benchmark, I wrote a script to create a local Simple API-compliant registry (see the commit history) for a single package. Then I ran the `install-cold` benchmark against that registry to install NumPy. On ARM: ``` ❯ uv run resolver --uv-pip-path ../../zlib-ng --uv-pip-path ../../zlib-rs \ --benchmark install-cold \ req.txt --warmup 10 --min-runs 30 Benchmark 1: ../../zlib-ng (install-cold) Time (mean ± σ): 165.7 ms ± 34.7 ms [User: 64.4 ms, System: 93.2 ms] Range (min … max): 141.8 ms … 293.2 ms 30 runs Benchmark 2: ../../zlib-rs (install-cold) Time (mean ± σ): 150.9 ms ± 16.2 ms [User: 57.4 ms, System: 86.4 ms] Range (min … max): 135.3 ms … 202.4 ms 30 runs Summary ../../zlib-rs (install-cold) ran 1.10 ± 0.26 times faster than ../../zlib-ng (install-cold) ``` I benchmarked this about 100 times on my Windows machine and found it difficult to conclude anything beyond "They're nearly the same". Here's an example: ``` PS C:\Users\crmar\workspace\puffin> hyperfine --prepare "uv venv" "zlib-rs.exe pip sync ./scripts/benchmark/req.txt" "zlib-ng.exe pip sync ./scripts/benchmark/req.txt" "zlib-rs.exe pip sync ./scripts/benchmark/req.txt" "zlib-ng.exe pip sync ./scripts/benchmark/req.txt" --runs 10 --warmup 5 Benchmark 1: zlib-rs.exe pip sync ./scripts/benchmark/req.txt Time (mean ± σ): 240.6 ms ± 10.8 ms [User: 6.1 ms, System: 92.2 ms] Range (min … max): 229.4 ms … 267.9 ms 10 runs Benchmark 2: zlib-ng.exe pip sync ./scripts/benchmark/req.txt Time (mean ± σ): 241.3 ms ± 6.2 ms [User: 7.7 ms, System: 90.6 ms] Range (min … max): 233.9 ms … 252.1 ms 10 runs Benchmark 3: zlib-rs.exe pip sync ./scripts/benchmark/req.txt Time (mean ± σ): 242.8 ms ± 7.7 ms [User: 6.2 ms, System: 23.4 ms] Range (min … max): 236.1 ms … 262.8 ms 10 runs Benchmark 4: zlib-ng.exe pip sync ./scripts/benchmark/req.txt Time (mean ± σ): 245.9 ms ± 5.7 ms [User: 1.5 ms, System: 59.4 ms] Range (min … max): 240.9 ms … 257.3 ms 10 runs Summary zlib-rs.exe pip sync ./scripts/benchmark/req.txt ran 1.00 ± 0.05 times faster than zlib-ng.exe pip sync ./scripts/benchmark/req.txt 1.01 ± 0.06 times faster than zlib-rs.exe pip sync ./scripts/benchmark/req.txt 1.02 ± 0.05 times faster than zlib-ng.exe pip sync ./scripts/benchmark/req.txt ``` Closes #11885. --- CONTRIBUTING.md | 23 +--- Cargo.lock | 30 ----- Cargo.toml | 3 +- crates/uv-dev/Cargo.toml | 5 +- .../uv-performance-flate2-backend/Cargo.lock | 108 ------------------ .../uv-performance-flate2-backend/Cargo.toml | 18 --- .../uv-performance-flate2-backend/src/lib.rs | 5 - crates/uv/Cargo.toml | 4 - pyproject.toml | 1 - 9 files changed, 5 insertions(+), 192 deletions(-) delete mode 100644 crates/uv-performance-flate2-backend/Cargo.lock delete mode 100644 crates/uv-performance-flate2-backend/Cargo.toml delete mode 100644 crates/uv-performance-flate2-backend/src/lib.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b69356013..e5fa1b237 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,31 +8,14 @@ which are good opportunities for new contributors. ## Setup -[Rust](https://rustup.rs/), a C compiler, and CMake are required to build uv. +[Rust](https://rustup.rs/) (and a C compiler) are required to build uv. -### Linux - -On Ubuntu and other Debian-based distributions, you can install the C compiler and CMake with: +On Ubuntu and other Debian-based distributions, you can install a C compiler with: ```shell -sudo apt install build-essential cmake +sudo apt install build-essential ``` -### macOS - -You can install CMake with Homebrew: - -```shell -brew install cmake -``` - -See the [Python](#python) section for instructions on installing the Python versions. - -### Windows - -You can install CMake from the [installers](https://cmake.org/download/) or with -`pipx install cmake`. - ## Testing For running tests, we recommend [nextest](https://nexte.st/). diff --git a/Cargo.lock b/Cargo.lock index 3cb7d3b00..77fa03001 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -645,15 +645,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" -[[package]] -name = "cmake" -version = "0.1.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" -dependencies = [ - "cc", -] - [[package]] name = "codspeed" version = "2.8.1" @@ -1126,7 +1117,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", - "libz-ng-sys", "libz-rs-sys", "miniz_oxide", ] @@ -2055,16 +2045,6 @@ dependencies = [ "redox_syscall 0.5.8", ] -[[package]] -name = "libz-ng-sys" -version = "1.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4436751a01da56f1277f323c80d584ffad94a3d14aecd959dd0dff75aa73a438" -dependencies = [ - "cmake", - "libc", -] - [[package]] name = "libz-rs-sys" version = "0.4.2" @@ -4581,7 +4561,6 @@ dependencies = [ "uv-normalize", "uv-pep440", "uv-pep508", - "uv-performance-flate2-backend", "uv-performance-memory-allocator", "uv-platform-tags", "uv-publish", @@ -4926,7 +4905,6 @@ dependencies = [ "uv-macros", "uv-options-metadata", "uv-pep508", - "uv-performance-flate2-backend", "uv-performance-memory-allocator", "uv-pypi-types", "uv-python", @@ -5336,14 +5314,6 @@ dependencies = [ "version-ranges", ] -[[package]] -name = "uv-performance-flate2-backend" -version = "0.1.0" -dependencies = [ - "flate2", - "libz-ng-sys", -] - [[package]] name = "uv-performance-memory-allocator" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 900eb5bc2..ce7e88fd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,6 @@ exclude = [ # Only used to pull in features, allocators, etc. — we specifically don't want them # to be part of a workspace-wide cargo check, cargo clippy, etc. "crates/uv-performance-memory-allocator", - "crates/uv-performance-flate2-backend", ] resolver = "2" @@ -100,7 +99,7 @@ dunce = { version = "1.0.5" } either = { version = "1.13.0" } encoding_rs_io = { version = "0.1.7" } etcetera = { version = "0.9.0" } -flate2 = { version = "1.0.33", default-features = false } +flate2 = { version = "1.0.33", default-features = false, features = ["zlib-rs"] } fs-err = { version = "3.0.0", features = ["tokio"] } fs2 = { version = "0.4.3" } futures = { version = "0.3.30" } diff --git a/crates/uv-dev/Cargo.toml b/crates/uv-dev/Cargo.toml index 955b1bb67..905b59dc5 100644 --- a/crates/uv-dev/Cargo.toml +++ b/crates/uv-dev/Cargo.toml @@ -54,7 +54,6 @@ tokio = { workspace = true } tracing = { workspace = true } tracing-durations-export = { workspace = true, features = ["plot"] } tracing-subscriber = { workspace = true } -uv-performance-flate2-backend = { path = "../uv-performance-flate2-backend", optional = true } uv-performance-memory-allocator = { path = "../uv-performance-memory-allocator", optional = true } walkdir = { workspace = true } @@ -72,12 +71,10 @@ default = ["performance"] dev = [] performance = [ "performance-memory-allocator", - "performance-flate2-backend", "uv-extract/performance" ] performance-memory-allocator = ["dep:uv-performance-memory-allocator"] -performance-flate2-backend = ["dep:uv-performance-flate2-backend"] render = ["poloto", "resvg", "tagu"] [package.metadata.cargo-shear] -ignored = ["flate2", "uv-extract", "uv-performance-memory-allocator", "uv-performance-flate2-backend"] +ignored = ["flate2", "uv-extract", "uv-performance-memory-allocator"] diff --git a/crates/uv-performance-flate2-backend/Cargo.lock b/crates/uv-performance-flate2-backend/Cargo.lock deleted file mode 100644 index 666f9097d..000000000 --- a/crates/uv-performance-flate2-backend/Cargo.lock +++ /dev/null @@ -1,108 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - -[[package]] -name = "cc" -version = "1.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cmake" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" -dependencies = [ - "cc", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "flate2" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" -dependencies = [ - "crc32fast", - "libz-ng-sys", - "libz-rs-sys", - "miniz_oxide", -] - -[[package]] -name = "libc" -version = "0.2.169" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" - -[[package]] -name = "libz-ng-sys" -version = "1.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4436751a01da56f1277f323c80d584ffad94a3d14aecd959dd0dff75aa73a438" -dependencies = [ - "cmake", - "libc", -] - -[[package]] -name = "libz-rs-sys" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" -dependencies = [ - "zlib-rs", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" -dependencies = [ - "adler2", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "uv-performance-flate2-backend" -version = "0.1.0" -dependencies = [ - "flate2", - "libz-ng-sys", -] - -[[package]] -name = "zlib-rs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" diff --git a/crates/uv-performance-flate2-backend/Cargo.toml b/crates/uv-performance-flate2-backend/Cargo.toml deleted file mode 100644 index 61249be4e..000000000 --- a/crates/uv-performance-flate2-backend/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "uv-performance-flate2-backend" -version = "0.1.0" -publish = false -edition = "2021" - -[lib] -doctest = false - -# Use `zlib-ng` on all supported platforms. -[target.'cfg(not(any(target_arch = "s390x", target_arch = "powerpc64", target_os = "freebsd")))'.dependencies] -flate2 = { version = "1.0.28", default-features = false, features = ["zlib-ng"] } -# See: https://github.com/rust-lang/libz-sys/issues/225 -libz-ng-sys = { version = "<1.1.20" } - -# Use `zlib-rs` everywhere else. -[target.'cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_os = "freebsd"))'.dependencies] -flate2 = { version = "1.0.28", default-features = false, features = ["zlib-rs"] } diff --git a/crates/uv-performance-flate2-backend/src/lib.rs b/crates/uv-performance-flate2-backend/src/lib.rs deleted file mode 100644 index 46340679c..000000000 --- a/crates/uv-performance-flate2-backend/src/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! The sole purpose of this crate is to enable one of -//! `flate2/zlib-ng` (on most platforms) or `flate2/rust_backend` -//! (on s390x, powerpc64, etc. — anywhere libz-ng doesn't build) -//! -//! See `Cargo.toml` diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index ff9b90b49..6dbd7b4a5 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -37,7 +37,6 @@ uv-installer = { workspace = true } uv-normalize = { workspace = true } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } -uv-performance-flate2-backend = { path = "../uv-performance-flate2-backend", optional = true } uv-performance-memory-allocator = { path = "../uv-performance-memory-allocator", optional = true } uv-platform-tags = { workspace = true } uv-publish = { workspace = true } @@ -130,7 +129,6 @@ nix = { workspace = true } ignored = [ "flate2", "uv-performance-memory-allocator", - "uv-performance-flate2-backend", ] [features] @@ -138,11 +136,9 @@ default = ["python", "python-managed", "pypi", "git", "performance", "crates-io" # Use better memory allocators, etc. performance = [ "performance-memory-allocator", - "performance-flate2-backend", "uv-distribution/performance", ] performance-memory-allocator = ["dep:uv-performance-memory-allocator"] -performance-flate2-backend = ["dep:uv-performance-flate2-backend"] tracing-durations-export = ["dep:tracing-durations-export", "uv-resolver/tracing-durations-export"] # Introduces a dependency on a local Python installation. diff --git a/pyproject.toml b/pyproject.toml index 84a61b872..0961ec7ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ include = [ { path = "rust-toolchain.toml", format = ["sdist", "wheel"] }, # this one isn't discovered by maturin because it's behind a feature flag { path = "crates/uv-performance-memory-allocator/**/*", format = ["sdist", "wheel"] }, - { path = "crates/uv-performance-flate2-backend/**/*", format = ["sdist", "wheel"] }, { path = "crates/uv-trampoline/trampolines/*", format = "sdist" }, { path = "LICENSE-APACHE", format = "sdist" }, { path = "LICENSE-MIT", format = "sdist" },