From 44a77a04d0d419bbfc52e4a7b06da1c94f54e77b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 28 Jul 2024 18:03:52 -0400 Subject: [PATCH] Move `bench` directory to `benchmark` (#5529) ## Summary Removes the legacy `benchmark` directory (we'll always have it in Git) and renames `bench` to `benchmark` for clarity. Fixes a variety of commands and references. --- BENCHMARKS.md | 21 ++++---- CONTRIBUTING.md | 14 +++--- scripts/bench/README.md | 14 ------ scripts/{bench => benchmark}/.python-version | 0 scripts/benchmark/README.md | 16 ++++++ scripts/{bench => benchmark}/pyproject.toml | 4 +- .../src/benchmark}/__init__.py | 49 +++++++++---------- scripts/{bench => benchmark}/uv.lock | 2 +- scripts/benchmarks/compile.sh | 27 ---------- scripts/benchmarks/requirements-large.txt | 49 ------------------- scripts/benchmarks/requirements.in | 32 ------------ scripts/benchmarks/requirements.txt | 10 ---- scripts/benchmarks/sync.sh | 39 --------------- scripts/benchmarks/uninstall.sh | 17 ------- scripts/benchmarks/venv.sh | 33 ------------- 15 files changed, 60 insertions(+), 267 deletions(-) delete mode 100644 scripts/bench/README.md rename scripts/{bench => benchmark}/.python-version (100%) create mode 100644 scripts/benchmark/README.md rename scripts/{bench => benchmark}/pyproject.toml (86%) rename scripts/{bench/src/bench => benchmark/src/benchmark}/__init__.py (97%) rename scripts/{bench => benchmark}/uv.lock (99%) delete mode 100755 scripts/benchmarks/compile.sh delete mode 100644 scripts/benchmarks/requirements-large.txt delete mode 100644 scripts/benchmarks/requirements.in delete mode 100644 scripts/benchmarks/requirements.txt delete mode 100755 scripts/benchmarks/sync.sh delete mode 100755 scripts/benchmarks/uninstall.sh delete mode 100755 scripts/benchmarks/venv.sh diff --git a/BENCHMARKS.md b/BENCHMARKS.md index cc64b84c7..bf51fde26 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -54,41 +54,44 @@ manager cache is not shared across runs). ## Reproduction -All benchmarks were generated using the `scripts/bench/__main__.py` script, which wraps +All benchmarks were generated using the `scripts/benchmark` package, which wraps [`hyperfine`](https://github.com/sharkdp/hyperfine) to facilitate benchmarking uv against a variety of other tools. The benchmark script itself has a several requirements: - A local uv release build (`cargo build --release`). -- A virtual environment with the script's own dependencies installed (`uv venv && uv pip sync scripts/bench/requirements.txt`). +- An installation of the production `uv` binary in your path. - The [`hyperfine`](https://github.com/sharkdp/hyperfine) command-line tool installed on your system. To benchmark resolution against pip-compile, Poetry, and PDM: ```shell -python -m scripts.bench \ - --uv \ +uv run benchmark \ + --uv-pip \ --poetry \ --pdm \ --pip-compile \ --benchmark resolve-warm --benchmark resolve-cold \ - scripts/requirements/trio.in \ - --json + --json \ + ../requirements/trio.in ``` To benchmark installation against pip-sync, Poetry, and PDM: ```shell -python -m scripts.bench \ - --uv \ +uv run benchmark \ + --uv-pip \ --poetry \ --pdm \ --pip-sync \ --benchmark install-warm --benchmark install-cold \ - --json + --json \ + ../requirements/compiled/trio.txt ``` +Both commands should be run from the `scripts/benchmark` directory. + After running the benchmark script, you can generate the corresponding graph via: ```shell diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f52e96a6..52d4a7d62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,15 +83,15 @@ Please refer to Ruff's [Profiling Guide](https://github.com/astral-sh/ruff/blob/ We provide diverse sets of requirements for testing and benchmarking the resolver in `scripts/requirements` and for the installer in `scripts/requirements/compiled`. -You can use `scripts/bench` to benchmark predefined workloads between uv versions and with other tools, e.g., from the `scripts/bench` directory: +You can use `scripts/benchmark` to benchmark predefined workloads between uv versions and with other tools, e.g., from the `scripts/benchmark` directory: ```shell -uv run bench \ - --uv-path ./target/release/before \ - --uv-path ./target/release/after \ - ../scripts/requirements/jupyter.in \ - --benchmark resolve-cold \ - --min-runs 20 +uv run benchmark \ + --uv-pip \ + --poetry \ + --benchmark \ + resolve-cold \ + ../scripts/requirements/trio.in ``` ### Analyzing concurrency diff --git a/scripts/bench/README.md b/scripts/bench/README.md deleted file mode 100644 index 4d9a98dd7..000000000 --- a/scripts/bench/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# bench - -Benchmarking scripts for uv and other package management tools. - -## Getting Started - -From the `bench` directory: - -```shell -uv run __main__.py \ - --uv-pip \ - --poetry \ - ../scripts/requirements/trio.in --benchmark resolve-cold --min-runs 20 -``` diff --git a/scripts/bench/.python-version b/scripts/benchmark/.python-version similarity index 100% rename from scripts/bench/.python-version rename to scripts/benchmark/.python-version diff --git a/scripts/benchmark/README.md b/scripts/benchmark/README.md new file mode 100644 index 000000000..d0936ccb8 --- /dev/null +++ b/scripts/benchmark/README.md @@ -0,0 +1,16 @@ +# benchmark + +Benchmarking scripts for uv and other package management tools. + +## Getting Started + +From the `scripts/benchmark` directory: + +```shell +uv run benchmark \ + --uv-pip \ + --poetry \ + --benchmark \ + resolve-cold \ + ../requirements/trio.in +``` diff --git a/scripts/bench/pyproject.toml b/scripts/benchmark/pyproject.toml similarity index 86% rename from scripts/bench/pyproject.toml rename to scripts/benchmark/pyproject.toml index 85874b7de..e4af335db 100644 --- a/scripts/bench/pyproject.toml +++ b/scripts/benchmark/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "bench" +name = "benchmark" version = "0.0.1" description = "Benchmark package resolution tools" requires-python = ">=3.12" @@ -13,7 +13,7 @@ dependencies = [ ] [project.scripts] -bench = "bench:main" +benchmark = "benchmark:main" [build-system] requires = ["hatchling"] diff --git a/scripts/bench/src/bench/__init__.py b/scripts/benchmark/src/benchmark/__init__.py similarity index 97% rename from scripts/bench/src/bench/__init__.py rename to scripts/benchmark/src/benchmark/__init__.py index 5e6d58c7d..b0e80ef71 100644 --- a/scripts/bench/src/bench/__init__.py +++ b/scripts/benchmark/src/benchmark/__init__.py @@ -6,58 +6,53 @@ By default, this script also assumes that `pip`, `pip-tools`, `virtualenv`, `poe `hyperfine` are installed, and that a uv release builds exists at `./target/release/uv` (relative to the repository root). However, the set of tools is configurable. -To set up the required environment, run: +For example, to benchmark uv's `pip compile` command against `pip-tools`, run the +following from the `scripts/benchmark` directory: - cargo build --release - ./target/release/uv venv - source .venv/bin/activate - ./target/release/uv pip sync ./scripts/bench/requirements.txt - -Then, to benchmark uv against `pip-tools`: - - python -m scripts.bench --uv --pip-compile requirements.in + uv run benchmark --uv-pip --pip-compile ../requirements/trio.in It's most common to benchmark multiple uv versions against one another by building from multiple branches and specifying the path to each binary, as in: - # Build the baseline version. + # Build the baseline version, from the repo root. git checkout main cargo build --release mv ./target/release/uv ./target/release/baseline - # Build the feature version. + # Build the feature version, again from the repo root. git checkout feature cargo build --release # Run the benchmark. - python -m scripts.bench \ - --uv-path ./target/release/uv \ - --uv-path ./target/release/baseline \ - requirements.in + cd scripts/benchmark + uv run benchmark \ + --uv-pip-path ../../target/release/uv \ + --uv-pip-path ../../target/release/baseline \ + ../requirements/trio.in By default, the script will run the resolution benchmarks when a `requirements.in` file is provided, and the installation benchmarks when a `requirements.txt` file is provided: # Run the resolution benchmarks against the Trio project. - python -m scripts.bench \ - --uv-path ./target/release/uv \ - --uv-path ./target/release/baseline \ - ./scripts/requirements/trio.in + uv run bench \ + --uv-path ../../target/release/uv \ + --uv-path ../../target/release/baseline \ + ../requirements/trio.in # Run the installation benchmarks against the Trio project. - python -m scripts.bench \ - --uv-path ./target/release/uv \ - --uv-path ./target/release/baseline \ - ./scripts/requirements/compiled/trio.txt + uv run bench \ + --uv-path ../../target/release/uv \ + --uv-path ../../target/release/baseline \ + ../requirements/compiled/trio.txt You can also specify the benchmark to run explicitly: # Run the "uncached install" benchmark against the Trio project. - python -m scripts.bench \ - --uv-path ./target/release/uv \ - --uv-path ./target/release/baseline \ + uv run bench \ + --uv-path ../../target/release/uv \ + --uv-path ../../target/release/baseline \ --benchmark install-cold \ - ./scripts/requirements/compiled/trio.txt + ../requirements/compiled/trio.txt """ import abc diff --git a/scripts/bench/uv.lock b/scripts/benchmark/uv.lock similarity index 99% rename from scripts/bench/uv.lock rename to scripts/benchmark/uv.lock index 78733f492..c5b3f64c4 100644 --- a/scripts/bench/uv.lock +++ b/scripts/benchmark/uv.lock @@ -15,7 +15,7 @@ wheels = [ ] [[distribution]] -name = "bench" +name = "benchmark" version = "0.0.1" source = { editable = "." } dependencies = [ diff --git a/scripts/benchmarks/compile.sh b/scripts/benchmarks/compile.sh deleted file mode 100755 index 70e23fc30..000000000 --- a/scripts/benchmarks/compile.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env sh - -### -# Benchmark the resolver against `pip-compile`. -# -# Example usage: -# -# ./scripts/benchmarks/compile.sh ./scripts/benchmarks/requirements.in -### - -set -euxo pipefail - -TARGET=${1} - -### -# Resolution with a cold cache. -### -hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \ - "./target/release/uv --no-cache pip-compile ${TARGET} > /tmp/requirements.txt" \ - "./target/release/main --no-cache pip-compile ${TARGET} > /tmp/requirements.txt" - -### -# Resolution with a warm cache. -### -hyperfine --runs 20 --warmup 3 --prepare "rm -f /tmp/requirements.txt" \ - "./target/release/uv pip compile ${TARGET} > /tmp/requirements.txt" \ - "./target/release/main pip-compile ${TARGET} > /tmp/requirements.txt" diff --git a/scripts/benchmarks/requirements-large.txt b/scripts/benchmarks/requirements-large.txt deleted file mode 100644 index 21be1ef3a..000000000 --- a/scripts/benchmarks/requirements-large.txt +++ /dev/null @@ -1,49 +0,0 @@ -### -# A large-ish set of dependencies, including several packages with a large number of small files -# (like Django) and several packages with a small number of large files (like Ruff). -### -pygments==2.16.1 -packaging==23.2 -click==8.1.7 -threadpoolctl==3.2.0 -flake8-docstrings==1.7.0 -pytest==7.4.2 -mdurl==0.1.2 -typeguard==3.0.2 -tokenize-rt==5.2.0 -typing-extensions==4.8.0 -markupsafe==2.1.3 -attrs==23.1.0 -lsprotocol==2023.0.0b1 -markdown-it-py==3.0.0 -joblib==1.3.2 -cattrs==23.1.2 -tomlkit==0.12.1 -mccabe==0.7.0 -iniconfig==2.0.0 -rich==13.6.0 -django==5.0a1 -isort==5.12.0 -flake8==6.1.0 -snowballstemmer==2.2.0 -pycodestyle==2.11.0 -mypy-extensions==1.0.0 -pluggy==1.3.0 -pyflakes==3.1.0 -pydocstyle==6.3.0 -scipy==1.11.3 -jinja2==3.1.2 -ruff==0.0.292 -pygls==1.1.1 -pyupgrade==3.15.0 -platformdirs==3.11.0 -pylint==3.0.1 -pathspec==0.11.2 -astroid==3.0.0 -dill==0.3.7 -scikit-learn==1.3.1 -mypy==1.5.1 -numpy==1.26.0 -asgiref==3.7.2 -black==23.9.1 -sqlparse==0.4.4 diff --git a/scripts/benchmarks/requirements.in b/scripts/benchmarks/requirements.in deleted file mode 100644 index 7173573c4..000000000 --- a/scripts/benchmarks/requirements.in +++ /dev/null @@ -1,32 +0,0 @@ -### -# A small set of pure-Python packages. -### -packaging>=23.1 -pygls>=1.0.1 -lsprotocol>=2023.0.0a1 -ruff>=0.0.274 -flask @ git+https://github.com/pallets/flask.git@d92b64a -typing_extensions -scipy -numpy -pandas<2.0.0 -matplotlib>=3.0.0 -scikit-learn -rich -textual -jupyter>=1.0.0,<2.0.0 -transformers[torch] -django<4.0.0 -sqlalchemy -psycopg2-binary -trio<0.20 -trio-websocket -trio-asyncio -trio-typing -trio-protocol -fastapi -typer -pydantic -uvicorn -traitlets - diff --git a/scripts/benchmarks/requirements.txt b/scripts/benchmarks/requirements.txt deleted file mode 100644 index 3992d867b..000000000 --- a/scripts/benchmarks/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -### -# A small set of pure-Python packages. -### -attrs==23.1.0 -cattrs==23.1.2 -lsprotocol==2023.0.0b1 -packaging==23.2 -pygls==1.1.1 -typeguard==3.0.2 -typing-extensions==4.8.0 diff --git a/scripts/benchmarks/sync.sh b/scripts/benchmarks/sync.sh deleted file mode 100755 index 9550341ef..000000000 --- a/scripts/benchmarks/sync.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -### -# Benchmark the installer against `pip`. -# -# Example usage: -# -# ./scripts/benchmarks/sync.sh ./scripts/benchmarks/requirements.txt -### - -set -euxo pipefail - -TARGET=${1} - -### -# Installation with a cold cache. -### -hyperfine --runs 20 --warmup 3 \ - --prepare "virtualenv --clear .venv" \ - "./target/release/uv pip sync ${TARGET} --no-cache" \ - --prepare "rm -rf /tmp/site-packages" \ - "pip install -r ${TARGET} --target /tmp/site-packages --no-cache-dir --no-deps" - -### -# Installation with a warm cache, similar to blowing away and re-creating a virtual environment. -### -hyperfine --runs 20 --warmup 3 \ - --prepare "virtualenv --clear .venv" \ - "./target/release/uv pip sync ${TARGET}" \ - --prepare "rm -rf /tmp/site-packages" \ - "pip install -r ${TARGET} --target /tmp/site-packages --no-deps" - -### -# Installation with all dependencies already installed (no-op). -### -hyperfine --runs 20 --warmup 3 \ - --setup "virtualenv --clear .venv && source .venv/bin/activate" \ - "./target/release/uv pip sync ${TARGET}" \ - "pip install -r ${TARGET} --no-deps" diff --git a/scripts/benchmarks/uninstall.sh b/scripts/benchmarks/uninstall.sh deleted file mode 100755 index 906ac362c..000000000 --- a/scripts/benchmarks/uninstall.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env sh - -### -# Benchmark the uninstall command against `pip`. -# -# Example usage: -# -# ./scripts/benchmarks/uninstall.sh numpy -### - -set -euxo pipefail - -TARGET=${1} - -hyperfine --runs 20 --warmup 3 --prepare "rm -rf .venv && virtualenv .venv && source activate .venv/bin/activate && pip install ${TARGET}" \ - "./target/release/uv uninstall ${TARGET}" \ - "pip uninstall -y ${TARGET}" diff --git a/scripts/benchmarks/venv.sh b/scripts/benchmarks/venv.sh deleted file mode 100755 index 72f299e84..000000000 --- a/scripts/benchmarks/venv.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -### -# Benchmark the virtualenv initialization against `virtualenv`. -# -# Example usage: -# -# ./scripts/benchmarks/venv.sh -### - -set -euxo pipefail - -### -# Create a virtual environment without seed packages. -### -hyperfine --runs 20 --warmup 3 \ - --prepare "rm -rf .venv" \ - "./target/release/uv venv" \ - --prepare "rm -rf .venv" \ - "virtualenv --without-pip .venv" \ - --prepare "rm -rf .venv" \ - "python -m venv --without-pip .venv" - -### -# Create a virtual environment with seed packages. -### -hyperfine --runs 20 --warmup 3 \ - --prepare "rm -rf .venv" \ - "./target/release/uv venv --seed" \ - --prepare "rm -rf .venv" \ - "virtualenv .venv" \ - --prepare "rm -rf .venv" \ - "python -m venv .venv"