Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use Python 3.12 in benchmarks (#1215) 

I originally used Python 3.10, since 3.10 and 3.11 are by far the most
common (at least for [Ruff](https://pypistats.org/packages/ruff)). But
3.12 should give Python tools the most favorable benchmarks.
This commit is contained in:
Charlie Marsh 2024-01-31 12:51:13 -08:00 committed by GitHub
parent ee69fb51ea
commit 51e8609ee8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
BENCHMARKS.md
View File

@ -1,6 +1,7 @@
# Benchmarks # Benchmarks
All benchmarks were computed on macOS, and come with a few important caveats: All benchmarks were computed on macOS using Python 3.12 (for non-Puffin tools), and come with a few
important caveats:
- Benchmark performance may vary dramatically across different operating systems and filesystems. - Benchmark performance may vary dramatically across different operating systems and filesystems.
In particular, Puffin uses different installation strategies based on the underlying filesystem's In particular, Puffin uses different installation strategies based on the underlying filesystem's
@ -25,7 +26,7 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a warm cache. T
to removing and recreating a virtual environment, and then populating it with dependencies that to removing and recreating a virtual environment, and then populating it with dependencies that
you've installed previously on the same machine. you've installed previously on the same machine.
![](https://github.com/astral-sh/ruff/assets/1309177/6ceea7aa-4813-4ea8-8c95-b8013d702cf4) ![install-warm](https://github.com/astral-sh/ruff/assets/1309177/b6cb8d48-52e0-45c2-ae15-0a3f69ec3263)
## Cold Installation ## Cold Installation
@ -33,7 +34,7 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a cold cache. T
to running `puffin pip sync` on a new machine or in CI (assuming that the package manager cache is to running `puffin pip sync` on a new machine or in CI (assuming that the package manager cache is
not shared across runs). not shared across runs).
![](https://github.com/astral-sh/ruff/assets/1309177/c960d6fd-ec34-467e-9aa2-d4e6713abed0) ![install-cold](https://github.com/astral-sh/ruff/assets/1309177/ed86c193-582f-4163-b369-f12ec3905c3c)
## Warm Resolution ## Warm Resolution
@ -41,7 +42,7 @@ Benchmarking dependency resolution (e.g., `puffin pip compile`) with a warm cach
lockfile. This is equivalent to blowing away an existing `requirements.txt` file to regenerate it lockfile. This is equivalent to blowing away an existing `requirements.txt` file to regenerate it
from a `requirements.in` file. from a `requirements.in` file.
![](https://github.com/astral-sh/ruff/assets/1309177/aab99181-e54e-4bdb-9ce6-15b018ef8466) ![resolve-warm](https://github.com/astral-sh/ruff/assets/1309177/a4ca9d23-1148-4103-abe7-a35fa488409d)
## Cold Resolution ## Cold Resolution
@ -49,7 +50,7 @@ Benchmarking dependency resolution (e.g., `puffin pip compile`) with a cold cach
equivalent to running `puffin pip compile` on a new machine or in CI (assuming that the package equivalent to running `puffin pip compile` on a new machine or in CI (assuming that the package
manager cache is not shared across runs). manager cache is not shared across runs).
![](https://github.com/astral-sh/ruff/assets/1309177/a6075ebc-bb8f-46db-a3b4-14ee5f713565) ![resolve-cold](https://github.com/astral-sh/ruff/assets/1309177/556ac7aa-0a6a-4f94-b0d9-90b25461de7b)
## Reproduction ## Reproduction

6
crates/puffin-dev/src/render_benchmarks.rs
View File

@ -23,13 +23,15 @@ pub(crate) fn render_benchmarks(args: &RenderBenchmarksArgs) -> Result<()> {
// but we assume we're running over a single benchmark here.) // but we assume we're running over a single benchmark here.)
for result in &mut results.results { for result in &mut results.results {
if result.command.starts_with("puffin") { if result.command.starts_with("puffin") {
result.command = "puffin".into(); result.command = "Puffin".into();
} else if result.command.starts_with("pip-compile") { } else if result.command.starts_with("pip-compile") {
result.command = "pip-compile".into(); result.command = "pip-compile".into();
} else if result.command.starts_with("pip-sync") { } else if result.command.starts_with("pip-sync") {
result.command = "pip-sync".into(); result.command = "pip-sync".into();
} else if result.command.starts_with("poetry") { } else if result.command.starts_with("poetry") {
result.command = "poetry".into(); result.command = "Poetry".into();
} else if result.command.starts_with("pdm") {
result.command = "PDM".into();
} else { } else {
return Err(anyhow!("unknown command: {}", result.command)); return Err(anyhow!("unknown command: {}", result.command));
} }

30
scripts/bench/__main__.py
View File

@ -4,7 +4,7 @@ This script assumes that `pip`, `pip-tools`, `virtualenv`, `poetry` and `hyperfi
installed, and that a Puffin release builds exists at `./target/release/puffin` installed, and that a Puffin release builds exists at `./target/release/puffin`
(relative to the repository root). (relative to the repository root).
This script assumes that Python 3.10 is installed. This script assumes that Python 3.12 is installed.
To set up the required environment, run: To set up the required environment, run:
@ -300,7 +300,7 @@ class PipSync(Suite):
return Command( return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}", prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
command=[ command=[
self.path, self.path,
os.path.abspath(requirements_file), os.path.abspath(requirements_file),
@ -317,7 +317,7 @@ class PipSync(Suite):
return Command( return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.10 {venv_dir}", prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
command=[ command=[
self.path, self.path,
os.path.abspath(requirements_file), os.path.abspath(requirements_file),
@ -357,9 +357,11 @@ class Poetry(Suite):
"bench", "bench",
"--no-interaction", "--no-interaction",
"--python", "--python",
">=3.10", "3.12",
], ],
cwd=cwd, cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
) )
# Parse the pyproject.toml. # Parse the pyproject.toml.
@ -518,7 +520,7 @@ class Poetry(Suite):
f"rm -rf {config_dir} && " f"rm -rf {config_dir} && "
f"rm -rf {cache_dir} && " f"rm -rf {cache_dir} && "
f"rm -rf {data_dir} &&" f"rm -rf {data_dir} &&"
f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
), ),
command=[ command=[
f"POETRY_CONFIG_DIR={config_dir}", f"POETRY_CONFIG_DIR={config_dir}",
@ -558,7 +560,7 @@ class Poetry(Suite):
return Command( return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.10 {venv_dir}", prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
command=[ command=[
f"POETRY_CONFIG_DIR={config_dir}", f"POETRY_CONFIG_DIR={config_dir}",
f"POETRY_CACHE_DIR={cache_dir}", f"POETRY_CACHE_DIR={cache_dir}",
@ -595,8 +597,10 @@ class Pdm(Suite):
# Create a PDM project. # Create a PDM project.
subprocess.check_call( subprocess.check_call(
[self.path, "init", "--non-interactive", "--python", "3.10"], [self.path, "init", "--non-interactive", "--python", "3.12"],
cwd=cwd, cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
) )
# Parse the pyproject.toml. # Parse the pyproject.toml.
@ -635,7 +639,7 @@ class Pdm(Suite):
cache_dir = os.path.join(cwd, "cache", "pdm") cache_dir = os.path.join(cwd, "cache", "pdm")
return Command( return Command(
name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})", name=f"{self.name} ({Benchmark.RESOLVE_WARM.value})",
prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}", prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}",
command=[ command=[
self.path, self.path,
@ -719,7 +723,7 @@ class Pdm(Suite):
prepare=( prepare=(
f"rm -rf {cache_dir} && " f"rm -rf {cache_dir} && "
f"{self.path} config cache_dir {cache_dir} && " f"{self.path} config cache_dir {cache_dir} && "
f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
), ),
command=[ command=[
f"VIRTUAL_ENV={venv_dir}", f"VIRTUAL_ENV={venv_dir}",
@ -750,10 +754,10 @@ class Pdm(Suite):
cache_dir = os.path.join(cwd, "cache", "pdm") cache_dir = os.path.join(cwd, "cache", "pdm")
return Command( return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=( prepare=(
f"{self.path} config cache_dir {cache_dir} && " f"{self.path} config cache_dir {cache_dir} && "
f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
), ),
command=[ command=[
f"VIRTUAL_ENV={venv_dir}", f"VIRTUAL_ENV={venv_dir}",
@ -871,7 +875,7 @@ class Puffin(Suite):
return Command( return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}", prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
command=[ command=[
f"VIRTUAL_ENV={venv_dir}", f"VIRTUAL_ENV={venv_dir}",
self.path, self.path,
@ -889,7 +893,7 @@ class Puffin(Suite):
return Command( return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.10 {venv_dir}", prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
command=[ command=[
f"VIRTUAL_ENV={venv_dir}", f"VIRTUAL_ENV={venv_dir}",
self.path, self.path,