From ddb82a01c816cf673d1c39e9dc94427d556c8b99 Mon Sep 17 00:00:00 2001 From: Ibraheem Ahmed Date: Fri, 9 Aug 2024 12:52:28 -0400 Subject: [PATCH] Add basic universal benchmarks to CI (#5938) ## Summary Resolves https://github.com/astral-sh/uv/issues/4921. --- .github/workflows/ci.yml | 3 +- Cargo.lock | 1 + crates/bench/Cargo.toml | 1 + crates/bench/benches/uv.rs | 122 ++++++++++++++++++++++--------------- 4 files changed, 76 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8f9a71c8..87d83c008 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1494,7 +1494,8 @@ jobs: sudo apt-get update sudo apt-get install -y libsasl2-dev libldap2-dev libkrb5-dev cargo run --bin uv -- venv --cache-dir .cache - cargo run --bin uv -- pip compile scripts/requirements/airflow.in --exclude-newer 2024-06-20 --cache-dir .cache + cargo run --bin uv -- pip compile scripts/requirements/jupyter.in --universal --exclude-newer 2024-08-08 --cache-dir .cache + cargo run --bin uv -- pip compile scripts/requirements/airflow.in --universal --exclude-newer 2024-08-08 --cache-dir .cache - name: "Build benchmarks" run: cargo codspeed build --features codspeed -p bench diff --git a/Cargo.lock b/Cargo.lock index 569fcb18c..d84dfa972 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,6 +393,7 @@ dependencies = [ "distribution-filename", "distribution-types", "install-wheel-rs", + "pep440_rs", "pep508_rs", "platform-tags", "pypi-types", diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml index 9f6cb68fe..f2eb325d5 100644 --- a/crates/bench/Cargo.toml +++ b/crates/bench/Cargo.toml @@ -31,6 +31,7 @@ harness = false distribution-filename = { workspace = true } distribution-types = { workspace = true } install-wheel-rs = { workspace = true } +pep440_rs = { workspace = true } pep508_rs = { workspace = true } platform-tags = { workspace = true } pypi-types = { workspace = true } diff --git a/crates/bench/benches/uv.rs b/crates/bench/benches/uv.rs index 20971e200..be6078548 100644 --- a/crates/bench/benches/uv.rs +++ b/crates/bench/benches/uv.rs @@ -9,67 +9,75 @@ use uv_python::PythonEnvironment; use uv_resolver::Manifest; fn resolve_warm_jupyter(c: &mut Criterion) { - let runtime = &tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - let cache = &Cache::from_path("../../.cache").init().unwrap(); - let venv = PythonEnvironment::from_root("../../.venv", cache).unwrap(); - let client = &RegistryClientBuilder::new(cache.clone()).build(); - let manifest = &Manifest::simple(vec![Requirement::from( + let run = setup(Manifest::simple(vec![Requirement::from( pep508_rs::Requirement::from_str("jupyter==1.0.0").unwrap(), - )]); + )])); + c.bench_function("resolve_warm_jupyter", |b| b.iter(|| run(false))); +} - let run = || { - runtime - .block_on(resolver::resolve( - black_box(manifest.clone()), - black_box(cache.clone()), - black_box(client), - &venv, - )) - .unwrap(); - }; - - c.bench_function("resolve_warm_jupyter", |b| b.iter(run)); +fn resolve_warm_jupyter_universal(c: &mut Criterion) { + let run = setup(Manifest::simple(vec![Requirement::from( + pep508_rs::Requirement::from_str("jupyter==1.0.0").unwrap(), + )])); + c.bench_function("resolve_warm_jupyter_universal", |b| b.iter(|| run(true))); } fn resolve_warm_airflow(c: &mut Criterion) { - let runtime = &tokio::runtime::Builder::new_current_thread() + let run = setup(Manifest::simple(vec![ + Requirement::from(pep508_rs::Requirement::from_str("apache-airflow[all]==2.9.3").unwrap()), + Requirement::from( + pep508_rs::Requirement::from_str("apache-airflow-providers-apache-beam>3.0.0").unwrap(), + ), + ])); + c.bench_function("resolve_warm_airflow", |b| b.iter(|| run(false))); +} + +// This takes >5m to run in CodSpeed. +// fn resolve_warm_airflow_universal(c: &mut Criterion) { +// let run = setup(Manifest::simple(vec![ +// Requirement::from(pep508_rs::Requirement::from_str("apache-airflow[all]").unwrap()), +// Requirement::from( +// pep508_rs::Requirement::from_str("apache-airflow-providers-apache-beam>3.0.0").unwrap(), +// ), +// ])); +// c.bench_function("resolve_warm_airflow_universal", |b| b.iter(|| run(true))); +// } + +criterion_group!( + uv, + resolve_warm_jupyter, + resolve_warm_jupyter_universal, + resolve_warm_airflow +); +criterion_main!(uv); + +fn setup(manifest: Manifest) -> impl Fn(bool) { + let runtime = tokio::runtime::Builder::new_current_thread() // CodSpeed limits the total number of threads to 500 .max_blocking_threads(256) .enable_all() .build() .unwrap(); - let cache = &Cache::from_path("../../.cache").init().unwrap(); - let venv = PythonEnvironment::from_root("../../.venv", cache).unwrap(); - let client = &RegistryClientBuilder::new(cache.clone()).build(); - let manifest = &Manifest::simple(vec![ - Requirement::from(pep508_rs::Requirement::from_str("apache-airflow[all]==2.9.2").unwrap()), - Requirement::from( - pep508_rs::Requirement::from_str("apache-airflow-providers-apache-beam>3.0.0").unwrap(), - ), - ]); + let cache = Cache::from_path("../../.cache").init().unwrap(); + let interpreter = PythonEnvironment::from_root("../../.venv", &cache) + .unwrap() + .into_interpreter(); + let client = RegistryClientBuilder::new(cache.clone()).build(); - let run = || { + move |universal| { runtime .block_on(resolver::resolve( black_box(manifest.clone()), black_box(cache.clone()), - black_box(client), - &venv, + black_box(&client), + &interpreter, + universal, )) .unwrap(); - }; - - c.bench_function("resolve_warm_airflow", |b| b.iter(run)); + } } -criterion_group!(uv, resolve_warm_airflow, resolve_warm_jupyter); -criterion_main!(uv); - mod resolver { use std::sync::LazyLock; @@ -78,6 +86,7 @@ mod resolver { use distribution_types::IndexLocations; use install_wheel_rs::linker::LinkMode; + use pep440_rs::Version; use pep508_rs::{MarkerEnvironment, MarkerEnvironmentBuilder}; use platform_tags::{Arch, Os, Platform, Tags}; use uv_cache::Cache; @@ -89,10 +98,10 @@ mod resolver { use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; use uv_git::GitResolver; - use uv_python::PythonEnvironment; + use uv_python::Interpreter; use uv_resolver::{ - FlatIndex, InMemoryIndex, Manifest, OptionsBuilder, PythonRequirement, ResolutionGraph, - Resolver, ResolverMarkers, + FlatIndex, InMemoryIndex, Manifest, OptionsBuilder, PythonRequirement, RequiresPython, + ResolutionGraph, Resolver, ResolverMarkers, }; use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight}; @@ -127,14 +136,15 @@ mod resolver { manifest: Manifest, cache: Cache, client: &RegistryClient, - venv: &PythonEnvironment, + interpreter: &Interpreter, + universal: bool, ) -> Result { let build_isolation = BuildIsolation::Isolated; let build_options = BuildOptions::default(); let concurrency = Concurrency::default(); let config_settings = ConfigSettings::default(); let exclude_newer = Some( - NaiveDate::from_ymd_opt(2024, 6, 20) + NaiveDate::from_ymd_opt(2024, 8, 8) .unwrap() .and_hms_opt(0, 0, 0) .unwrap() @@ -148,13 +158,19 @@ mod resolver { let index = InMemoryIndex::default(); let index_locations = IndexLocations::default(); let installed_packages = EmptyInstalledPackages; - let interpreter = venv.interpreter(); - let python_requirement = PythonRequirement::from_interpreter(interpreter); let sources = SourceStrategy::default(); - let options = OptionsBuilder::new().exclude_newer(exclude_newer).build(); let build_constraints = []; + let python_requirement = if universal { + PythonRequirement::from_requires_python( + interpreter, + &RequiresPython::greater_than_equal_version(&Version::new([3, 11])), + ) + } else { + PythonRequirement::from_interpreter(interpreter) + }; + let build_context = BuildDispatch::new( client, &cache, @@ -177,11 +193,17 @@ mod resolver { PreviewMode::Disabled, ); + let markers = if universal { + ResolverMarkers::universal(None) + } else { + ResolverMarkers::specific_environment(MARKERS.clone()) + }; + let resolver = Resolver::new( manifest, options, &python_requirement, - ResolverMarkers::SpecificEnvironment(MARKERS.clone()), + markers, Some(&TAGS), &flat_index, &index,