diff --git a/Cargo.lock b/Cargo.lock index d1fcb1700..971beeae8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4703,14 +4703,11 @@ dependencies = [ "distribution-types", "fs-err", "futures", - "indicatif", "install-wheel-rs", "itertools 0.12.1", "mimalloc", "owo-colors", - "pep440_rs", "pep508_rs", - "petgraph", "poloto", "pretty_assertions", "resvg", @@ -4730,11 +4727,9 @@ dependencies = [ "uv-client", "uv-configuration", "uv-dispatch", - "uv-distribution", "uv-fs", "uv-installer", "uv-interpreter", - "uv-normalize", "uv-requirements", "uv-resolver", "uv-types", diff --git a/crates/uv-dev/Cargo.toml b/crates/uv-dev/Cargo.toml index f69f07a59..9ae242f71 100644 --- a/crates/uv-dev/Cargo.toml +++ b/crates/uv-dev/Cargo.toml @@ -19,18 +19,15 @@ workspace = true distribution-filename = { workspace = true } distribution-types = { workspace = true } install-wheel-rs = { workspace = true } -pep440_rs = { workspace = true } pep508_rs = { workspace = true } uv-build = { workspace = true } uv-cache = { workspace = true, features = ["clap"] } uv-client = { workspace = true } uv-configuration = { workspace = true } uv-dispatch = { workspace = true } -uv-distribution = { workspace = true } uv-fs = { workspace = true } uv-installer = { workspace = true } uv-interpreter = { workspace = true } -uv-normalize = { workspace = true } uv-requirements = { workspace = true, features = ["schemars"] } uv-resolver = { workspace = true } uv-types = { workspace = true } @@ -43,10 +40,8 @@ anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "wrap_help"] } fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } -indicatif = { workspace = true } itertools = { workspace = true } owo-colors = { workspace = true } -petgraph = { workspace = true } poloto = { version = "19.1.2" } pretty_assertions = { version = "1.4.0" } resvg = { version = "0.29.0" } diff --git a/crates/uv-dev/src/main.rs b/crates/uv-dev/src/main.rs index 02a3e5376..77806498b 100644 --- a/crates/uv-dev/src/main.rs +++ b/crates/uv-dev/src/main.rs @@ -16,15 +16,12 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; -use resolve_many::ResolveManyArgs; - use crate::build::{build, BuildArgs}; use crate::clear_compile::ClearCompileArgs; use crate::compile::CompileArgs; use crate::fetch_python::FetchPythonArgs; use crate::generate_json_schema::GenerateJsonSchemaArgs; use crate::render_benchmarks::RenderBenchmarksArgs; -use crate::resolve_cli::ResolveCliArgs; use crate::wheel_metadata::WheelMetadataArgs; #[cfg(target_os = "windows")] @@ -49,36 +46,23 @@ mod compile; mod fetch_python; mod generate_json_schema; mod render_benchmarks; -mod resolve_cli; -mod resolve_many; mod wheel_metadata; const ROOT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../"); #[derive(Parser)] enum Cli { - /// Build a source distribution into a wheel + /// Build a source distribution into a wheel. Build(BuildArgs), - /// Resolve many requirements independently in parallel and report failures and successes. - /// - /// Run `scripts/popular_packages/pypi_8k_downloads.sh` once, then - /// ```bash - /// cargo run --bin uv-dev -- resolve-many scripts/popular_packages/pypi_8k_downloads.txt - /// ``` - /// or - /// ```bash - /// cargo run --bin uv-dev -- resolve-many scripts/popular_packages/pypi_10k_most_dependents.txt - /// ``` - ResolveMany(ResolveManyArgs), - /// Resolve requirements passed on the CLI - Resolve(ResolveCliArgs), + /// Display the metadata for a `.whl` at a given URL. WheelMetadata(WheelMetadataArgs), + /// Render the benchmarks. RenderBenchmarks(RenderBenchmarksArgs), /// Compile all `.py` to `.pyc` files in the tree. Compile(CompileArgs), /// Remove all `.pyc` in the tree. ClearCompile(ClearCompileArgs), - /// Fetch Python versions for testing + /// Fetch Python versions for testing. FetchPython(FetchPythonArgs), /// Generate JSON schema for the TOML configuration file. GenerateJSONSchema(GenerateJsonSchemaArgs), @@ -92,12 +76,6 @@ async fn run() -> Result<()> { let target = build(args).await?; println!("Wheel built to {}", target.display()); } - Cli::ResolveMany(args) => { - resolve_many::resolve_many(args).await?; - } - Cli::Resolve(args) => { - resolve_cli::resolve_cli(args).await?; - } Cli::WheelMetadata(args) => wheel_metadata::wheel_metadata(args).await?, Cli::RenderBenchmarks(args) => render_benchmarks::render_benchmarks(&args)?, Cli::Compile(args) => compile::compile(args).await?, diff --git a/crates/uv-dev/src/resolve_cli.rs b/crates/uv-dev/src/resolve_cli.rs deleted file mode 100644 index 8c0e3d351..000000000 --- a/crates/uv-dev/src/resolve_cli.rs +++ /dev/null @@ -1,162 +0,0 @@ -use std::io::{BufWriter, Write}; -use std::path::PathBuf; - -use anstream::println; -use anyhow::{Context, Result}; -use clap::{Parser, ValueEnum}; -use fs_err::File; -use itertools::Itertools; -use petgraph::dot::{Config as DotConfig, Dot}; - -use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Requirement, Resolution}; -use uv_cache::{Cache, CacheArgs}; -use uv_client::{FlatIndexClient, RegistryClientBuilder}; -use uv_configuration::{Concurrency, ConfigSettings, NoBinary, NoBuild, SetupPyStrategy}; -use uv_dispatch::BuildDispatch; -use uv_distribution::DistributionDatabase; -use uv_installer::SitePackages; -use uv_interpreter::PythonEnvironment; -use uv_resolver::{ - ExcludeNewer, FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, Resolver, -}; -use uv_types::{BuildIsolation, HashStrategy, InFlight}; - -#[derive(ValueEnum, Default, Clone)] -pub(crate) enum ResolveCliFormat { - #[default] - Compact, - Expanded, -} - -#[derive(Parser)] -pub(crate) struct ResolveCliArgs { - requirements: Vec, - /// Write debug output in DOT format for graphviz to this file - #[clap(long)] - graphviz: Option, - /// Don't build source distributions. This means resolving will not run arbitrary code. The - /// cached wheels of already built source distributions will be reused. - #[clap(long)] - no_build: bool, - #[clap(long, default_value = "compact")] - format: ResolveCliFormat, - #[command(flatten)] - cache_args: CacheArgs, - #[arg(long)] - exclude_newer: Option, - #[clap(long, short, env = "UV_INDEX_URL")] - index_url: Option, - #[clap(long, env = "UV_EXTRA_INDEX_URL")] - extra_index_url: Vec, - #[clap(long)] - find_links: Vec, -} - -pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { - let cache = Cache::try_from(args.cache_args)?; - - let venv = PythonEnvironment::from_virtualenv(&cache)?; - let index_locations = - IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false); - let index = InMemoryIndex::default(); - let in_flight = InFlight::default(); - let no_build = if args.no_build { - NoBuild::All - } else { - NoBuild::None - }; - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_locations.index_urls()) - .build(); - let flat_index = { - let client = FlatIndexClient::new(&client, &cache); - let entries = client.fetch(index_locations.flat_index()).await?; - FlatIndex::from_entries( - entries, - venv.interpreter().tags()?, - &HashStrategy::None, - &no_build, - &NoBinary::None, - ) - }; - let config_settings = ConfigSettings::default(); - let concurrency = Concurrency::default(); - - let build_dispatch = BuildDispatch::new( - &client, - &cache, - venv.interpreter(), - &index_locations, - &flat_index, - &index, - &in_flight, - SetupPyStrategy::default(), - &config_settings, - BuildIsolation::Isolated, - install_wheel_rs::linker::LinkMode::default(), - &no_build, - &NoBinary::None, - concurrency, - ); - - let site_packages = SitePackages::from_executable(&venv)?; - - // Copied from `BuildDispatch` - let tags = venv.interpreter().tags()?; - let markers = venv.interpreter().markers(); - let python_requirement = - PythonRequirement::from_marker_environment(venv.interpreter(), markers); - let resolver = Resolver::new( - Manifest::simple( - args.requirements - .iter() - .cloned() - .map(Requirement::from_pep508) - .collect::>()?, - ), - Options::default(), - &python_requirement, - Some(venv.interpreter().markers()), - tags, - &flat_index, - &index, - &HashStrategy::None, - &build_dispatch, - &site_packages, - DistributionDatabase::new(&client, &build_dispatch, concurrency.downloads), - )?; - let resolution_graph = resolver.resolve().await.with_context(|| { - format!( - "No solution found when resolving: {}", - args.requirements.iter().map(ToString::to_string).join(", "), - ) - })?; - - if let Some(graphviz) = args.graphviz { - let mut writer = BufWriter::new(File::create(graphviz)?); - let graphviz = Dot::with_attr_getters( - resolution_graph.petgraph(), - &[DotConfig::NodeNoLabel, DotConfig::EdgeNoLabel], - &|_graph, edge_ref| format!("label={:?}", edge_ref.weight().to_string()), - &|_graph, (_node_index, dist)| { - format!("label={:?}", dist.to_string().replace("==", "\n")) - }, - ); - write!(&mut writer, "{graphviz:?}")?; - } - - let requirements = Resolution::from(resolution_graph).requirements(); - - match args.format { - ResolveCliFormat::Compact => { - println!("{}", requirements.iter().map(ToString::to_string).join(" ")); - } - ResolveCliFormat::Expanded => { - for package in requirements { - println!("{}", package); - } - } - } - - Ok(()) -} diff --git a/crates/uv-dev/src/resolve_many.rs b/crates/uv-dev/src/resolve_many.rs deleted file mode 100644 index 75cd47bf0..000000000 --- a/crates/uv-dev/src/resolve_many.rs +++ /dev/null @@ -1,214 +0,0 @@ -use std::path::PathBuf; -use std::str::FromStr; - -use anyhow::Result; -use clap::Parser; -use futures::StreamExt; -use indicatif::ProgressStyle; -use itertools::Itertools; -use tokio::time::Instant; -use tracing::{info, info_span, Span}; -use tracing_indicatif::span_ext::IndicatifSpanExt; - -use distribution_types::{IndexLocations, Requirement}; -use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; -use pep508_rs::VersionOrUrl; -use uv_cache::{Cache, CacheArgs}; -use uv_client::{OwnedArchive, RegistryClient, RegistryClientBuilder}; -use uv_configuration::{Concurrency, ConfigSettings, NoBinary, NoBuild, SetupPyStrategy}; -use uv_dispatch::BuildDispatch; -use uv_interpreter::PythonEnvironment; -use uv_normalize::PackageName; -use uv_resolver::{FlatIndex, InMemoryIndex}; -use uv_types::{BuildContext, BuildIsolation, InFlight}; - -#[derive(Parser)] -pub(crate) struct ResolveManyArgs { - /// Path to a file containing one requirement per line. - requirements: PathBuf, - #[clap(long)] - limit: Option, - /// Don't build source distributions. This means resolving will not run arbitrary code. The - /// cached wheels of already built source distributions will be reused. - #[clap(long)] - no_build: bool, - /// Run this many tasks in parallel. - #[clap(long, default_value = "50")] - num_tasks: usize, - /// Force the latest version when no version is given. - #[clap(long)] - latest_version: bool, - #[command(flatten)] - cache_args: CacheArgs, -} - -/// Try to find the latest version of a package, ignoring error because we report them during resolution properly -async fn find_latest_version( - client: &RegistryClient, - package_name: &PackageName, -) -> Option { - client - .simple(package_name) - .await - .ok() - .into_iter() - .flatten() - .filter_map(|(_index, raw_simple_metadata)| { - let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata); - Some(simple_metadata.into_iter().next()?.version) - }) - .max() -} - -pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> { - let cache = Cache::try_from(args.cache_args)?; - - let data = fs_err::read_to_string(&args.requirements)?; - - let tf_models_nightly = PackageName::from_str("tf-models-nightly").unwrap(); - let lines = data - .lines() - .map(pep508_rs::Requirement::from_str) - .filter_ok(|req| req.name != tf_models_nightly); - - let requirements: Vec = if let Some(limit) = args.limit { - lines.take(limit).collect::>()? - } else { - lines.collect::>()? - }; - let total = requirements.len(); - - let venv = PythonEnvironment::from_virtualenv(&cache)?; - let in_flight = InFlight::default(); - let concurrency = Concurrency::default(); - let client = RegistryClientBuilder::new(cache.clone()).build(); - - let header_span = info_span!("resolve many"); - header_span.pb_set_style(&ProgressStyle::default_bar()); - header_span.pb_set_length(total as u64); - let _header_span_enter = header_span.enter(); - - let no_build = if args.no_build { - NoBuild::All - } else { - NoBuild::None - }; - - let mut tasks = futures::stream::iter(requirements) - .map(|requirement| { - async { - // Use a separate `InMemoryIndex` for each requirement. - let index = InMemoryIndex::default(); - let index_locations = IndexLocations::default(); - let setup_py = SetupPyStrategy::default(); - let flat_index = FlatIndex::default(); - let config_settings = ConfigSettings::default(); - - // Create a `BuildDispatch` for each requirement. - let build_dispatch = BuildDispatch::new( - &client, - &cache, - venv.interpreter(), - &index_locations, - &flat_index, - &index, - &in_flight, - setup_py, - &config_settings, - BuildIsolation::Isolated, - install_wheel_rs::linker::LinkMode::default(), - &no_build, - &NoBinary::None, - concurrency, - ); - - let start = Instant::now(); - - let requirement = if args.latest_version && requirement.version_or_url.is_none() { - if let Some(version) = find_latest_version(&client, &requirement.name).await { - let equals_version = VersionOrUrl::VersionSpecifier( - VersionSpecifiers::from(VersionSpecifier::equals_version(version)), - ); - pep508_rs::Requirement { - name: requirement.name, - extras: requirement.extras, - version_or_url: Some(equals_version), - marker: None, - origin: requirement.origin, - } - } else { - requirement - } - } else { - requirement - }; - - let result = build_dispatch - .resolve(&[ - Requirement::from_pep508(requirement.clone()).expect("Invalid requirement") - ]) - .await; - (requirement.to_string(), start.elapsed(), result) - } - }) - .buffer_unordered(args.num_tasks); - - let mut success = 0usize; - let mut errors = Vec::new(); - while let Some(result) = tasks.next().await { - let (package, duration, result) = result; - match result { - Ok(_) => { - info!( - "Success ({}/{}, {} ms): {}", - success + errors.len(), - total, - duration.as_millis(), - package, - ); - success += 1; - } - Err(err) => { - let err_formatted = - if err - .source() - .and_then(|err| err.source()) - .is_some_and(|err| { - err.to_string() == "Building source distributions is disabled" - }) - { - "Building source distributions is disabled".to_string() - } else { - err.chain() - .map(|err| { - let formatted = err.to_string(); - // Cut overly long c/c++ compile output - if formatted.lines().count() > 20 { - let formatted: Vec<_> = formatted.lines().collect(); - formatted[..20].join("\n") - + "\n[...]\n" - + &formatted[formatted.len() - 20..].join("\n") - } else { - formatted - } - }) - .join("\n Caused by: ") - }; - info!( - "Error for {} ({}/{}, {} ms): {}", - package, - success + errors.len(), - total, - duration.as_millis(), - err_formatted - ); - errors.push(package); - } - } - Span::current().pb_inc(1); - } - - info!("Errors: {}", errors.join(", ")); - info!("Success: {}, Error: {}", success, errors.len()); - Ok(()) -} diff --git a/crates/uv-resolver/src/resolution/graph.rs b/crates/uv-resolver/src/resolution/graph.rs index f560ee4ad..ae0789b83 100644 --- a/crates/uv-resolver/src/resolution/graph.rs +++ b/crates/uv-resolver/src/resolution/graph.rs @@ -376,13 +376,6 @@ impl ResolutionGraph { &self.diagnostics } - /// Return the underlying graph. - pub fn petgraph( - &self, - ) -> &petgraph::graph::Graph, petgraph::Directed> { - &self.petgraph - } - /// Return the marker tree specific to this resolution. /// /// This accepts a manifest, in-memory-index and marker environment. All diff --git a/crates/uv-resolver/src/resolution/mod.rs b/crates/uv-resolver/src/resolution/mod.rs index f9d9f6432..e2553d69c 100644 --- a/crates/uv-resolver/src/resolution/mod.rs +++ b/crates/uv-resolver/src/resolution/mod.rs @@ -14,7 +14,7 @@ mod graph; /// specific distribution (e.g., a specific wheel), while the [`Metadata23`] refers to the metadata /// for the package-version pair. #[derive(Debug)] -pub struct AnnotatedDist { +pub(crate) struct AnnotatedDist { pub(crate) dist: ResolvedDist, pub(crate) extras: Vec, pub(crate) hashes: Vec,