diff --git a/crates/uv-requirements/src/source_tree.rs b/crates/uv-requirements/src/source_tree.rs index c88411949..f26dba2f1 100644 --- a/crates/uv-requirements/src/source_tree.rs +++ b/crates/uv-requirements/src/source_tree.rs @@ -1,10 +1,13 @@ use std::borrow::Cow; +use std::collections::VecDeque; use std::path::Path; +use std::slice; use std::sync::Arc; use anyhow::{Context, Result}; use futures::stream::FuturesOrdered; use futures::TryStreamExt; +use rustc_hash::FxHashSet; use url::Url; use uv_configuration::ExtrasSpecification; @@ -14,7 +17,7 @@ use uv_distribution_types::{ }; use uv_fs::Simplified; use uv_normalize::{ExtraName, PackageName}; -use uv_pep508::RequirementOrigin; +use uv_pep508::{MarkerTree, RequirementOrigin}; use uv_pypi_types::Requirement; use uv_resolver::{InMemoryIndex, MetadataResponse}; use uv_types::{BuildContext, HashStrategy}; @@ -89,16 +92,13 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> { let origin = RequirementOrigin::Project(path.to_path_buf(), metadata.name.clone()); // Determine the extras to include when resolving the requirements. - let extras: Vec<_> = self + let extras = self .extras .extra_names(metadata.provides_extras.iter()) .cloned() - .collect(); + .collect::>(); - // Determine the appropriate requirements to return based on the extras. This involves - // evaluating the `extras` expression in any markers, but preserving the remaining marker - // conditions. - let mut requirements: Vec = metadata + let dependencies = metadata .requires_dist .into_iter() .map(|requirement| Requirement { @@ -106,30 +106,61 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> { marker: requirement.marker.simplify_extras(&extras), ..requirement }) + .collect::>(); + + // Transitively process all extras that are recursively included, starting with the current + // extra. + let mut requirements = dependencies.clone(); + let mut seen = FxHashSet::<(ExtraName, MarkerTree)>::default(); + let mut queue: VecDeque<_> = requirements + .iter() + .filter(|req| req.name == metadata.name) + .flat_map(|req| { + req.extras + .iter() + .cloned() + .map(|extra| (extra, req.marker.clone().simplify_extras(&extras))) + }) .collect(); + while let Some((extra, marker)) = queue.pop_front() { + if !seen.insert((extra.clone(), marker.clone())) { + continue; + } - // Resolve any recursive extras. - loop { - // Find the first recursive requirement. - // TODO(charlie): Respect markers on recursive extras. - let Some(index) = requirements.iter().position(|requirement| { - requirement.name == metadata.name && requirement.marker.is_true() - }) else { - break; - }; - - // Remove the requirement that points to us. - let recursive = requirements.remove(index); - - // Re-simplify the requirements. - for requirement in &mut requirements { - requirement.marker = requirement - .marker - .clone() - .simplify_extras(&recursive.extras); + // Find the requirements for the extra. + for requirement in &dependencies { + if requirement.marker.top_level_extra_name().as_ref() == Some(&extra) { + let requirement = { + let mut marker = marker.clone(); + marker.and(requirement.marker.clone()); + Requirement { + name: requirement.name.clone(), + extras: requirement.extras.clone(), + source: requirement.source.clone(), + origin: requirement.origin.clone(), + marker: marker.simplify_extras(slice::from_ref(&extra)), + } + }; + if requirement.name == metadata.name { + // Add each transitively included extra. + queue.extend( + requirement + .extras + .iter() + .cloned() + .map(|extra| (extra, requirement.marker.clone())), + ); + } else { + // Add the requirements for that extra. + requirements.push(requirement); + } + } } } + // Drop all the self-requirements now that we flattened them out. + requirements.retain(|req| req.name != metadata.name); + let project = metadata.name; let extras = metadata.provides_extras; diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index d8c312530..7b48689dd 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -7,7 +7,7 @@ use std::fmt::{Display, Formatter, Write}; use std::ops::Bound; use std::sync::Arc; use std::time::Instant; -use std::{iter, thread}; +use std::{iter, slice, thread}; use dashmap::DashMap; use either::Either; @@ -1262,11 +1262,9 @@ impl ResolverState { let no_dev_deps = BTreeMap::default(); - let no_provides_extras = []; let requirements = self.flatten_requirements( &self.requirements, &no_dev_deps, - &no_provides_extras, None, None, None, @@ -1454,7 +1452,6 @@ impl ResolverState ResolverState>, - extras: &'a [ExtraName], extra: Option<&'a ExtraName>, dev: Option<&'a GroupName>, name: Option<&PackageName>, @@ -1622,7 +1618,7 @@ impl ResolverState ResolverState { - requirement.marker.and(marker.clone()); - Cow::Owned(requirement) - } - Cow::Borrowed(requirement) => { - let mut marker = marker.clone(); - marker.and(requirement.marker.clone()); - Cow::Owned(Requirement { - name: requirement.name.clone(), - extras: requirement.extras.clone(), - source: requirement.source.clone(), - origin: requirement.origin.clone(), - marker, - }) + let requirement = match requirement { + Cow::Owned(mut requirement) => { + requirement.marker.and(marker.clone()); + requirement + } + Cow::Borrowed(requirement) => { + let mut marker = marker.clone(); + marker.and(requirement.marker.clone()); + Requirement { + name: requirement.name.clone(), + extras: requirement.extras.clone(), + source: requirement.source.clone(), + origin: requirement.origin.clone(), + marker: marker.simplify_extras(slice::from_ref(&extra)), } } }; if name == Some(&requirement.name) { // Add each transitively included extra. queue.extend( - requirement.extras.iter().cloned().map(|extra| { - (extra, requirement.marker.clone().simplify_extras(extras)) - }), + requirement + .extras + .iter() + .cloned() + .map(|extra| (extra, requirement.marker.clone())), ); } else { // Add the requirements for that extra. - requirements.push(requirement); + requirements.push(Cow::Owned(requirement)); } } } diff --git a/crates/uv/tests/it/pip_compile.rs b/crates/uv/tests/it/pip_compile.rs index a9747974a..39af23050 100644 --- a/crates/uv/tests/it/pip_compile.rs +++ b/crates/uv/tests/it/pip_compile.rs @@ -9870,6 +9870,136 @@ dev = [ Ok(()) } +/// Resolve from a `pyproject.toml` file with a recursive extra, with a marker attached. +#[test] +fn compile_pyproject_toml_recursive_extra_marker() -> Result<()> { + let context = TestContext::new("3.12"); + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" +[project] +name = "project" +version = "0.0.1" +dependencies = [ + "anyio" +] + +[project.optional-dependencies] +test = [ + "iniconfig", +] +dev = [ + "project[test] ; sys_platform == 'darwin'", +] +"#, + )?; + + uv_snapshot!(context.filters(), context.pip_compile() + .arg("pyproject.toml") + .arg("--extra") + .arg("dev") + .arg("--universal"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --extra dev --universal + anyio==4.3.0 + # via project (pyproject.toml) + idna==3.6 + # via anyio + iniconfig==2.0.0 ; sys_platform == 'darwin' + # via project (pyproject.toml) + sniffio==1.3.1 + # via anyio + + ----- stderr ----- + Resolved 4 packages in [TIME] + "### + ); + + Ok(()) +} + +/// Resolve from a `pyproject.toml` file with multiple recursive extras. +#[test] +fn compile_pyproject_toml_deeply_recursive_extra() -> Result<()> { + let context = TestContext::new("3.12"); + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" +[project] +name = "project" +version = "0.0.1" +dependencies = [] + +[project.optional-dependencies] +foo = ["iniconfig"] +bar = ["project[foo]"] +baz = ["project[bar]"] +bop = ["project[bar] ; sys_platform == 'darwin'"] +qux = ["project[bop] ; python_version == '3.12'"] +"#, + )?; + + uv_snapshot!(context.filters(), context.pip_compile() + .arg("pyproject.toml") + .arg("--universal") + .arg("--extra") + .arg("qux"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --extra qux + iniconfig==2.0.0 ; python_full_version < '3.13' and sys_platform == 'darwin' + # via project (pyproject.toml) + + ----- stderr ----- + Resolved 1 package in [TIME] + "### + ); + + uv_snapshot!(context.filters(), context.pip_compile() + .arg("pyproject.toml") + .arg("--universal") + .arg("--extra") + .arg("bop") + .arg("--extra") + .arg("bar"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --extra bop --extra bar + iniconfig==2.0.0 + # via project (pyproject.toml) + + ----- stderr ----- + Resolved 1 package in [TIME] + "### + ); + + uv_snapshot!(context.filters(), context.pip_compile() + .arg("pyproject.toml") + .arg("--universal") + .arg("--all-extras"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --all-extras + iniconfig==2.0.0 + # via project (pyproject.toml) + + ----- stderr ----- + Resolved 1 package in [TIME] + "### + ); + + Ok(()) +} + /// The dependencies of a local editable dependency should be considered "direct" dependencies. #[test] fn editable_direct_dependency() -> Result<()> {