Support recursive extras with marker in `pip compile -r pyproject.toml` (#9535)

## Summary

Closes https://github.com/astral-sh/uv/issues/9530.

crates/uv-requirements/src/source_tree.rs

@@ -1,10 +1,13 @@
 use std::borrow::Cow;
+use std::collections::VecDeque;
 use std::path::Path;
+use std::slice;
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use futures::stream::FuturesOrdered;
 use futures::TryStreamExt;
+use rustc_hash::FxHashSet;
 use url::Url;
 
 use uv_configuration::ExtrasSpecification;
@@ -14,7 +17,7 @@ use uv_distribution_types::{
 };
 use uv_fs::Simplified;
 use uv_normalize::{ExtraName, PackageName};
-use uv_pep508::RequirementOrigin;
+use uv_pep508::{MarkerTree, RequirementOrigin};
 use uv_pypi_types::Requirement;
 use uv_resolver::{InMemoryIndex, MetadataResponse};
 use uv_types::{BuildContext, HashStrategy};
@@ -89,16 +92,13 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
         let origin = RequirementOrigin::Project(path.to_path_buf(), metadata.name.clone());
 
         // Determine the extras to include when resolving the requirements.
-        let extras: Vec<_> = self
+        let extras = self
             .extras
             .extra_names(metadata.provides_extras.iter())
             .cloned()
-            .collect();
+            .collect::<Vec<_>>();
 
-        // Determine the appropriate requirements to return based on the extras. This involves
+        let dependencies = metadata
-        // evaluating the `extras` expression in any markers, but preserving the remaining marker
-        // conditions.
-        let mut requirements: Vec<Requirement> = metadata
             .requires_dist
             .into_iter()
             .map(|requirement| Requirement {
@@ -106,29 +106,60 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
                 marker: requirement.marker.simplify_extras(&extras),
                 ..requirement
             })
+            .collect::<Vec<_>>();
+
+        // Transitively process all extras that are recursively included, starting with the current
+        // extra.
+        let mut requirements = dependencies.clone();
+        let mut seen = FxHashSet::<(ExtraName, MarkerTree)>::default();
+        let mut queue: VecDeque<_> = requirements
+            .iter()
+            .filter(|req| req.name == metadata.name)
+            .flat_map(|req| {
+                req.extras
+                    .iter()
+                    .cloned()
+                    .map(|extra| (extra, req.marker.clone().simplify_extras(&extras)))
+            })
             .collect();
+        while let Some((extra, marker)) = queue.pop_front() {
+            if !seen.insert((extra.clone(), marker.clone())) {
+                continue;
+            }
 
-        // Resolve any recursive extras.
+            // Find the requirements for the extra.
-        loop {
+            for requirement in &dependencies {
-            // Find the first recursive requirement.
+                if requirement.marker.top_level_extra_name().as_ref() == Some(&extra) {
-            // TODO(charlie): Respect markers on recursive extras.
+                    let requirement = {
-            let Some(index) = requirements.iter().position(|requirement| {
+                        let mut marker = marker.clone();
-                requirement.name == metadata.name && requirement.marker.is_true()
+                        marker.and(requirement.marker.clone());
-            }) else {
+                        Requirement {
-                break;
+                            name: requirement.name.clone(),
+                            extras: requirement.extras.clone(),
+                            source: requirement.source.clone(),
+                            origin: requirement.origin.clone(),
+                            marker: marker.simplify_extras(slice::from_ref(&extra)),
+                        }
                     };
-
+                    if requirement.name == metadata.name {
-            // Remove the requirement that points to us.
+                        // Add each transitively included extra.
-            let recursive = requirements.remove(index);
+                        queue.extend(
-
+                            requirement
-            // Re-simplify the requirements.
+                                .extras
-            for requirement in &mut requirements {
+                                .iter()
-                requirement.marker = requirement
+                                .cloned()
-                    .marker
+                                .map(|extra| (extra, requirement.marker.clone())),
-                    .clone()
+                        );
-                    .simplify_extras(&recursive.extras);
+                    } else {
+                        // Add the requirements for that extra.
+                        requirements.push(requirement);
                     }
                 }
+            }
+        }
+
+        // Drop all the self-requirements now that we flattened them out.
+        requirements.retain(|req| req.name != metadata.name);
 
         let project = metadata.name;
         let extras = metadata.provides_extras;

crates/uv-resolver/src/resolver/mod.rs

@@ -7,7 +7,7 @@ use std::fmt::{Display, Formatter, Write};
 use std::ops::Bound;
 use std::sync::Arc;
 use std::time::Instant;
-use std::{iter, thread};
+use std::{iter, slice, thread};
 
 use dashmap::DashMap;
 use either::Either;
@@ -1262,11 +1262,9 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
         let dependencies = match &**package {
             PubGrubPackageInner::Root(_) => {
                 let no_dev_deps = BTreeMap::default();
-                let no_provides_extras = [];
                 let requirements = self.flatten_requirements(
                     &self.requirements,
                     &no_dev_deps,
-                    &no_provides_extras,
                     None,
                     None,
                     None,
@@ -1454,7 +1452,6 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
                 let requirements = self.flatten_requirements(
                     &metadata.requires_dist,
                     &metadata.dependency_groups,
-                    &metadata.provides_extras,
                     extra.as_ref(),
                     dev.as_ref(),
                     Some(name),
@@ -1579,7 +1576,6 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
         &'a self,
         dependencies: &'a [Requirement],
         dev_dependencies: &'a BTreeMap<GroupName, Vec<Requirement>>,
-        extras: &'a [ExtraName],
         extra: Option<&'a ExtraName>,
         dev: Option<&'a GroupName>,
         name: Option<&PackageName>,
@@ -1622,7 +1618,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
                 req.extras
                     .iter()
                     .cloned()
-                    .map(|extra| (extra, req.marker.clone().simplify_extras(extras)))
+                    .map(|extra| (extra, req.marker.clone()))
             })
             .collect();
         while let Some((extra, marker)) = queue.pop_front() {
@@ -1632,37 +1628,35 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
             for requirement in
                 self.requirements_for_extra(dependencies, Some(&extra), env, python_requirement)
             {
-                let requirement = if marker.is_true() {
+                let requirement = match requirement {
-                    requirement
-                } else {
-                    match requirement {
                     Cow::Owned(mut requirement) => {
                         requirement.marker.and(marker.clone());
-                            Cow::Owned(requirement)
+                        requirement
                     }
                     Cow::Borrowed(requirement) => {
                         let mut marker = marker.clone();
                         marker.and(requirement.marker.clone());
-                            Cow::Owned(Requirement {
+                        Requirement {
                             name: requirement.name.clone(),
                             extras: requirement.extras.clone(),
                             source: requirement.source.clone(),
                             origin: requirement.origin.clone(),
-                                marker,
+                            marker: marker.simplify_extras(slice::from_ref(&extra)),
-                            })
                         }
                     }
                 };
                 if name == Some(&requirement.name) {
                     // Add each transitively included extra.
                     queue.extend(
-                        requirement.extras.iter().cloned().map(|extra| {
+                        requirement
-                            (extra, requirement.marker.clone().simplify_extras(extras))
+                            .extras
-                        }),
+                            .iter()
+                            .cloned()
+                            .map(|extra| (extra, requirement.marker.clone())),
                     );
                 } else {
                     // Add the requirements for that extra.
-                    requirements.push(requirement);
+                    requirements.push(Cow::Owned(requirement));
                 }
             }
         }

crates/uv/tests/it/pip_compile.rs

@@ -9870,6 +9870,136 @@ dev = [
     Ok(())
 }
 
+/// Resolve from a `pyproject.toml` file with a recursive extra, with a marker attached.
+#[test]
+fn compile_pyproject_toml_recursive_extra_marker() -> Result<()> {
+    let context = TestContext::new("3.12");
+    let pyproject_toml = context.temp_dir.child("pyproject.toml");
+    pyproject_toml.write_str(
+        r#"
+[project]
+name = "project"
+version = "0.0.1"
+dependencies = [
+    "anyio"
+]
+
+[project.optional-dependencies]
+test = [
+    "iniconfig",
+]
+dev = [
+    "project[test] ; sys_platform == 'darwin'",
+]
+"#,
+    )?;
+
+    uv_snapshot!(context.filters(), context.pip_compile()
+            .arg("pyproject.toml")
+            .arg("--extra")
+            .arg("dev")
+            .arg("--universal"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    # This file was autogenerated by uv via the following command:
+    #    uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --extra dev --universal
+    anyio==4.3.0
+        # via project (pyproject.toml)
+    idna==3.6
+        # via anyio
+    iniconfig==2.0.0 ; sys_platform == 'darwin'
+        # via project (pyproject.toml)
+    sniffio==1.3.1
+        # via anyio
+
+    ----- stderr -----
+    Resolved 4 packages in [TIME]
+    "###
+    );
+
+    Ok(())
+}
+
+/// Resolve from a `pyproject.toml` file with multiple recursive extras.
+#[test]
+fn compile_pyproject_toml_deeply_recursive_extra() -> Result<()> {
+    let context = TestContext::new("3.12");
+    let pyproject_toml = context.temp_dir.child("pyproject.toml");
+    pyproject_toml.write_str(
+        r#"
+[project]
+name = "project"
+version = "0.0.1"
+dependencies = []
+
+[project.optional-dependencies]
+foo = ["iniconfig"]
+bar = ["project[foo]"]
+baz = ["project[bar]"]
+bop = ["project[bar] ; sys_platform == 'darwin'"]
+qux = ["project[bop] ; python_version == '3.12'"]
+"#,
+    )?;
+
+    uv_snapshot!(context.filters(), context.pip_compile()
+            .arg("pyproject.toml")
+            .arg("--universal")
+            .arg("--extra")
+            .arg("qux"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    # This file was autogenerated by uv via the following command:
+    #    uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --extra qux
+    iniconfig==2.0.0 ; python_full_version < '3.13' and sys_platform == 'darwin'
+        # via project (pyproject.toml)
+
+    ----- stderr -----
+    Resolved 1 package in [TIME]
+    "###
+    );
+
+    uv_snapshot!(context.filters(), context.pip_compile()
+            .arg("pyproject.toml")
+            .arg("--universal")
+            .arg("--extra")
+            .arg("bop")
+            .arg("--extra")
+            .arg("bar"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    # This file was autogenerated by uv via the following command:
+    #    uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --extra bop --extra bar
+    iniconfig==2.0.0
+        # via project (pyproject.toml)
+
+    ----- stderr -----
+    Resolved 1 package in [TIME]
+    "###
+    );
+
+    uv_snapshot!(context.filters(), context.pip_compile()
+            .arg("pyproject.toml")
+            .arg("--universal")
+            .arg("--all-extras"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    # This file was autogenerated by uv via the following command:
+    #    uv pip compile --cache-dir [CACHE_DIR] pyproject.toml --universal --all-extras
+    iniconfig==2.0.0
+        # via project (pyproject.toml)
+
+    ----- stderr -----
+    Resolved 1 package in [TIME]
+    "###
+    );
+
+    Ok(())
+}
+
 /// The dependencies of a local editable dependency should be considered "direct" dependencies.
 #[test]
 fn editable_direct_dependency() -> Result<()> {