From 1bd5d8bc34d2a59f85f78511145a8706bb308d9b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 29 May 2024 15:08:20 -0400 Subject: [PATCH] Include all extras when generating lockfile (#3912) ## Summary This PR just ensures that when running `uv lock` (or `uv run`), we lock with all extras. When we later install, we'll also _install_ with all extras, but that will be changed in a future PR. --- crates/uv-requirements/src/specification.rs | 4 +- crates/uv-requirements/src/workspace.rs | 62 ++++-- crates/uv-resolver/src/resolution/graph.rs | 4 + crates/uv/src/commands/project/lock.rs | 48 ++--- crates/uv/src/commands/project/run.rs | 3 +- crates/uv/tests/lock.rs | 202 ++++++++++++++++++++ 6 files changed, 280 insertions(+), 43 deletions(-) diff --git a/crates/uv-requirements/src/specification.rs b/crates/uv-requirements/src/specification.rs index 46992e1d7..8e9286595 100644 --- a/crates/uv-requirements/src/specification.rs +++ b/crates/uv-requirements/src/specification.rs @@ -451,11 +451,13 @@ impl RequirementsSpecification { } /// Read the combined requirements and constraints from a set of sources. + /// + /// If a [`Workspace`] is provided, it will be used as-is without re-discovering a workspace + /// from the filesystem. pub async fn from_sources( requirements: &[RequirementsSource], constraints: &[RequirementsSource], overrides: &[RequirementsSource], - // Avoid re-discovering the workspace if we already loaded it. workspace: Option<&Workspace>, extras: &ExtrasSpecification, client_builder: &BaseClientBuilder<'_>, diff --git a/crates/uv-requirements/src/workspace.rs b/crates/uv-requirements/src/workspace.rs index dc1f73da9..94a48925a 100644 --- a/crates/uv-requirements/src/workspace.rs +++ b/crates/uv-requirements/src/workspace.rs @@ -3,16 +3,17 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use distribution_types::{Requirement, RequirementSource}; use glob::{glob, GlobError, PatternError}; +use pep508_rs::{VerbatimUrl, VerbatimUrlError}; use rustc_hash::FxHashSet; use tracing::{debug, trace}; use uv_fs::{absolutize_path, Simplified}; -use uv_normalize::PackageName; +use uv_normalize::{ExtraName, PackageName}; use uv_warnings::warn_user; use crate::pyproject::{PyProjectToml, Source, ToolUvWorkspace}; -use crate::RequirementsSource; #[derive(thiserror::Error, Debug)] pub enum WorkspaceError { @@ -32,6 +33,8 @@ pub enum WorkspaceError { DynamicNotAllowed(&'static str), #[error("Failed to normalize workspace member path")] Normalize(#[source] std::io::Error), + #[error("Failed to normalize workspace member path")] + VerbatimUrl(#[from] VerbatimUrlError), } /// A workspace, consisting of a root directory and members. See [`ProjectWorkspace`]. @@ -172,6 +175,8 @@ pub struct ProjectWorkspace { project_root: PathBuf, /// The name of the package. project_name: PackageName, + /// The extras available in the project. + extras: Vec, /// The workspace the project is part of. workspace: Workspace, } @@ -235,31 +240,46 @@ impl ProjectWorkspace { )) } - /// The directory containing the closest `pyproject.toml`, defining the current project. + /// Returns the directory containing the closest `pyproject.toml` that defines the current + /// project. pub fn project_root(&self) -> &Path { &self.project_root } - /// The name of the current project. + /// Returns the [`PackageName`] of the current project. pub fn project_name(&self) -> &PackageName { &self.project_name } - /// The workspace definition. + /// Returns the extras available in the project. + pub fn project_extras(&self) -> &[ExtraName] { + &self.extras + } + + /// Returns the [`Workspace`] containing the current project. pub fn workspace(&self) -> &Workspace { &self.workspace } - /// The current project. + /// Returns the current project as a [`WorkspaceMember`]. pub fn current_project(&self) -> &WorkspaceMember { &self.workspace().packages[&self.project_name] } - /// Return the requirements for the project, which is the current project as editable. - pub fn requirements(&self) -> Vec { - vec![RequirementsSource::Editable( - self.project_root.to_string_lossy().to_string(), - )] + /// Return the [`Requirement`] entries for the project, which is the current project as + /// editable. + pub fn requirements(&self) -> Vec { + vec![Requirement { + name: self.project_name.clone(), + extras: self.extras.clone(), + marker: None, + source: RequirementSource::Path { + path: self.project_root.clone(), + editable: true, + url: VerbatimUrl::from_path(&self.project_root).expect("path is valid URL"), + }, + origin: None, + }] } /// Find the workspace for a project. @@ -272,6 +292,18 @@ impl ProjectWorkspace { .map_err(WorkspaceError::Normalize)? .to_path_buf(); + // Extract the extras available in the project. + let extras = project + .project + .as_ref() + .and_then(|project| project.optional_dependencies.as_ref()) + .map(|optional_dependencies| { + let mut extras = optional_dependencies.keys().cloned().collect::>(); + extras.sort_unstable(); + extras + }) + .unwrap_or_default(); + let mut workspace_members = BTreeMap::new(); // The current project is always a workspace member, especially in a single project // workspace. @@ -305,6 +337,7 @@ impl ProjectWorkspace { return Ok(Self { project_root: project_path.clone(), project_name, + extras, workspace: Workspace { root: project_path, packages: workspace_members, @@ -385,6 +418,7 @@ impl ProjectWorkspace { Ok(Self { project_root: project_path.clone(), project_name, + extras, workspace: Workspace { root: workspace_root, packages: workspace_members, @@ -412,6 +446,7 @@ impl ProjectWorkspace { Self { project_root: root.to_path_buf(), project_name: project_name.clone(), + extras: Vec::new(), workspace: Workspace { root: root.to_path_buf(), packages: [(project_name.clone(), root_member)].into_iter().collect(), @@ -627,6 +662,7 @@ mod tests { { "project_root": "[ROOT]/albatross-in-example/examples/bird-feeder", "project_name": "bird-feeder", + "extras": [], "workspace": { "root": "[ROOT]/albatross-in-example/examples/bird-feeder", "packages": { @@ -657,6 +693,7 @@ mod tests { { "project_root": "[ROOT]/albatross-project-in-excluded/excluded/bird-feeder", "project_name": "bird-feeder", + "extras": [], "workspace": { "root": "[ROOT]/albatross-project-in-excluded/excluded/bird-feeder", "packages": { @@ -686,6 +723,7 @@ mod tests { { "project_root": "[ROOT]/albatross-root-workspace", "project_name": "albatross", + "extras": [], "workspace": { "root": "[ROOT]/albatross-root-workspace", "packages": { @@ -729,6 +767,7 @@ mod tests { { "project_root": "[ROOT]/albatross-virtual-workspace/packages/albatross", "project_name": "albatross", + "extras": [], "workspace": { "root": "[ROOT]/albatross-virtual-workspace", "packages": { @@ -766,6 +805,7 @@ mod tests { { "project_root": "[ROOT]/albatross-just-project", "project_name": "albatross", + "extras": [], "workspace": { "root": "[ROOT]/albatross-just-project", "packages": { diff --git a/crates/uv-resolver/src/resolution/graph.rs b/crates/uv-resolver/src/resolution/graph.rs index 594f4d70b..8c8a19d98 100644 --- a/crates/uv-resolver/src/resolution/graph.rs +++ b/crates/uv-resolver/src/resolution/graph.rs @@ -445,6 +445,10 @@ impl ResolutionGraph { let mut locked_dists = vec![]; for node_index in self.petgraph.node_indices() { let dist = &self.petgraph[node_index]; + if dist.extra.is_some() { + continue; + } + let mut locked_dist = lock::Distribution::from_annotated_dist(dist)?; for edge in self.petgraph.neighbors(node_index) { let dependency_dist = &self.petgraph[edge]; diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index 7df3fca9a..8c5f36fe8 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -1,17 +1,17 @@ use anstream::eprint; use anyhow::Result; -use distribution_types::IndexLocations; +use distribution_types::{IndexLocations, UnresolvedRequirementSpecification}; use install_wheel_rs::linker::LinkMode; use uv_cache::Cache; -use uv_client::{BaseClientBuilder, RegistryClientBuilder}; +use uv_client::RegistryClientBuilder; use uv_configuration::{ Concurrency, ConfigSettings, ExtrasSpecification, NoBinary, NoBuild, PreviewMode, Reinstall, SetupPyStrategy, Upgrade, }; use uv_dispatch::BuildDispatch; use uv_interpreter::PythonEnvironment; -use uv_requirements::{ProjectWorkspace, RequirementsSpecification}; +use uv_requirements::ProjectWorkspace; use uv_resolver::{ExcludeNewer, FlatIndex, InMemoryIndex, Lock, OptionsBuilder}; use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight}; use uv_warnings::warn_user; @@ -39,7 +39,7 @@ pub(crate) async fn lock( let venv = project::init_environment(&project, preview, cache, printer)?; // Perform the lock operation. - match do_lock(&project, &venv, exclude_newer, preview, cache, printer).await { + match do_lock(&project, &venv, exclude_newer, cache, printer).await { Ok(_) => Ok(ExitStatus::Success), Err(ProjectError::Operation(pip::operations::Error::Resolve( uv_resolver::ResolveError::NoSolution(err), @@ -58,29 +58,19 @@ pub(super) async fn do_lock( project: &ProjectWorkspace, venv: &PythonEnvironment, exclude_newer: Option, - preview: PreviewMode, cache: &Cache, printer: Printer, ) -> Result { - // TODO(zanieb): Support client configuration - let client_builder = BaseClientBuilder::default(); - - // Read all requirements from the provided sources. - // TODO(zanieb): Consider allowing constraints and extras - // TODO(zanieb): Allow specifying extras somehow - let spec = RequirementsSpecification::from_sources( - // TODO(konsti): With workspace (just like with extras), these are the requirements for - // syncing. For locking, we want to use the entire workspace with all extras. - // See https://github.com/astral-sh/uv/issues/3700 - &project.requirements(), - &[], - &[], - None, - &ExtrasSpecification::None, - &client_builder, - preview, - ) - .await?; + // When locking, include the project itself (as editable). + let requirements = project + .requirements() + .into_iter() + .map(UnresolvedRequirementSpecification::from) + .collect::>(); + let constraints = vec![]; + let overrides = vec![]; + let source_trees = vec![]; + let project_name = project.project_name().clone(); // Determine the tags, markers, and interpreter to use for resolution. let interpreter = venv.interpreter().clone(); @@ -133,11 +123,11 @@ pub(super) async fn do_lock( // Resolve the requirements. let resolution = pip::operations::resolve( - spec.requirements, - spec.constraints, - spec.overrides, - spec.source_trees, - spec.project, + requirements, + constraints, + overrides, + source_trees, + Some(project_name), &extras, EmptyInstalledPackages, &hasher, diff --git a/crates/uv/src/commands/project/run.rs b/crates/uv/src/commands/project/run.rs index b4c1f8e94..5abef5a08 100644 --- a/crates/uv/src/commands/project/run.rs +++ b/crates/uv/src/commands/project/run.rs @@ -46,8 +46,7 @@ pub(crate) async fn run( let venv = project::init_environment(&project, preview, cache, printer)?; // Lock and sync the environment. - let lock = - project::lock::do_lock(&project, &venv, exclude_newer, preview, cache, printer).await?; + let lock = project::lock::do_lock(&project, &venv, exclude_newer, cache, printer).await?; project::sync::do_sync(&project, &venv, &lock, cache, printer).await?; Some(venv) diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index 4a21c35b4..bcb641416 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -561,3 +561,205 @@ fn lock_sdist_url() -> Result<()> { Ok(()) } + +/// Lock a project with an extra. When resolving, all extras should be included. +#[test] +fn lock_extra() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + dependencies = ["anyio==3.7.0"] + + [project.optional-dependencies] + test = ["pytest"] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + warning: `uv lock` is experimental and may change without warning. + Resolved 8 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock"))?; + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + + [[distribution]] + name = "anyio" + version = "3.7.0" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/c6/b3/fefbf7e78ab3b805dec67d698dc18dd505af7a18a8dd08868c9b4fa736b5/anyio-3.7.0.tar.gz" + hash = "sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce" + size = 142737 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/68/fe/7ce1926952c8a403b35029e194555558514b365ad77d75125f521a2bec62/anyio-3.7.0-py3-none-any.whl" + hash = "sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0" + size = 80873 + + [[distribution.dependencies]] + name = "idna" + version = "3.6" + source = "registry+https://pypi.org/simple" + + [[distribution.dependencies]] + name = "sniffio" + version = "1.3.1" + source = "registry+https://pypi.org/simple" + + [[distribution]] + name = "idna" + version = "3.6" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz" + hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca" + size = 175426 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl" + hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" + size = 61567 + + [[distribution]] + name = "iniconfig" + version = "2.0.0" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz" + hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3" + size = 4646 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" + hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" + size = 5892 + + [[distribution]] + name = "packaging" + version = "24.0" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/ee/b5/b43a27ac7472e1818c4bafd44430e69605baefe1f34440593e0332ec8b4d/packaging-24.0.tar.gz" + hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" + size = 147882 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl" + hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5" + size = 53488 + + [[distribution]] + name = "pluggy" + version = "1.4.0" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/54/c6/43f9d44d92aed815e781ca25ba8c174257e27253a94630d21be8725a2b59/pluggy-1.4.0.tar.gz" + hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be" + size = 65812 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/a5/5b/0cc789b59e8cc1bf288b38111d002d8c5917123194d45b29dcdac64723cc/pluggy-1.4.0-py3-none-any.whl" + hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981" + size = 20120 + + [[distribution]] + name = "project" + version = "0.1.0" + source = "editable+file://[TEMP_DIR]/" + + [distribution.sdist] + url = "file://[TEMP_DIR]/" + + [[distribution.dependencies]] + name = "anyio" + version = "3.7.0" + source = "registry+https://pypi.org/simple" + + [[distribution]] + name = "pytest" + version = "8.1.1" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/30/b7/7d44bbc04c531dcc753056920e0988032e5871ac674b5a84cb979de6e7af/pytest-8.1.1.tar.gz" + hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044" + size = 1409703 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/4d/7e/c79cecfdb6aa85c6c2e3cf63afc56d0f165f24f5c66c03c695c4d9b84756/pytest-8.1.1-py3-none-any.whl" + hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7" + size = 337359 + + [[distribution.dependencies]] + name = "iniconfig" + version = "2.0.0" + source = "registry+https://pypi.org/simple" + + [[distribution.dependencies]] + name = "packaging" + version = "24.0" + source = "registry+https://pypi.org/simple" + + [[distribution.dependencies]] + name = "pluggy" + version = "1.4.0" + source = "registry+https://pypi.org/simple" + + [[distribution]] + name = "sniffio" + version = "1.3.1" + source = "registry+https://pypi.org/simple" + + [distribution.sdist] + url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz" + hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc" + size = 20372 + + [[distribution.wheel]] + url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl" + hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2" + size = 10235 + "### + ); + }); + + // Install from the lockfile. + uv_snapshot!(context.filters(), context.sync(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + warning: `uv sync` is experimental and may change without warning. + Downloaded 4 packages in [TIME] + Installed 4 packages in [TIME] + + anyio==3.7.0 + + idna==3.6 + + project==0.1.0 (from file://[TEMP_DIR]/) + + sniffio==1.3.1 + "###); + + Ok(()) +}