From e843433b07db08bdabd64b4cfe822e57b3a31154 Mon Sep 17 00:00:00 2001 From: konsti Date: Mon, 10 Mar 2025 22:03:30 +0100 Subject: [PATCH] Cache workspace discovery (#12096) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce the overhead of `uv run` in large workspaces. Instead of re-discovering the entire workspace each time we resolve the metadata of a member, we can the discovered set of workspace members. Care needs to be taken to not cache the discovery for `uv init`, `uv add` and `uv remove`, which change the definitions of workspace members. Below is apache airflow e3fe06382df4b19f2c0de40ce7c0bdc726754c74 `uv run python` with a minimal payload. With this change, we avoid a ~350ms overhead of each `uv run` invocation. ``` $ hyperfine --warmup 2 \ "uv run --no-dev python -c \"print('hi')\"" \ "uv-profiling run --no-dev python -c \"print('hi')\"" Benchmark 1: uv run --no-dev python -c "print('hi')" Time (mean ± σ): 492.6 ms ± 7.0 ms [User: 393.2 ms, System: 97.1 ms] Range (min … max): 482.3 ms … 501.5 ms 10 runs Benchmark 2: uv-profiling run --no-dev python -c "print('hi')" Time (mean ± σ): 129.7 ms ± 2.5 ms [User: 105.4 ms, System: 23.2 ms] Range (min … max): 126.0 ms … 136.1 ms 22 runs Summary uv-profiling run --no-dev python -c "print('hi')" ran 3.80 ± 0.09 times faster than uv run --no-dev python -c "print('hi')" ``` The profile after those change below. We still spend a large chunk in toml parsing (both `uv.lock` and `pyproject.toml`), but it's not excessive anymore. ![image](https://github.com/user-attachments/assets/6fe78510-7e25-48ee-8a6d-220ee98ad120) --- Cargo.lock | 4 + crates/uv-bench/Cargo.toml | 1 + crates/uv-bench/benches/uv.rs | 3 + crates/uv-build-frontend/Cargo.toml | 1 + crates/uv-build-frontend/src/lib.rs | 8 + crates/uv-dispatch/Cargo.toml | 1 + crates/uv-dispatch/src/lib.rs | 10 +- .../src/metadata/build_requires.rs | 7 +- crates/uv-distribution/src/metadata/mod.rs | 4 +- .../src/metadata/requires_dist.rs | 12 +- crates/uv-distribution/src/source/mod.rs | 9 + crates/uv-types/Cargo.toml | 1 + crates/uv-types/src/traits.rs | 4 + crates/uv-workspace/src/lib.rs | 4 +- crates/uv-workspace/src/workspace.rs | 284 +++++++++++------- crates/uv/src/commands/build_frontend.rs | 12 +- crates/uv/src/commands/pip/compile.rs | 2 + crates/uv/src/commands/pip/install.rs | 2 + crates/uv/src/commands/pip/sync.rs | 2 + crates/uv/src/commands/project/add.rs | 24 +- crates/uv/src/commands/project/export.rs | 9 +- crates/uv/src/commands/project/init.rs | 6 +- crates/uv/src/commands/project/lock.rs | 10 +- crates/uv/src/commands/project/mod.rs | 11 +- crates/uv/src/commands/project/remove.rs | 22 +- crates/uv/src/commands/project/run.rs | 15 +- crates/uv/src/commands/project/sync.rs | 11 +- crates/uv/src/commands/project/tree.rs | 7 +- crates/uv/src/commands/python/find.rs | 7 +- crates/uv/src/commands/python/pin.rs | 7 +- crates/uv/src/commands/tool/install.rs | 7 +- crates/uv/src/commands/tool/run.rs | 5 + crates/uv/src/commands/tool/upgrade.rs | 3 + crates/uv/src/commands/venv.rs | 9 +- crates/uv/src/lib.rs | 5 +- crates/uv/tests/it/lock.rs | 6 +- 36 files changed, 381 insertions(+), 154 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 941802e2d..77b483a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4669,6 +4669,7 @@ dependencies = [ "uv-python", "uv-resolver", "uv-types", + "uv-workspace", ] [[package]] @@ -4745,6 +4746,7 @@ dependencies = [ "uv-types", "uv-virtualenv", "uv-warnings", + "uv-workspace", ] [[package]] @@ -4997,6 +4999,7 @@ dependencies = [ "uv-resolver", "uv-types", "uv-version", + "uv-workspace", ] [[package]] @@ -5778,6 +5781,7 @@ dependencies = [ "uv-pep508", "uv-pypi-types", "uv-python", + "uv-workspace", ] [[package]] diff --git a/crates/uv-bench/Cargo.toml b/crates/uv-bench/Cargo.toml index 81f6d973d..fe02b3173 100644 --- a/crates/uv-bench/Cargo.toml +++ b/crates/uv-bench/Cargo.toml @@ -45,6 +45,7 @@ uv-pypi-types = { workspace = true } uv-python = { workspace = true } uv-resolver = { workspace = true } uv-types = { workspace = true } +uv-workspace = { workspace = true } anyhow = { workspace = true } codspeed-criterion-compat = { version = "2.7.2", default-features = false, optional = true } diff --git a/crates/uv-bench/benches/uv.rs b/crates/uv-bench/benches/uv.rs index 16d428ff2..6e546c0ba 100644 --- a/crates/uv-bench/benches/uv.rs +++ b/crates/uv-bench/benches/uv.rs @@ -103,6 +103,7 @@ mod resolver { Resolver, ResolverEnvironment, ResolverOutput, }; use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy}; + use uv_workspace::WorkspaceCache; static MARKERS: LazyLock = LazyLock::new(|| { MarkerEnvironment::try_from(MarkerEnvironmentBuilder { @@ -161,6 +162,7 @@ mod resolver { let sources = SourceStrategy::default(); let dependency_metadata = DependencyMetadata::default(); let conflicts = Conflicts::empty(); + let workspace_cache = WorkspaceCache::default(); let python_requirement = if universal { PythonRequirement::from_requires_python( @@ -188,6 +190,7 @@ mod resolver { &hashes, exclude_newer, sources, + workspace_cache, concurrency, PreviewMode::Enabled, ); diff --git a/crates/uv-build-frontend/Cargo.toml b/crates/uv-build-frontend/Cargo.toml index d18336350..83f8008d9 100644 --- a/crates/uv-build-frontend/Cargo.toml +++ b/crates/uv-build-frontend/Cargo.toml @@ -29,6 +29,7 @@ uv-static = { workspace = true } uv-types = { workspace = true } uv-virtualenv = { workspace = true } uv-warnings = { workspace = true } +uv-workspace = { workspace = true } anstream = { workspace = true } fs-err = { workspace = true } diff --git a/crates/uv-build-frontend/src/lib.rs b/crates/uv-build-frontend/src/lib.rs index efd2c2e7e..1ea3bc587 100644 --- a/crates/uv-build-frontend/src/lib.rs +++ b/crates/uv-build-frontend/src/lib.rs @@ -38,6 +38,7 @@ use uv_python::{Interpreter, PythonEnvironment}; use uv_static::EnvVars; use uv_types::{AnyErrorBuild, BuildContext, BuildIsolation, BuildStack, SourceBuildTrait}; use uv_warnings::warn_user_once; +use uv_workspace::WorkspaceCache; pub use crate::error::{Error, MissingHeaderCause}; @@ -269,6 +270,7 @@ impl SourceBuild { version_id: Option<&str>, locations: &IndexLocations, source_strategy: SourceStrategy, + workspace_cache: &WorkspaceCache, config_settings: ConfigSettings, build_isolation: BuildIsolation<'_>, build_stack: &BuildStack, @@ -294,6 +296,7 @@ impl SourceBuild { fallback_package_name, locations, source_strategy, + workspace_cache, &default_backend, ) .await @@ -396,6 +399,7 @@ impl SourceBuild { version_id, locations, source_strategy, + workspace_cache, build_stack, build_kind, level, @@ -466,6 +470,7 @@ impl SourceBuild { package_name: Option<&PackageName>, locations: &IndexLocations, source_strategy: SourceStrategy, + workspace_cache: &WorkspaceCache, default_backend: &Pep517Backend, ) -> Result<(Pep517Backend, Option), Box> { match fs::read_to_string(source_tree.join("pyproject.toml")) { @@ -496,6 +501,7 @@ impl SourceBuild { install_path, locations, source_strategy, + workspace_cache, ) .await .map_err(Error::Lowering)?; @@ -857,6 +863,7 @@ async fn create_pep517_build_environment( version_id: Option<&str>, locations: &IndexLocations, source_strategy: SourceStrategy, + workspace_cache: &WorkspaceCache, build_stack: &BuildStack, build_kind: BuildKind, level: BuildOutput, @@ -957,6 +964,7 @@ async fn create_pep517_build_environment( install_path, locations, source_strategy, + workspace_cache, ) .await .map_err(Error::Lowering)?; diff --git a/crates/uv-dispatch/Cargo.toml b/crates/uv-dispatch/Cargo.toml index 07b0e1ab2..06cb2db15 100644 --- a/crates/uv-dispatch/Cargo.toml +++ b/crates/uv-dispatch/Cargo.toml @@ -34,6 +34,7 @@ uv-python = { workspace = true } uv-resolver = { workspace = true } uv-types = { workspace = true } uv-version = { workspace = true } +uv-workspace = { workspace = true } anyhow = { workspace = true } futures = { workspace = true } diff --git a/crates/uv-dispatch/src/lib.rs b/crates/uv-dispatch/src/lib.rs index 921752439..421a0a2a5 100644 --- a/crates/uv-dispatch/src/lib.rs +++ b/crates/uv-dispatch/src/lib.rs @@ -38,6 +38,7 @@ use uv_types::{ AnyErrorBuild, BuildContext, BuildIsolation, BuildStack, EmptyInstalledPackages, HashStrategy, InFlight, }; +use uv_workspace::WorkspaceCache; #[derive(Debug, Error)] pub enum BuildDispatchError { @@ -94,6 +95,7 @@ pub struct BuildDispatch<'a> { source_build_context: SourceBuildContext, build_extra_env_vars: FxHashMap, sources: SourceStrategy, + workspace_cache: WorkspaceCache, concurrency: Concurrency, preview: PreviewMode, } @@ -116,6 +118,7 @@ impl<'a> BuildDispatch<'a> { hasher: &'a HashStrategy, exclude_newer: Option, sources: SourceStrategy, + workspace_cache: WorkspaceCache, concurrency: Concurrency, preview: PreviewMode, ) -> Self { @@ -137,8 +140,8 @@ impl<'a> BuildDispatch<'a> { exclude_newer, source_build_context: SourceBuildContext::default(), build_extra_env_vars: FxHashMap::default(), - sources, + workspace_cache, concurrency, preview, } @@ -200,6 +203,10 @@ impl BuildContext for BuildDispatch<'_> { self.index_locations } + fn workspace_cache(&self) -> &WorkspaceCache { + &self.workspace_cache + } + async fn resolve<'data>( &'data self, requirements: &'data [Requirement], @@ -417,6 +424,7 @@ impl BuildContext for BuildDispatch<'_> { version_id, self.index_locations, sources, + self.workspace_cache(), self.config_settings.clone(), self.build_isolation, &build_stack, diff --git a/crates/uv-distribution/src/metadata/build_requires.rs b/crates/uv-distribution/src/metadata/build_requires.rs index 91c47c4bf..56d35b63f 100644 --- a/crates/uv-distribution/src/metadata/build_requires.rs +++ b/crates/uv-distribution/src/metadata/build_requires.rs @@ -5,7 +5,9 @@ use uv_configuration::SourceStrategy; use uv_distribution_types::IndexLocations; use uv_normalize::PackageName; use uv_workspace::pyproject::ToolUvSources; -use uv_workspace::{DiscoveryOptions, MemberDiscovery, ProjectWorkspace, Workspace}; +use uv_workspace::{ + DiscoveryOptions, MemberDiscovery, ProjectWorkspace, Workspace, WorkspaceCache, +}; use crate::metadata::{LoweredRequirement, MetadataError}; @@ -37,6 +39,7 @@ impl BuildRequires { install_path: &Path, locations: &IndexLocations, sources: SourceStrategy, + cache: &WorkspaceCache, ) -> Result { let discovery = match sources { SourceStrategy::Enabled => DiscoveryOptions::default(), @@ -48,7 +51,7 @@ impl BuildRequires { // TODO(konsti): Cache workspace discovery. let Some(project_workspace) = - ProjectWorkspace::from_maybe_project_root(install_path, &discovery).await? + ProjectWorkspace::from_maybe_project_root(install_path, &discovery, cache).await? else { return Ok(Self::from_metadata23(metadata)); }; diff --git a/crates/uv-distribution/src/metadata/mod.rs b/crates/uv-distribution/src/metadata/mod.rs index fdba4ad98..cfd6d174f 100644 --- a/crates/uv-distribution/src/metadata/mod.rs +++ b/crates/uv-distribution/src/metadata/mod.rs @@ -9,7 +9,7 @@ use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::{Version, VersionSpecifiers}; use uv_pypi_types::{HashDigests, ResolutionMetadata}; use uv_workspace::dependency_groups::DependencyGroupError; -use uv_workspace::WorkspaceError; +use uv_workspace::{WorkspaceCache, WorkspaceError}; pub use crate::metadata::build_requires::BuildRequires; pub use crate::metadata::lowering::LoweredRequirement; @@ -80,6 +80,7 @@ impl Metadata { git_source: Option<&GitWorkspaceMember<'_>>, locations: &IndexLocations, sources: SourceStrategy, + cache: &WorkspaceCache, ) -> Result { // Lower the requirements. let requires_dist = uv_pypi_types::RequiresDist { @@ -100,6 +101,7 @@ impl Metadata { git_source, locations, sources, + cache, ) .await?; diff --git a/crates/uv-distribution/src/metadata/requires_dist.rs b/crates/uv-distribution/src/metadata/requires_dist.rs index 0653fd3c9..add14ee9d 100644 --- a/crates/uv-distribution/src/metadata/requires_dist.rs +++ b/crates/uv-distribution/src/metadata/requires_dist.rs @@ -10,7 +10,7 @@ use uv_normalize::{ExtraName, GroupName, PackageName, DEV_DEPENDENCIES}; use uv_pep508::MarkerTree; use uv_workspace::dependency_groups::FlatDependencyGroups; use uv_workspace::pyproject::{Sources, ToolUvSources}; -use uv_workspace::{DiscoveryOptions, MemberDiscovery, ProjectWorkspace}; +use uv_workspace::{DiscoveryOptions, MemberDiscovery, ProjectWorkspace, WorkspaceCache}; use crate::metadata::{GitWorkspaceMember, LoweredRequirement, MetadataError}; use crate::Metadata; @@ -49,6 +49,7 @@ impl RequiresDist { git_member: Option<&GitWorkspaceMember<'_>>, locations: &IndexLocations, sources: SourceStrategy, + cache: &WorkspaceCache, ) -> Result { // TODO(konsti): Cache workspace discovery. let discovery_options = DiscoveryOptions { @@ -57,6 +58,7 @@ impl RequiresDist { .fetch_root .parent() .expect("git checkout has a parent") + .to_path_buf() }), members: match sources { SourceStrategy::Enabled => MemberDiscovery::default(), @@ -64,7 +66,8 @@ impl RequiresDist { }, }; let Some(project_workspace) = - ProjectWorkspace::from_maybe_project_root(install_path, &discovery_options).await? + ProjectWorkspace::from_maybe_project_root(install_path, &discovery_options, cache) + .await? else { return Ok(Self::from_metadata23(metadata)); }; @@ -475,7 +478,7 @@ mod test { use uv_normalize::PackageName; use uv_pep508::Requirement; use uv_workspace::pyproject::PyProjectToml; - use uv_workspace::{DiscoveryOptions, ProjectWorkspace}; + use uv_workspace::{DiscoveryOptions, ProjectWorkspace, WorkspaceCache}; use crate::metadata::requires_dist::FlatRequiresDist; use crate::RequiresDist; @@ -491,9 +494,10 @@ mod test { .context("metadata field project not found")?, &pyproject_toml, &DiscoveryOptions { - stop_discovery_at: Some(path), + stop_discovery_at: Some(path.to_path_buf()), ..DiscoveryOptions::default() }, + &WorkspaceCache::default(), ) .await?; let requires_dist = uv_pypi_types::RequiresDist::parse_pyproject_toml(contents)?; diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 0ed2560e4..d7af5d331 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -1171,6 +1171,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { None, self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )); @@ -1223,6 +1224,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { None, self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )); @@ -1271,6 +1273,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { None, self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )); @@ -1328,6 +1331,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { None, self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )) @@ -1395,6 +1399,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { None, self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?; Ok(Some(requires_dist)) @@ -1653,6 +1658,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { Some(&git_member), self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )); @@ -1685,6 +1691,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { Some(&git_member), self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )); @@ -1736,6 +1743,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { Some(&git_member), self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )); @@ -1793,6 +1801,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { Some(&git_member), self.build_context.locations(), self.build_context.sources(), + self.build_context.workspace_cache(), ) .await?, )) diff --git a/crates/uv-types/Cargo.toml b/crates/uv-types/Cargo.toml index 192e50eb6..08a00e006 100644 --- a/crates/uv-types/Cargo.toml +++ b/crates/uv-types/Cargo.toml @@ -27,6 +27,7 @@ uv-pep440 = { workspace = true } uv-pep508 = { workspace = true } uv-pypi-types = { workspace = true } uv-python = { workspace = true } +uv-workspace = { workspace = true } anyhow = { workspace = true } rustc-hash = { workspace = true } diff --git a/crates/uv-types/src/traits.rs b/crates/uv-types/src/traits.rs index 8cc37e714..5beef36e7 100644 --- a/crates/uv-types/src/traits.rs +++ b/crates/uv-types/src/traits.rs @@ -17,6 +17,7 @@ use uv_git::GitResolver; use uv_pep508::PackageName; use uv_pypi_types::Requirement; use uv_python::{Interpreter, PythonEnvironment}; +use uv_workspace::WorkspaceCache; /// Avoids cyclic crate dependencies between resolver, installer and builder. /// @@ -88,6 +89,9 @@ pub trait BuildContext { /// The index locations being searched. fn locations(&self) -> &IndexLocations; + /// Workspace discovery caching. + fn workspace_cache(&self) -> &WorkspaceCache; + /// Resolve the given requirements into a ready-to-install set of package versions. fn resolve<'a>( &'a self, diff --git a/crates/uv-workspace/src/lib.rs b/crates/uv-workspace/src/lib.rs index bd435c1f1..83be6bd88 100644 --- a/crates/uv-workspace/src/lib.rs +++ b/crates/uv-workspace/src/lib.rs @@ -1,6 +1,6 @@ pub use workspace::{ - DiscoveryOptions, MemberDiscovery, ProjectWorkspace, VirtualProject, Workspace, WorkspaceError, - WorkspaceMember, + DiscoveryOptions, MemberDiscovery, ProjectWorkspace, VirtualProject, Workspace, WorkspaceCache, + WorkspaceError, WorkspaceMember, }; pub mod dependency_groups; diff --git a/crates/uv-workspace/src/workspace.rs b/crates/uv-workspace/src/workspace.rs index aca1a593c..33103ab7b 100644 --- a/crates/uv-workspace/src/workspace.rs +++ b/crates/uv-workspace/src/workspace.rs @@ -1,11 +1,13 @@ //! Resolve the current [`ProjectWorkspace`] or [`Workspace`]. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use glob::{glob, GlobError, PatternError}; -use rustc_hash::FxHashSet; +use rustc_hash::{FxHashMap, FxHashSet}; use tracing::{debug, trace, warn}; + use uv_distribution_types::Index; use uv_fs::{Simplified, CWD}; use uv_normalize::{GroupName, PackageName, DEV_DEPENDENCIES}; @@ -22,6 +24,24 @@ use crate::pyproject::{ Project, PyProjectToml, PyprojectTomlError, Sources, ToolUvSources, ToolUvWorkspace, }; +type WorkspaceMembers = Arc>; + +/// Cache key for workspace discovery. +/// +/// Given this key, the discovered workspace member list is the same. +#[derive(Debug, Default, Clone, Hash, PartialEq, Eq)] +struct WorkspaceCacheKey { + workspace_root: PathBuf, + discovery_options: DiscoveryOptions, +} + +/// Cache for workspace discovery. +/// +/// Avoid re-reading the `pyproject.toml` files in a workspace for each member by caching the +/// workspace members by their workspace root. +#[derive(Debug, Default, Clone)] +pub struct WorkspaceCache(Arc>>); + #[derive(thiserror::Error, Debug)] pub enum WorkspaceError { // Workspace structure errors. @@ -58,23 +78,23 @@ pub enum WorkspaceError { Normalize(#[source] std::io::Error), } -#[derive(Debug, Default, Clone)] -pub enum MemberDiscovery<'a> { +#[derive(Debug, Default, Clone, Hash, PartialEq, Eq)] +pub enum MemberDiscovery { /// Discover all workspace members. #[default] All, /// Don't discover any workspace members. None, /// Discover workspace members, but ignore the given paths. - Ignore(FxHashSet<&'a Path>), + Ignore(BTreeSet), } -#[derive(Debug, Default, Clone)] -pub struct DiscoveryOptions<'a> { +#[derive(Debug, Default, Clone, Hash, PartialEq, Eq)] +pub struct DiscoveryOptions { /// The path to stop discovery at. - pub stop_discovery_at: Option<&'a Path>, + pub stop_discovery_at: Option, /// The strategy to use when discovering workspace members. - pub members: MemberDiscovery<'a>, + pub members: MemberDiscovery, } /// A workspace, consisting of a root directory and members. See [`ProjectWorkspace`]. @@ -87,7 +107,7 @@ pub struct Workspace { /// the `uv.tool.workspace`, or the `pyproject.toml` in an implicit single workspace project. install_path: PathBuf, /// The members of the workspace. - packages: BTreeMap, + packages: WorkspaceMembers, /// The sources table from the workspace `pyproject.toml`. /// /// This table is overridden by the project sources. @@ -124,7 +144,8 @@ impl Workspace { /// ``` pub async fn discover( path: &Path, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result { let path = std::path::absolute(path) .map_err(WorkspaceError::Normalize)? @@ -211,6 +232,7 @@ impl Workspace { workspace_pyproject_toml, current_project, options, + cache, ) .await } @@ -237,7 +259,7 @@ impl Workspace { pyproject_toml: PyProjectToml, ) -> Option { let mut packages = self.packages; - let member = packages.get_mut(package_name)?; + let member = Arc::make_mut(&mut packages).get_mut(package_name)?; if member.root == self.install_path { // If the member is also the workspace root, update _both_ the member entry and the @@ -656,57 +678,113 @@ impl Workspace { workspace_definition: ToolUvWorkspace, workspace_pyproject_toml: PyProjectToml, current_project: Option, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result { + let cache_key = WorkspaceCacheKey { + workspace_root: workspace_root.clone(), + discovery_options: options.clone(), + }; + let cache_entry = { + // Acquire the lock for the minimal required region + let cache = cache.0.lock().expect("there was a panic in another thread"); + cache.get(&cache_key).cloned() + }; + let mut workspace_members = if let Some(workspace_members) = cache_entry { + trace!( + "Cached workspace members for: `{}`", + &workspace_root.simplified_display() + ); + workspace_members + } else { + trace!( + "Discovering workspace members for: `{}`", + &workspace_root.simplified_display() + ); + let workspace_members = Self::collect_members_only( + &workspace_root, + &workspace_definition, + &workspace_pyproject_toml, + options, + ) + .await?; + { + // Acquire the lock for the minimal required region + let mut cache = cache.0.lock().expect("there was a panic in another thread"); + cache.insert(cache_key, Arc::new(workspace_members.clone())); + } + Arc::new(workspace_members) + }; + + // For the cases such as `MemberDiscovery::None`, add the current project if missing. + if let Some(root_member) = current_project { + if !workspace_members.contains_key(&root_member.project.name) { + debug!( + "Adding current workspace member: `{}`", + root_member.root.simplified_display() + ); + + Arc::make_mut(&mut workspace_members) + .insert(root_member.project.name.clone(), root_member); + } + } + + let workspace_sources = workspace_pyproject_toml + .tool + .clone() + .and_then(|tool| tool.uv) + .and_then(|uv| uv.sources) + .map(ToolUvSources::into_inner) + .unwrap_or_default(); + + let workspace_indexes = workspace_pyproject_toml + .tool + .clone() + .and_then(|tool| tool.uv) + .and_then(|uv| uv.index) + .unwrap_or_default(); + + Ok(Workspace { + install_path: workspace_root, + packages: workspace_members, + sources: workspace_sources, + indexes: workspace_indexes, + pyproject_toml: workspace_pyproject_toml, + }) + } + + async fn collect_members_only( + workspace_root: &PathBuf, + workspace_definition: &ToolUvWorkspace, + workspace_pyproject_toml: &PyProjectToml, + options: &DiscoveryOptions, + ) -> Result, WorkspaceError> { let mut workspace_members = BTreeMap::new(); // Avoid reading a `pyproject.toml` more than once. let mut seen = FxHashSet::default(); // Add the project at the workspace root, if it exists and if it's distinct from the current - // project. - if current_project - .as_ref() - .map(|root_member| root_member.root != workspace_root) - .unwrap_or(true) - { - if let Some(project) = &workspace_pyproject_toml.project { - let pyproject_path = workspace_root.join("pyproject.toml"); - let contents = fs_err::read_to_string(&pyproject_path)?; - let pyproject_toml = PyProjectToml::from_string(contents) - .map_err(|err| WorkspaceError::Toml(pyproject_path.clone(), Box::new(err)))?; + // project. If it is the current project, it is added as such in the next step. + if let Some(project) = &workspace_pyproject_toml.project { + let pyproject_path = workspace_root.join("pyproject.toml"); + let contents = fs_err::read_to_string(&pyproject_path)?; + let pyproject_toml = PyProjectToml::from_string(contents) + .map_err(|err| WorkspaceError::Toml(pyproject_path.clone(), Box::new(err)))?; - debug!( - "Adding root workspace member: `{}`", - workspace_root.simplified_display() - ); - - seen.insert(workspace_root.clone()); - if let Some(existing) = workspace_members.insert( - project.name.clone(), - WorkspaceMember { - root: workspace_root.clone(), - project: project.clone(), - pyproject_toml, - }, - ) { - return Err(WorkspaceError::DuplicatePackage { - name: project.name.clone(), - first: existing.root.clone(), - second: workspace_root, - }); - } - }; - } - - // The current project is a workspace member, especially in a single project workspace. - if let Some(root_member) = current_project { debug!( - "Adding current workspace member: `{}`", - root_member.root.simplified_display() + "Adding root workspace member: `{}`", + workspace_root.simplified_display() ); - seen.insert(root_member.root.clone()); - workspace_members.insert(root_member.project.name.clone(), root_member); + seen.insert(workspace_root.clone()); + workspace_members.insert( + project.name.clone(), + WorkspaceMember { + root: workspace_root.clone(), + project: project.clone(), + pyproject_toml, + }, + ); } // Add all other workspace members. @@ -744,8 +822,7 @@ impl Workspace { } // If the member is excluded, ignore it. - if is_excluded_from_workspace(&member_root, &workspace_root, &workspace_definition)? - { + if is_excluded_from_workspace(&member_root, workspace_root, workspace_definition)? { debug!( "Ignoring workspace member: `{}`", member_root.simplified_display() @@ -842,7 +919,7 @@ impl Workspace { // Test for nested workspaces. for member in workspace_members.values() { - if member.root() != &workspace_root + if member.root() != workspace_root && member .pyproject_toml .tool @@ -854,34 +931,12 @@ impl Workspace { return Err(WorkspaceError::NestedWorkspace(member.root.clone())); } } - - let workspace_sources = workspace_pyproject_toml - .tool - .clone() - .and_then(|tool| tool.uv) - .and_then(|uv| uv.sources) - .map(ToolUvSources::into_inner) - .unwrap_or_default(); - - let workspace_indexes = workspace_pyproject_toml - .tool - .clone() - .and_then(|tool| tool.uv) - .and_then(|uv| uv.index) - .unwrap_or_default(); - - Ok(Workspace { - install_path: workspace_root, - packages: workspace_members, - sources: workspace_sources, - indexes: workspace_indexes, - pyproject_toml: workspace_pyproject_toml, - }) + Ok(workspace_members) } } /// A project in a workspace. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] #[cfg_attr(test, derive(serde::Serialize))] pub struct WorkspaceMember { /// The path to the project root. @@ -1006,7 +1061,8 @@ impl ProjectWorkspace { /// only directories between the current path and `stop_discovery_at` are considered. pub async fn discover( path: &Path, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result { let project_root = path .ancestors() @@ -1014,6 +1070,7 @@ impl ProjectWorkspace { // Only walk up the given directory, if any. options .stop_discovery_at + .as_deref() .and_then(Path::parent) .map(|stop_discovery_at| stop_discovery_at != *path) .unwrap_or(true) @@ -1026,13 +1083,14 @@ impl ProjectWorkspace { project_root.simplified_display() ); - Self::from_project_root(project_root, options).await + Self::from_project_root(project_root, options, cache).await } /// Discover the workspace starting from the directory containing the `pyproject.toml`. async fn from_project_root( project_root: &Path, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result { // Read the current `pyproject.toml`. let pyproject_path = project_root.join("pyproject.toml"); @@ -1046,14 +1104,15 @@ impl ProjectWorkspace { .clone() .ok_or(WorkspaceError::MissingProject(pyproject_path))?; - Self::from_project(project_root, &project, &pyproject_toml, options).await + Self::from_project(project_root, &project, &pyproject_toml, options, cache).await } /// If the current directory contains a `pyproject.toml` with a `project` table, discover the /// workspace and return it, otherwise it is a dynamic path dependency and we return `Ok(None)`. pub async fn from_maybe_project_root( install_path: &Path, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result, WorkspaceError> { // Read the `pyproject.toml`. let pyproject_path = install_path.join("pyproject.toml"); @@ -1070,7 +1129,7 @@ impl ProjectWorkspace { return Ok(None); }; - match Self::from_project(install_path, &project, &pyproject_toml, options).await { + match Self::from_project(install_path, &project, &pyproject_toml, options, cache).await { Ok(workspace) => Ok(Some(workspace)), Err(WorkspaceError::NonWorkspace(_)) => Ok(None), Err(err) => Err(err), @@ -1116,7 +1175,8 @@ impl ProjectWorkspace { install_path: &Path, project: &Project, project_pyproject_toml: &PyProjectToml, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result { let project_path = std::path::absolute(install_path) .map_err(WorkspaceError::Normalize)? @@ -1166,8 +1226,10 @@ impl ProjectWorkspace { // above it, so the project is an implicit workspace root identical to the project root. debug!("No workspace root found, using project root"); - let current_project_as_members = - BTreeMap::from_iter([(project.name.clone(), current_project)]); + let current_project_as_members = Arc::new(BTreeMap::from_iter([( + project.name.clone(), + current_project, + )])); return Ok(Self { project_root: project_path.clone(), project_name: project.name.clone(), @@ -1194,6 +1256,7 @@ impl ProjectWorkspace { workspace_pyproject_toml, Some(current_project), options, + cache, ) .await?; @@ -1208,7 +1271,7 @@ impl ProjectWorkspace { /// Find the workspace root above the current project, if any. async fn find_workspace( project_root: &Path, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, ) -> Result, WorkspaceError> { // Skip 1 to ignore the current project itself. for workspace_root in project_root @@ -1217,6 +1280,7 @@ async fn find_workspace( // Only walk up the given directory, if any. options .stop_discovery_at + .as_deref() .and_then(Path::parent) .map(|stop_discovery_at| stop_discovery_at != *path) .unwrap_or(true) @@ -1366,7 +1430,8 @@ impl VirtualProject { /// discovering the main workspace. pub async fn discover( path: &Path, - options: &DiscoveryOptions<'_>, + options: &DiscoveryOptions, + cache: &WorkspaceCache, ) -> Result { assert!( path.is_absolute(), @@ -1378,6 +1443,7 @@ impl VirtualProject { // Only walk up the given directory, if any. options .stop_discovery_at + .as_deref() .and_then(Path::parent) .map(|stop_discovery_at| stop_discovery_at != *path) .unwrap_or(true) @@ -1398,9 +1464,14 @@ impl VirtualProject { if let Some(project) = pyproject_toml.project.as_ref() { // If the `pyproject.toml` contains a `[project]` table, it's a project. - let project = - ProjectWorkspace::from_project(project_root, project, &pyproject_toml, options) - .await?; + let project = ProjectWorkspace::from_project( + project_root, + project, + &pyproject_toml, + options, + cache, + ) + .await?; Ok(Self::Project(project)) } else if let Some(workspace) = pyproject_toml .tool @@ -1420,6 +1491,7 @@ impl VirtualProject { pyproject_toml, None, options, + cache, ) .await?; @@ -1504,7 +1576,7 @@ mod tests { use crate::pyproject::PyProjectToml; use crate::workspace::{DiscoveryOptions, ProjectWorkspace}; - use crate::WorkspaceError; + use crate::{WorkspaceCache, WorkspaceError}; async fn workspace_test(folder: &str) -> (ProjectWorkspace, String) { let root_dir = env::current_dir() @@ -1515,10 +1587,13 @@ mod tests { .unwrap() .join("scripts") .join("workspaces"); - let project = - ProjectWorkspace::discover(&root_dir.join(folder), &DiscoveryOptions::default()) - .await - .unwrap(); + let project = ProjectWorkspace::discover( + &root_dir.join(folder), + &DiscoveryOptions::default(), + &WorkspaceCache::default(), + ) + .await + .unwrap(); let root_escaped = regex::escape(root_dir.to_string_lossy().as_ref()); (project, root_escaped) } @@ -1527,9 +1602,13 @@ mod tests { folder: &Path, ) -> Result<(ProjectWorkspace, String), (WorkspaceError, String)> { let root_escaped = regex::escape(folder.to_string_lossy().as_ref()); - let project = ProjectWorkspace::discover(folder, &DiscoveryOptions::default()) - .await - .map_err(|error| (error, root_escaped.clone()))?; + let project = ProjectWorkspace::discover( + folder, + &DiscoveryOptions::default(), + &WorkspaceCache::default(), + ) + .await + .map_err(|error| (error, root_escaped.clone()))?; Ok((project, root_escaped)) } @@ -1903,6 +1982,7 @@ mod tests { "###); }); } + #[tokio::test] async fn exclude_package() -> Result<()> { let root = tempfile::TempDir::new()?; diff --git a/crates/uv/src/commands/build_frontend.rs b/crates/uv/src/commands/build_frontend.rs index 01de12771..cda03204b 100644 --- a/crates/uv/src/commands/build_frontend.rs +++ b/crates/uv/src/commands/build_frontend.rs @@ -43,7 +43,7 @@ use uv_requirements::RequirementsSource; use uv_resolver::{ExcludeNewer, FlatIndex, RequiresPython}; use uv_settings::PythonInstallMirrors; use uv_types::{AnyErrorBuild, BuildContext, BuildIsolation, BuildStack, HashStrategy}; -use uv_workspace::{DiscoveryOptions, Workspace, WorkspaceError}; +use uv_workspace::{DiscoveryOptions, Workspace, WorkspaceCache, WorkspaceError}; #[derive(Debug, Error)] enum Error { @@ -241,7 +241,13 @@ async fn build_impl( }; // Attempt to discover the workspace; on failure, save the error for later. - let workspace = Workspace::discover(src.directory(), &DiscoveryOptions::default()).await; + let workspace_cache = WorkspaceCache::default(); + let workspace = Workspace::discover( + src.directory(), + &DiscoveryOptions::default(), + &workspace_cache, + ) + .await; // If a `--package` or `--all-packages` was provided, adjust the source directory. let packages = if let Some(package) = package { @@ -553,6 +559,7 @@ async fn build_package( // Initialize any shared state. let state = SharedState::default(); + let workspace_cache = WorkspaceCache::default(); // Create a build dispatch. let build_dispatch = BuildDispatch::new( @@ -572,6 +579,7 @@ async fn build_package( &hasher, exclude_newer, sources, + workspace_cache, concurrency, preview, ); diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index 512769bfe..823b601b8 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -40,6 +40,7 @@ use uv_resolver::{ }; use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy}; use uv_warnings::warn_user; +use uv_workspace::WorkspaceCache; use crate::commands::pip::loggers::DefaultResolveLogger; use crate::commands::pip::{operations, resolution_environment}; @@ -395,6 +396,7 @@ pub(crate) async fn pip_compile( &build_hashes, exclude_newer, sources, + WorkspaceCache::default(), concurrency, preview, ); diff --git a/crates/uv/src/commands/pip/install.rs b/crates/uv/src/commands/pip/install.rs index f9a599cae..dc5116558 100644 --- a/crates/uv/src/commands/pip/install.rs +++ b/crates/uv/src/commands/pip/install.rs @@ -34,6 +34,7 @@ use uv_resolver::{ ResolutionMode, ResolverEnvironment, }; use uv_types::{BuildIsolation, HashStrategy}; +use uv_workspace::WorkspaceCache; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger}; use crate::commands::pip::operations::Modifications; @@ -400,6 +401,7 @@ pub(crate) async fn pip_install( &build_hasher, exclude_newer, sources, + WorkspaceCache::default(), concurrency, preview, ); diff --git a/crates/uv/src/commands/pip/sync.rs b/crates/uv/src/commands/pip/sync.rs index 8c1225264..dfd0af29e 100644 --- a/crates/uv/src/commands/pip/sync.rs +++ b/crates/uv/src/commands/pip/sync.rs @@ -31,6 +31,7 @@ use uv_resolver::{ ResolutionMode, ResolverEnvironment, }; use uv_types::{BuildIsolation, HashStrategy}; +use uv_workspace::WorkspaceCache; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger}; use crate::commands::pip::operations::Modifications; @@ -333,6 +334,7 @@ pub(crate) async fn pip_sync( &build_hasher, exclude_newer, sources, + WorkspaceCache::default(), concurrency, preview, ); diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index c0f17e79b..3657bbcb4 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -37,7 +37,7 @@ use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user_once; use uv_workspace::pyproject::{DependencyType, Source, SourceError}; use uv_workspace::pyproject_mut::{ArrayEdit, DependencyTarget, PyProjectTomlMut}; -use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace, WorkspaceCache}; use crate::commands::pip::loggers::{ DefaultInstallLogger, DefaultResolveLogger, SummaryResolveLogger, @@ -173,15 +173,25 @@ pub(crate) async fn add( AddTarget::Script(script, Box::new(interpreter)) } else { // Find the project in the workspace. + // No workspace caching since `uv add` changes the workspace definition. let project = if let Some(package) = package { VirtualProject::Project( - Workspace::discover(project_dir, &DiscoveryOptions::default()) - .await? - .with_current_project(package.clone()) - .with_context(|| format!("Package `{package}` not found in workspace"))?, + Workspace::discover( + project_dir, + &DiscoveryOptions::default(), + &WorkspaceCache::default(), + ) + .await? + .with_current_project(package.clone()) + .with_context(|| format!("Package `{package}` not found in workspace"))?, ) } else { - VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await? + VirtualProject::discover( + project_dir, + &DiscoveryOptions::default(), + &WorkspaceCache::default(), + ) + .await? }; // For non-project workspace roots, allow dev dependencies, but nothing else. @@ -340,6 +350,8 @@ pub(crate) async fn add( &build_hasher, settings.exclude_newer, sources, + // No workspace caching since `uv add` changes the workspace definition. + WorkspaceCache::default(), concurrency, preview, ); diff --git a/crates/uv/src/commands/project/export.rs b/crates/uv/src/commands/project/export.rs index 13244f70d..f7d5e0bba 100644 --- a/crates/uv/src/commands/project/export.rs +++ b/crates/uv/src/commands/project/export.rs @@ -15,7 +15,7 @@ use uv_normalize::PackageName; use uv_python::{PythonDownloads, PythonPreference, PythonRequest}; use uv_resolver::RequirementsTxtExport; use uv_scripts::{Pep723ItemRef, Pep723Script}; -use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace, WorkspaceCache}; use crate::commands::pip::loggers::DefaultResolveLogger; use crate::commands::project::install_target::InstallTarget; @@ -79,6 +79,7 @@ pub(crate) async fn export( preview: PreviewMode, ) -> Result { // Identify the target. + let workspace_cache = WorkspaceCache::default(); let target = if let Some(script) = script { ExportTarget::Script(script) } else { @@ -89,17 +90,19 @@ pub(crate) async fn export( members: MemberDiscovery::None, ..DiscoveryOptions::default() }, + &workspace_cache, ) .await? } else if let Some(package) = package.as_ref() { VirtualProject::Project( - Workspace::discover(project_dir, &DiscoveryOptions::default()) + Workspace::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) .await? .with_current_project(package.clone()) .with_context(|| format!("Package `{package}` not found in workspace"))?, ) } else { - VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await? + VirtualProject::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await? }; ExportTarget::Project(project) }; diff --git a/crates/uv/src/commands/project/init.rs b/crates/uv/src/commands/project/init.rs index 855849db3..44b3efabd 100644 --- a/crates/uv/src/commands/project/init.rs +++ b/crates/uv/src/commands/project/init.rs @@ -26,7 +26,7 @@ use uv_scripts::{Pep723Script, ScriptTag}; use uv_settings::PythonInstallMirrors; use uv_warnings::warn_user_once; use uv_workspace::pyproject_mut::{DependencyTarget, PyProjectTomlMut}; -use uv_workspace::{DiscoveryOptions, MemberDiscovery, Workspace, WorkspaceError}; +use uv_workspace::{DiscoveryOptions, MemberDiscovery, Workspace, WorkspaceCache, WorkspaceError}; use crate::commands::project::{find_requires_python, init_script_python_requirement}; use crate::commands::reporters::PythonDownloadReporter; @@ -291,14 +291,16 @@ async fn init_project( printer: Printer, ) -> Result<()> { // Discover the current workspace, if it exists. + let workspace_cache = WorkspaceCache::default(); let workspace = { let parent = path.parent().expect("Project path has no parent"); match Workspace::discover( parent, &DiscoveryOptions { - members: MemberDiscovery::Ignore(std::iter::once(path).collect()), + members: MemberDiscovery::Ignore(std::iter::once(path.to_path_buf()).collect()), ..DiscoveryOptions::default() }, + &workspace_cache, ) .await { diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index 715ffb6e5..3e7090dc8 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -37,7 +37,7 @@ use uv_scripts::{Pep723ItemRef, Pep723Script}; use uv_settings::PythonInstallMirrors; use uv_types::{BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy}; use uv_warnings::{warn_user, warn_user_once}; -use uv_workspace::{DiscoveryOptions, Workspace, WorkspaceMember}; +use uv_workspace::{DiscoveryOptions, Workspace, WorkspaceCache, WorkspaceMember}; use crate::commands::pip::loggers::{DefaultResolveLogger, ResolveLogger, SummaryResolveLogger}; use crate::commands::project::lock_target::LockTarget; @@ -123,11 +123,14 @@ pub(crate) async fn lock( }; // Find the project requirements. + let workspace_cache = WorkspaceCache::default(); let workspace; let target = if let Some(script) = script.as_ref() { LockTarget::Script(script) } else { - workspace = Workspace::discover(project_dir, &DiscoveryOptions::default()).await?; + workspace = + Workspace::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await?; LockTarget::Workspace(&workspace) }; @@ -596,6 +599,8 @@ async fn do_lock( FlatIndex::from_entries(entries, None, &hasher, build_options) }; + let workspace_cache = WorkspaceCache::default(); + // Create a build dispatch. let build_dispatch = BuildDispatch::new( &client, @@ -614,6 +619,7 @@ async fn do_lock( &build_hasher, exclude_newer, sources, + workspace_cache, concurrency, preview, ); diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index ee12cc34b..dfa8cdc75 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -46,7 +46,7 @@ use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy}; use uv_warnings::{warn_user, warn_user_once}; use uv_workspace::dependency_groups::DependencyGroupError; use uv_workspace::pyproject::PyProjectToml; -use uv_workspace::Workspace; +use uv_workspace::{Workspace, WorkspaceCache}; use crate::commands::pip::loggers::{InstallLogger, ResolveLogger}; use crate::commands::pip::operations::{Changelog, Modifications}; @@ -1482,6 +1482,7 @@ pub(crate) async fn resolve_names( state: &SharedState, concurrency: Concurrency, cache: &Cache, + workspace_cache: &WorkspaceCache, printer: Printer, preview: PreviewMode, ) -> Result, uv_requirements::Error> { @@ -1583,6 +1584,7 @@ pub(crate) async fn resolve_names( &build_hasher, *exclude_newer, *sources, + workspace_cache.clone(), concurrency, preview, ); @@ -1754,6 +1756,8 @@ pub(crate) async fn resolve_environment( FlatIndex::from_entries(entries, Some(tags), &hasher, build_options) }; + let workspace_cache = WorkspaceCache::default(); + // Create a build dispatch. let resolve_dispatch = BuildDispatch::new( &client, @@ -1772,6 +1776,7 @@ pub(crate) async fn resolve_environment( &build_hasher, exclude_newer, sources, + workspace_cache, concurrency, preview, ); @@ -1883,6 +1888,7 @@ pub(crate) async fn sync_environment( let build_hasher = HashStrategy::default(); let dry_run = DryRun::default(); let hasher = HashStrategy::default(); + let workspace_cache = WorkspaceCache::default(); // Resolve the flat indexes from `--find-links`. let flat_index = { @@ -1911,6 +1917,7 @@ pub(crate) async fn sync_environment( &build_hasher, exclude_newer, sources, + workspace_cache, concurrency, preview, ); @@ -1976,6 +1983,7 @@ pub(crate) async fn update_environment( installer_metadata: bool, concurrency: Concurrency, cache: &Cache, + workspace_cache: WorkspaceCache, dry_run: DryRun, printer: Printer, preview: PreviewMode, @@ -2129,6 +2137,7 @@ pub(crate) async fn update_environment( &build_hasher, *exclude_newer, *sources, + workspace_cache, concurrency, preview, ); diff --git a/crates/uv/src/commands/project/remove.rs b/crates/uv/src/commands/project/remove.rs index e217b414f..19c6e1ec3 100644 --- a/crates/uv/src/commands/project/remove.rs +++ b/crates/uv/src/commands/project/remove.rs @@ -21,7 +21,7 @@ use uv_settings::PythonInstallMirrors; use uv_warnings::warn_user_once; use uv_workspace::pyproject::DependencyType; use uv_workspace::pyproject_mut::{DependencyTarget, PyProjectTomlMut}; -use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace, WorkspaceCache}; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger}; use crate::commands::pip::operations::Modifications; @@ -87,15 +87,25 @@ pub(crate) async fn remove( RemoveTarget::Script(script) } else { // Find the project in the workspace. + // No workspace caching since `uv remove` changes the workspace definition. let project = if let Some(package) = package { VirtualProject::Project( - Workspace::discover(project_dir, &DiscoveryOptions::default()) - .await? - .with_current_project(package.clone()) - .with_context(|| format!("Package `{package}` not found in workspace"))?, + Workspace::discover( + project_dir, + &DiscoveryOptions::default(), + &WorkspaceCache::default(), + ) + .await? + .with_current_project(package.clone()) + .with_context(|| format!("Package `{package}` not found in workspace"))?, ) } else { - VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await? + VirtualProject::discover( + project_dir, + &DiscoveryOptions::default(), + &WorkspaceCache::default(), + ) + .await? }; RemoveTarget::Project(project) diff --git a/crates/uv/src/commands/project/run.rs b/crates/uv/src/commands/project/run.rs index 5df9cbf68..7b5c38516 100644 --- a/crates/uv/src/commands/project/run.rs +++ b/crates/uv/src/commands/project/run.rs @@ -35,7 +35,7 @@ use uv_scripts::Pep723Item; use uv_settings::PythonInstallMirrors; use uv_static::EnvVars; use uv_warnings::warn_user; -use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace, WorkspaceError}; +use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace, WorkspaceCache, WorkspaceError}; use crate::commands::pip::loggers::{ DefaultInstallLogger, DefaultResolveLogger, SummaryInstallLogger, SummaryResolveLogger, @@ -142,6 +142,7 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl // Initialize any shared state. let lock_state = UniversalState::default(); let sync_state = lock_state.fork(); + let workspace_cache = WorkspaceCache::default(); // Read from the `.env` file, if necessary. if !no_env_file { @@ -369,6 +370,7 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl installer_metadata, concurrency, cache, + workspace_cache, DryRun::Disabled, printer, preview, @@ -425,6 +427,7 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl let mut lock: Option<(Lock, PathBuf)> = None; // Discover and sync the base environment. + let workspace_cache = WorkspaceCache::default(); let temp_dir; let base_interpreter = if let Some(script_interpreter) = script_interpreter { // If we found a PEP 723 script and the user provided a project-only setting, warn. @@ -466,13 +469,19 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl // We need a workspace, but we don't need to have a current package, we can be e.g. in // the root of a virtual workspace and then switch into the selected package. Some(VirtualProject::Project( - Workspace::discover(project_dir, &DiscoveryOptions::default()) + Workspace::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) .await? .with_current_project(package.clone()) .with_context(|| format!("Package `{package}` not found in workspace"))?, )) } else { - match VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await { + match VirtualProject::discover( + project_dir, + &DiscoveryOptions::default(), + &workspace_cache, + ) + .await + { Ok(project) => { if no_project { debug!("Ignoring discovered project due to `--no-project`"); diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index e62e34217..9e992e122 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -30,7 +30,7 @@ use uv_settings::PythonInstallMirrors; use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user; use uv_workspace::pyproject::Source; -use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace, WorkspaceCache}; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger}; use crate::commands::pip::operations; @@ -76,6 +76,7 @@ pub(crate) async fn sync( preview: PreviewMode, ) -> Result { // Identify the target. + let workspace_cache = WorkspaceCache::default(); let target = if let Some(script) = script { SyncTarget::Script(script) } else { @@ -87,17 +88,19 @@ pub(crate) async fn sync( members: MemberDiscovery::None, ..DiscoveryOptions::default() }, + &workspace_cache, ) .await? } else if let Some(package) = package.as_ref() { VirtualProject::Project( - Workspace::discover(project_dir, &DiscoveryOptions::default()) + Workspace::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) .await? .with_current_project(package.clone()) .with_context(|| format!("Package `{package}` not found in workspace"))?, ) } else { - VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await? + VirtualProject::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await? }; // TODO(lucab): improve warning content @@ -286,6 +289,7 @@ pub(crate) async fn sync( installer_metadata, concurrency, cache, + workspace_cache, dry_run, printer, preview, @@ -676,6 +680,7 @@ pub(super) async fn do_sync( &build_hasher, exclude_newer, sources, + WorkspaceCache::default(), concurrency, preview, ); diff --git a/crates/uv/src/commands/project/tree.rs b/crates/uv/src/commands/project/tree.rs index 582393e33..746ad8deb 100644 --- a/crates/uv/src/commands/project/tree.rs +++ b/crates/uv/src/commands/project/tree.rs @@ -15,7 +15,7 @@ use uv_python::{PythonDownloads, PythonPreference, PythonRequest, PythonVersion} use uv_resolver::{PackageMap, TreeDisplay}; use uv_scripts::{Pep723ItemRef, Pep723Script}; use uv_settings::PythonInstallMirrors; -use uv_workspace::{DiscoveryOptions, Workspace}; +use uv_workspace::{DiscoveryOptions, Workspace, WorkspaceCache}; use crate::commands::pip::latest::LatestClient; use crate::commands::pip::loggers::DefaultResolveLogger; @@ -60,11 +60,14 @@ pub(crate) async fn tree( preview: PreviewMode, ) -> Result { // Find the project requirements. + let workspace_cache = WorkspaceCache::default(); let workspace; let target = if let Some(script) = script.as_ref() { LockTarget::Script(script) } else { - workspace = Workspace::discover(project_dir, &DiscoveryOptions::default()).await?; + workspace = + Workspace::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await?; LockTarget::Workspace(&workspace) }; diff --git a/crates/uv/src/commands/python/find.rs b/crates/uv/src/commands/python/find.rs index 327c703d7..72951537b 100644 --- a/crates/uv/src/commands/python/find.rs +++ b/crates/uv/src/commands/python/find.rs @@ -6,7 +6,7 @@ use uv_cache::Cache; use uv_fs::Simplified; use uv_python::{EnvironmentPreference, PythonInstallation, PythonPreference, PythonRequest}; use uv_warnings::{warn_user, warn_user_once}; -use uv_workspace::{DiscoveryOptions, VirtualProject, WorkspaceError}; +use uv_workspace::{DiscoveryOptions, VirtualProject, WorkspaceCache, WorkspaceError}; use crate::commands::{ project::{validate_project_requires_python, WorkspacePython}, @@ -29,10 +29,13 @@ pub(crate) async fn find( EnvironmentPreference::Any }; + let workspace_cache = WorkspaceCache::default(); let project = if no_project { None } else { - match VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await { + match VirtualProject::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await + { Ok(project) => Some(project), Err(WorkspaceError::MissingProject(_)) => None, Err(WorkspaceError::MissingPyprojectToml) => None, diff --git a/crates/uv/src/commands/python/pin.rs b/crates/uv/src/commands/python/pin.rs index 1c07b24d0..5d728e2f6 100644 --- a/crates/uv/src/commands/python/pin.rs +++ b/crates/uv/src/commands/python/pin.rs @@ -13,7 +13,7 @@ use uv_python::{ VersionFileDiscoveryOptions, PYTHON_VERSION_FILENAME, }; use uv_warnings::warn_user_once; -use uv_workspace::{DiscoveryOptions, VirtualProject}; +use uv_workspace::{DiscoveryOptions, VirtualProject, WorkspaceCache}; use crate::commands::{project::find_requires_python, ExitStatus}; use crate::printer::Printer; @@ -28,10 +28,13 @@ pub(crate) async fn pin( cache: &Cache, printer: Printer, ) -> Result { + let workspace_cache = WorkspaceCache::default(); let virtual_project = if no_project { None } else { - match VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await { + match VirtualProject::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await + { Ok(virtual_project) => Some(virtual_project), Err(err) => { debug!("Failed to discover virtual project: {err}"); diff --git a/crates/uv/src/commands/tool/install.rs b/crates/uv/src/commands/tool/install.rs index 4b94fed10..b480a47a0 100644 --- a/crates/uv/src/commands/tool/install.rs +++ b/crates/uv/src/commands/tool/install.rs @@ -21,9 +21,9 @@ use uv_requirements::{RequirementsSource, RequirementsSpecification}; use uv_settings::{PythonInstallMirrors, ResolverInstallerOptions, ToolOptions}; use uv_tool::InstalledTools; use uv_warnings::warn_user; +use uv_workspace::WorkspaceCache; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger}; - use crate::commands::pip::operations::Modifications; use crate::commands::project::{ resolve_environment, resolve_names, sync_environment, update_environment, @@ -86,6 +86,7 @@ pub(crate) async fn install( // Initialize any shared state. let state = PlatformState::default(); + let workspace_cache = WorkspaceCache::default(); let client_builder = BaseClientBuilder::new() .connectivity(network_settings.connectivity) @@ -133,6 +134,7 @@ pub(crate) async fn install( &state, concurrency, &cache, + &workspace_cache, printer, preview, ) @@ -245,6 +247,7 @@ pub(crate) async fn install( &state, concurrency, &cache, + &workspace_cache, printer, preview, ) @@ -269,6 +272,7 @@ pub(crate) async fn install( &state, concurrency, &cache, + &workspace_cache, printer, preview, ) @@ -400,6 +404,7 @@ pub(crate) async fn install( installer_metadata, concurrency, &cache, + workspace_cache, DryRun::Disabled, printer, preview, diff --git a/crates/uv/src/commands/tool/run.rs b/crates/uv/src/commands/tool/run.rs index d886487d9..720321bbf 100644 --- a/crates/uv/src/commands/tool/run.rs +++ b/crates/uv/src/commands/tool/run.rs @@ -37,6 +37,7 @@ use uv_settings::{PythonInstallMirrors, ResolverInstallerOptions, ToolOptions}; use uv_static::EnvVars; use uv_tool::{entrypoint_paths, InstalledTools}; use uv_warnings::warn_user; +use uv_workspace::WorkspaceCache; use crate::commands::pip::loggers::{ DefaultInstallLogger, DefaultResolveLogger, SummaryInstallLogger, SummaryResolveLogger, @@ -625,6 +626,7 @@ async fn get_or_create_environment( // Initialize any shared state. let state = PlatformState::default(); + let workspace_cache = WorkspaceCache::default(); let from = if request.is_python() { ToolRequirement::Python @@ -668,6 +670,7 @@ async fn get_or_create_environment( &state, concurrency, cache, + &workspace_cache, printer, preview, ) @@ -760,6 +763,7 @@ async fn get_or_create_environment( &state, concurrency, cache, + &workspace_cache, printer, preview, ) @@ -785,6 +789,7 @@ async fn get_or_create_environment( &state, concurrency, cache, + &workspace_cache, printer, preview, ) diff --git a/crates/uv/src/commands/tool/upgrade.rs b/crates/uv/src/commands/tool/upgrade.rs index cbf785069..5979e48d1 100644 --- a/crates/uv/src/commands/tool/upgrade.rs +++ b/crates/uv/src/commands/tool/upgrade.rs @@ -18,6 +18,7 @@ use uv_python::{ use uv_requirements::RequirementsSpecification; use uv_settings::{Combine, PythonInstallMirrors, ResolverInstallerOptions, ToolOptions}; use uv_tool::InstalledTools; +use uv_workspace::WorkspaceCache; use crate::commands::pip::loggers::{ DefaultInstallLogger, SummaryResolveLogger, UpgradeInstallLogger, @@ -281,6 +282,7 @@ async fn upgrade_tool( // Initialize any shared state. let state = PlatformState::default(); + let workspace_cache = WorkspaceCache::default(); // Check if we need to create a new environment — if so, resolve it first, then // install the requested tool @@ -340,6 +342,7 @@ async fn upgrade_tool( installer_metadata, concurrency, cache, + workspace_cache, DryRun::Disabled, printer, preview, diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index e9928a899..21270fe26 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -29,7 +29,7 @@ use uv_settings::PythonInstallMirrors; use uv_shell::{shlex_posix, shlex_windows, Shell}; use uv_types::{AnyErrorBuild, BuildContext, BuildIsolation, BuildStack, HashStrategy}; use uv_warnings::warn_user; -use uv_workspace::{DiscoveryOptions, VirtualProject, WorkspaceError}; +use uv_workspace::{DiscoveryOptions, VirtualProject, WorkspaceCache, WorkspaceError}; use crate::commands::pip::loggers::{DefaultInstallLogger, InstallLogger}; use crate::commands::pip::operations::{report_interpreter, Changelog}; @@ -150,10 +150,13 @@ async fn venv_impl( relocatable: bool, preview: PreviewMode, ) -> miette::Result { + let workspace_cache = WorkspaceCache::default(); let project = if no_project { None } else { - match VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await { + match VirtualProject::discover(project_dir, &DiscoveryOptions::default(), &workspace_cache) + .await + { Ok(project) => Some(project), Err(WorkspaceError::MissingProject(_)) => None, Err(WorkspaceError::MissingPyprojectToml) => None, @@ -318,6 +321,7 @@ async fn venv_impl( // Initialize any shared state. let state = SharedState::default(); + let workspace_cache = WorkspaceCache::default(); // For seed packages, assume a bunch of default settings are sufficient. let build_constraints = Constraints::default(); @@ -346,6 +350,7 @@ async fn venv_impl( &build_hasher, exclude_newer, sources, + workspace_cache, concurrency, preview, ); diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 363136f51..ccb0d8556 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -31,7 +31,7 @@ use uv_scripts::{Pep723Error, Pep723Item, Pep723Metadata, Pep723Script}; use uv_settings::{Combine, FilesystemOptions, Options}; use uv_static::EnvVars; use uv_warnings::{warn_user, warn_user_once}; -use uv_workspace::{DiscoveryOptions, Workspace}; +use uv_workspace::{DiscoveryOptions, Workspace, WorkspaceCache}; use crate::commands::{ExitStatus, RunCommand, ScriptPath, ToolRunCommand}; use crate::printer::Printer; @@ -109,6 +109,7 @@ async fn run(mut cli: Cli) -> Result { // If found, this file is combined with the user configuration file. // 3. The nearest configuration file (`uv.toml` or `pyproject.toml`) in the directory tree, // starting from the current directory. + let workspace_cache = WorkspaceCache::default(); let filesystem = if let Some(config_file) = cli.top_level.config_file.as_ref() { if config_file .file_name() @@ -123,7 +124,7 @@ async fn run(mut cli: Cli) -> Result { // For commands that operate at the user-level, ignore local configuration. FilesystemOptions::user()?.combine(FilesystemOptions::system()?) } else if let Ok(workspace) = - Workspace::discover(&project_dir, &DiscoveryOptions::default()).await + Workspace::discover(&project_dir, &DiscoveryOptions::default(), &workspace_cache).await { let project = FilesystemOptions::find(workspace.install_path())?; let system = FilesystemOptions::system()?; diff --git a/crates/uv/tests/it/lock.rs b/crates/uv/tests/it/lock.rs index 12af8d57e..3574d000a 100644 --- a/crates/uv/tests/it/lock.rs +++ b/crates/uv/tests/it/lock.rs @@ -15194,7 +15194,7 @@ fn lock_explicit_default_index() -> Result<()> { "#, )?; - uv_snapshot!(context.filters(), context.lock().arg("--verbose"), @r###" + uv_snapshot!(context.filters(), context.lock().arg("--verbose"), @r#" success: false exit_code: 1 ----- stdout ----- @@ -15202,7 +15202,7 @@ fn lock_explicit_default_index() -> Result<()> { ----- stderr ----- DEBUG uv [VERSION] ([COMMIT] DATE) DEBUG Found workspace root: `[TEMP_DIR]/` - DEBUG Adding current workspace member: `[TEMP_DIR]/` + DEBUG Adding root workspace member: `[TEMP_DIR]/` DEBUG Using Python request `>=3.12` from `requires-python` metadata DEBUG Checking for Python environment at `.venv` DEBUG The virtual environment's Python version satisfies `>=3.12` @@ -15226,7 +15226,7 @@ fn lock_explicit_default_index() -> Result<()> { ╰─▶ Because anyio was not found in the provided package locations and your project depends on anyio, we can conclude that your project's requirements are unsatisfiable. hint: Packages were unavailable because index lookups were disabled and no additional package locations were provided (try: `--find-links `) - "###); + "#); let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap();