From b12d5b619b4520b314e9b61330df6ec800662727 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 12 Oct 2024 16:46:55 +0200 Subject: [PATCH] Use shared index when fetching metadata in lock satisfaction routine (#8147) --- crates/uv-resolver/src/lock/mod.rs | 70 +++++++++++++++++++------- crates/uv/src/commands/project/lock.rs | 10 +++- crates/uv/tests/it/lock.rs | 2 +- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index c48480ff7..1c4de1cd0 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -10,7 +10,7 @@ use std::fmt::{Debug, Display}; use std::io; use std::path::{Path, PathBuf}; use std::str::FromStr; -use std::sync::LazyLock; +use std::sync::{Arc, LazyLock}; use toml_edit::{value, Array, ArrayOfTables, InlineTable, Item, Table, Value}; use url::Url; @@ -20,10 +20,10 @@ use uv_distribution::DistributionDatabase; use uv_distribution_filename::{DistExtension, ExtensionError, SourceDistExtension, WheelFilename}; use uv_distribution_types::{ BuiltDist, DependencyMetadata, DirectUrlBuiltDist, DirectUrlSourceDist, DirectorySourceDist, - Dist, DistributionMetadata, FileLocation, FlatIndexLocation, GitSourceDist, HashPolicy, - IndexLocations, IndexUrl, Name, PathBuiltDist, PathSourceDist, RegistryBuiltDist, - RegistryBuiltWheel, RegistrySourceDist, RemoteSource, Resolution, ResolvedDist, StaticMetadata, - ToUrlError, UrlString, + Dist, DistributionMetadata, FileLocation, FlatIndexLocation, GitSourceDist, IndexLocations, + IndexUrl, Name, PathBuiltDist, PathSourceDist, RegistryBuiltDist, RegistryBuiltWheel, + RegistrySourceDist, RemoteSource, Resolution, ResolvedDist, StaticMetadata, ToUrlError, + UrlString, }; use uv_fs::{relative_to, PortablePath, PortablePathBuf}; use uv_git::{GitReference, GitSha, RepositoryReference, ResolvedRepositoryReference}; @@ -35,14 +35,17 @@ use uv_pypi_types::{ redact_git_credentials, HashDigest, ParsedArchiveUrl, ParsedGitUrl, Requirement, RequirementSource, ResolverMarkerEnvironment, }; -use uv_types::BuildContext; +use uv_types::{BuildContext, HashStrategy}; use uv_workspace::{InstallTarget, Workspace}; pub use crate::lock::requirements_txt::RequirementsTxtExport; pub use crate::lock::tree::TreeDisplay; use crate::requires_python::SimplifiedMarkerTree; use crate::resolution::{AnnotatedDist, ResolutionGraphNode}; -use crate::{ExcludeNewer, PrereleaseMode, RequiresPython, ResolutionGraph, ResolutionMode}; +use crate::{ + ExcludeNewer, InMemoryIndex, MetadataResponse, PrereleaseMode, RequiresPython, ResolutionGraph, + ResolutionMode, +}; mod requirements_txt; mod tree; @@ -927,6 +930,8 @@ impl Lock { indexes: Option<&IndexLocations>, build_options: &BuildOptions, tags: &Tags, + hasher: &HashStrategy, + index: &InMemoryIndex, database: &DistributionDatabase<'_, Context>, ) -> Result, LockError> { let mut queue: VecDeque<&Package> = VecDeque::new(); @@ -1158,18 +1163,48 @@ impl Lock { build_options, )?; - let archive = database - .get_or_build_wheel_metadata(&dist, HashPolicy::None) - .await - .map_err(|err| LockErrorKind::Resolution { - id: package.id.clone(), - err, - })?; + // Fetch the metadata for the distribution. + let metadata = { + let id = dist.version_id(); + if let Some(archive) = + index + .distributions() + .get(&id) + .as_deref() + .and_then(|response| { + if let MetadataResponse::Found(archive, ..) = response { + Some(archive) + } else { + None + } + }) + { + // If the metadata is already in the index, return it. + archive.metadata.clone() + } else { + // Run the PEP 517 build process to extract metadata from the source distribution. + let archive = database + .get_or_build_wheel_metadata(&dist, hasher.get(&dist)) + .await + .map_err(|err| LockErrorKind::Resolution { + id: package.id.clone(), + err, + })?; + + let metadata = archive.metadata.clone(); + + // Insert the metadata into the index. + index + .distributions() + .done(id, Arc::new(MetadataResponse::Found(archive))); + + metadata + } + }; // Validate the `requires-dist` metadata. { - let expected: BTreeSet<_> = archive - .metadata + let expected: BTreeSet<_> = metadata .requires_dist .into_iter() .map(|requirement| normalize_requirement(requirement, workspace)) @@ -1194,8 +1229,7 @@ impl Lock { // Validate the `dev-dependencies` metadata. { - let expected: BTreeMap> = archive - .metadata + let expected: BTreeMap> = metadata .dev_dependencies .into_iter() .map(|(group, requirements)| { diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index aa56cff36..3683efcd9 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -28,8 +28,8 @@ use uv_python::{Interpreter, PythonDownloads, PythonEnvironment, PythonPreferenc use uv_requirements::upgrade::{read_lock_requirements, LockedRequirements}; use uv_requirements::ExtrasResolver; use uv_resolver::{ - FlatIndex, Lock, Options, OptionsBuilder, PythonRequirement, RequiresPython, ResolverManifest, - ResolverMarkers, SatisfiesResult, + FlatIndex, InMemoryIndex, Lock, Options, OptionsBuilder, PythonRequirement, RequiresPython, + ResolverManifest, ResolverMarkers, SatisfiesResult, }; use uv_types::{BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy}; use uv_warnings::{warn_user, warn_user_once}; @@ -462,6 +462,8 @@ async fn do_lock( build_options, upgrade, &options, + &hasher, + &state.index, &database, printer, ) @@ -643,6 +645,8 @@ impl ValidatedLock { build_options: &BuildOptions, upgrade: &Upgrade, options: &Options, + hasher: &HashStrategy, + index: &InMemoryIndex, database: &DistributionDatabase<'_, Context>, printer: Printer, ) -> Result { @@ -767,6 +771,8 @@ impl ValidatedLock { indexes, build_options, interpreter.tags()?, + hasher, + index, database, ) .await? diff --git a/crates/uv/tests/it/lock.rs b/crates/uv/tests/it/lock.rs index 93b583818..3a26fd424 100644 --- a/crates/uv/tests/it/lock.rs +++ b/crates/uv/tests/it/lock.rs @@ -13411,7 +13411,7 @@ fn lock_strip_fragment() -> Result<()> { ----- stdout ----- ----- stderr ----- - Prepared 2 packages in [TIME] + Prepared 1 package in [TIME] Installed 2 packages in [TIME] + iniconfig==2.0.0 (from https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl) + project==0.1.0 (from file://[TEMP_DIR]/)