Refactor `Resolution` type to retain dependency graph (#9106)

## Summary

This PR should not contain any user-visible changes, but the goal is to
refactor the `Resolution` type to retain a dependency graph. We want to
be able to explain _why_ a given package was excluded on error (see:
https://github.com/astral-sh/uv/issues/8962), which in turn requires
that at install time, we can go back and figure out the dependency
chain. At present, `Resolution` is just a map from package name to
distribution; this PR remodels it as a graph in which each node is a
package, and the edges contain markers plus extras or dependency groups.
This commit is contained in:
Charlie Marsh 2024-11-14 15:25:34 -05:00 committed by GitHub
parent ed130b0c11
commit a552f74308
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 432 additions and 182 deletions

1
Cargo.lock generated
View File

@ -4884,6 +4884,7 @@ dependencies = [
"fs-err", "fs-err",
"itertools 0.13.0", "itertools 0.13.0",
"jiff", "jiff",
"petgraph",
"rkyv", "rkyv",
"rustc-hash", "rustc-hash",
"schemars", "schemars",

View File

@ -33,6 +33,7 @@ bitflags = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
jiff = { workspace = true } jiff = { workspace = true }
petgraph = { workspace = true }
rkyv = { workspace = true } rkyv = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
schemars = { workspace = true, optional = true } schemars = { workspace = true, optional = true }

View File

@ -1,55 +1,76 @@
use std::collections::BTreeMap;
use uv_distribution_filename::DistExtension; use uv_distribution_filename::DistExtension;
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep508::MarkerTree;
use uv_pypi_types::{HashDigest, RequirementSource}; use uv_pypi_types::{HashDigest, RequirementSource};
use crate::{BuiltDist, Diagnostic, Dist, Name, ResolvedDist, SourceDist}; use crate::{BuiltDist, Diagnostic, Dist, Name, ResolvedDist, SourceDist};
/// A set of packages pinned at specific versions. /// A set of packages pinned at specific versions.
///
/// This is similar to [`ResolverOutput`], but represents a resolution for a subset of all
/// marker environments. For example, the resolution is guaranteed to contain at most one version
/// for a given package.
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct Resolution { pub struct Resolution {
packages: BTreeMap<PackageName, ResolvedDist>, graph: petgraph::graph::DiGraph<Node, Edge>,
hashes: BTreeMap<PackageName, Vec<HashDigest>>,
diagnostics: Vec<ResolutionDiagnostic>, diagnostics: Vec<ResolutionDiagnostic>,
} }
impl Resolution { impl Resolution {
/// Create a new resolution from the given pinned packages. /// Create a new resolution from the given pinned packages.
pub fn new( pub fn new(graph: petgraph::graph::DiGraph<Node, Edge>) -> Self {
packages: BTreeMap<PackageName, ResolvedDist>,
hashes: BTreeMap<PackageName, Vec<HashDigest>>,
diagnostics: Vec<ResolutionDiagnostic>,
) -> Self {
Self { Self {
packages, graph,
hashes, diagnostics: Vec::new(),
diagnostics,
} }
} }
/// Return the hashes for the given package name, if they exist. /// Return the underlying graph of the resolution.
pub fn get_hashes(&self, package_name: &PackageName) -> &[HashDigest] { pub fn graph(&self) -> &petgraph::graph::DiGraph<Node, Edge> {
self.hashes.get(package_name).map_or(&[], Vec::as_slice) &self.graph
} }
/// Iterate over the [`PackageName`] entities in this resolution. /// Add [`Diagnostics`] to the resolution.
pub fn packages(&self) -> impl Iterator<Item = &PackageName> { #[must_use]
self.packages.keys() pub fn with_diagnostics(mut self, diagnostics: Vec<ResolutionDiagnostic>) -> Self {
self.diagnostics.extend(diagnostics);
self
}
/// Return the hashes for the given package name, if they exist.
pub fn hashes(&self) -> impl Iterator<Item = (&ResolvedDist, &[HashDigest])> {
self.graph
.node_indices()
.filter_map(move |node| match &self.graph[node] {
Node::Dist {
dist,
hashes,
install,
..
} if *install => Some((dist, hashes.as_slice())),
_ => None,
})
} }
/// Iterate over the [`ResolvedDist`] entities in this resolution. /// Iterate over the [`ResolvedDist`] entities in this resolution.
pub fn distributions(&self) -> impl Iterator<Item = &ResolvedDist> { pub fn distributions(&self) -> impl Iterator<Item = &ResolvedDist> {
self.packages.values() self.graph
.raw_nodes()
.iter()
.filter_map(|node| match &node.weight {
Node::Dist { dist, install, .. } if *install => Some(dist),
_ => None,
})
} }
/// Return the number of distributions in this resolution. /// Return the number of distributions in this resolution.
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.packages.len() self.distributions().count()
} }
/// Return `true` if there are no pinned packages in this resolution. /// Return `true` if there are no pinned packages in this resolution.
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.packages.is_empty() self.distributions().next().is_none()
} }
/// Return the [`ResolutionDiagnostic`]s that were produced during resolution. /// Return the [`ResolutionDiagnostic`]s that were produced during resolution.
@ -59,44 +80,31 @@ impl Resolution {
/// Filter the resolution to only include packages that match the given predicate. /// Filter the resolution to only include packages that match the given predicate.
#[must_use] #[must_use]
pub fn filter(self, predicate: impl Fn(&ResolvedDist) -> bool) -> Self { pub fn filter(mut self, predicate: impl Fn(&ResolvedDist) -> bool) -> Self {
let packages = self for node in self.graph.node_weights_mut() {
.packages if let Node::Dist { dist, install, .. } = node {
.into_iter() if !predicate(dist) {
.filter(|(_, dist)| predicate(dist)) *install = false;
.collect::<BTreeMap<_, _>>(); }
let hashes = self }
.hashes
.into_iter()
.filter(|(name, _)| packages.contains_key(name))
.collect();
let diagnostics = self.diagnostics.clone();
Self {
packages,
hashes,
diagnostics,
} }
self
} }
/// Map over the resolved distributions in this resolution. /// Map over the resolved distributions in this resolution.
///
/// For efficiency, the map function should return `None` if the resolved distribution is
/// unchanged.
#[must_use] #[must_use]
pub fn map(self, predicate: impl Fn(ResolvedDist) -> ResolvedDist) -> Self { pub fn map(mut self, predicate: impl Fn(&ResolvedDist) -> Option<ResolvedDist>) -> Self {
let packages = self for node in self.graph.node_weights_mut() {
.packages if let Node::Dist { dist, .. } = node {
.into_iter() if let Some(transformed) = predicate(dist) {
.map(|(name, dist)| (name, predicate(dist))) *dist = transformed;
.collect::<BTreeMap<_, _>>(); }
let hashes = self }
.hashes
.into_iter()
.filter(|(name, _)| packages.contains_key(name))
.collect();
let diagnostics = self.diagnostics.clone();
Self {
packages,
hashes,
diagnostics,
} }
self
} }
} }
@ -166,15 +174,46 @@ impl Diagnostic for ResolutionDiagnostic {
} }
} }
/// A node in the resolution, along with whether its been filtered out.
///
/// We retain filtered nodes as we still need to be able to trace dependencies through the graph
/// (e.g., to determine why a package was included in the resolution).
#[derive(Debug, Clone)]
pub enum Node {
Root,
Dist {
dist: ResolvedDist,
hashes: Vec<HashDigest>,
install: bool,
},
}
/// An edge in the resolution graph, along with the marker that must be satisfied to traverse it.
#[derive(Debug, Clone)]
pub enum Edge {
Prod(MarkerTree),
Optional(ExtraName, MarkerTree),
Dev(GroupName, MarkerTree),
}
impl Edge {
/// Return the [`MarkerTree`] for this edge.
pub fn marker(&self) -> &MarkerTree {
match self {
Self::Prod(marker) => marker,
Self::Optional(_, marker) => marker,
Self::Dev(_, marker) => marker,
}
}
}
impl From<&ResolvedDist> for RequirementSource { impl From<&ResolvedDist> for RequirementSource {
fn from(resolved_dist: &ResolvedDist) -> Self { fn from(resolved_dist: &ResolvedDist) -> Self {
match resolved_dist { match resolved_dist {
ResolvedDist::Installable { dist, .. } => match dist { ResolvedDist::Installable { dist, version } => match dist {
Dist::Built(BuiltDist::Registry(wheels)) => RequirementSource::Registry { Dist::Built(BuiltDist::Registry(wheels)) => RequirementSource::Registry {
specifier: uv_pep440::VersionSpecifiers::from( specifier: uv_pep440::VersionSpecifiers::from(
uv_pep440::VersionSpecifier::equals_version( uv_pep440::VersionSpecifier::equals_version(version.clone()),
wheels.best_wheel().filename.version.clone(),
),
), ),
index: Some(wheels.best_wheel().index.url().clone()), index: Some(wheels.best_wheel().index.url().clone()),
}, },

View File

@ -6,7 +6,7 @@ use std::path::{Component, Path, PathBuf};
use either::Either; use either::Either;
use petgraph::visit::IntoNodeReferences; use petgraph::visit::IntoNodeReferences;
use petgraph::{Directed, Graph}; use petgraph::Graph;
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use url::Url; use url::Url;
@ -22,8 +22,6 @@ use crate::graph_ops::marker_reachability;
use crate::lock::{Package, PackageId, Source}; use crate::lock::{Package, PackageId, Source};
use crate::{InstallTarget, LockError}; use crate::{InstallTarget, LockError};
type LockGraph<'lock> = Graph<Node<'lock>, Edge, Directed>;
/// An export of a [`Lock`] that renders in `requirements.txt` format. /// An export of a [`Lock`] that renders in `requirements.txt` format.
#[derive(Debug)] #[derive(Debug)]
pub struct RequirementsTxtExport<'lock> { pub struct RequirementsTxtExport<'lock> {
@ -42,7 +40,7 @@ impl<'lock> RequirementsTxtExport<'lock> {
install_options: &'lock InstallOptions, install_options: &'lock InstallOptions,
) -> Result<Self, LockError> { ) -> Result<Self, LockError> {
let size_guess = target.lock().packages.len(); let size_guess = target.lock().packages.len();
let mut petgraph = LockGraph::with_capacity(size_guess, size_guess); let mut petgraph = Graph::with_capacity(size_guess, size_guess);
let mut inverse = FxHashMap::with_capacity_and_hasher(size_guess, FxBuildHasher); let mut inverse = FxHashMap::with_capacity_and_hasher(size_guess, FxBuildHasher);
let mut queue: VecDeque<(&Package, Option<&ExtraName>)> = VecDeque::new(); let mut queue: VecDeque<(&Package, Option<&ExtraName>)> = VecDeque::new();
@ -282,16 +280,13 @@ impl std::fmt::Display for RequirementsTxtExport<'_> {
} }
} }
/// A node in the [`LockGraph`]. /// A node in the graph.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
enum Node<'lock> { enum Node<'lock> {
Root, Root,
Package(&'lock Package), Package(&'lock Package),
} }
/// The edges of the [`LockGraph`].
type Edge = MarkerTree;
/// A flat requirement, with its associated marker. /// A flat requirement, with its associated marker.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
struct Requirement<'lock> { struct Requirement<'lock> {

View File

@ -1,11 +1,13 @@
use either::Either;
use petgraph::Graph;
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use std::collections::hash_map::Entry;
use std::collections::{BTreeMap, VecDeque}; use std::collections::{BTreeMap, VecDeque};
use either::Either;
use rustc_hash::FxHashSet;
use uv_configuration::{BuildOptions, DevGroupsManifest, ExtrasSpecification, InstallOptions}; use uv_configuration::{BuildOptions, DevGroupsManifest, ExtrasSpecification, InstallOptions};
use uv_distribution_types::{Resolution, ResolvedDist}; use uv_distribution_types::{Edge, Node, Resolution, ResolvedDist};
use uv_normalize::{ExtraName, GroupName, PackageName, DEV_DEPENDENCIES}; use uv_normalize::{ExtraName, GroupName, PackageName, DEV_DEPENDENCIES};
use uv_pep508::MarkerTree;
use uv_platform_tags::Tags; use uv_platform_tags::Tags;
use uv_pypi_types::{ResolverMarkerEnvironment, VerbatimParsedUrl}; use uv_pypi_types::{ResolverMarkerEnvironment, VerbatimParsedUrl};
use uv_workspace::dependency_groups::{DependencyGroupError, FlatDependencyGroups}; use uv_workspace::dependency_groups::{DependencyGroupError, FlatDependencyGroups};
@ -155,12 +157,18 @@ impl<'env> InstallTarget<'env> {
build_options: &BuildOptions, build_options: &BuildOptions,
install_options: &InstallOptions, install_options: &InstallOptions,
) -> Result<Resolution, LockError> { ) -> Result<Resolution, LockError> {
let size_guess = self.lock().packages.len();
let mut petgraph = Graph::with_capacity(size_guess, size_guess);
let mut inverse = FxHashMap::with_capacity_and_hasher(size_guess, FxBuildHasher);
let mut queue: VecDeque<(&Package, Option<&ExtraName>)> = VecDeque::new(); let mut queue: VecDeque<(&Package, Option<&ExtraName>)> = VecDeque::new();
let mut seen = FxHashSet::default(); let mut seen = FxHashSet::default();
let root = petgraph.add_node(Node::Root);
// Add the workspace packages to the queue. // Add the workspace packages to the queue.
for root_name in self.packages() { for root_name in self.packages() {
let root = self let dist = self
.lock() .lock()
.find_by_name(root_name) .find_by_name(root_name)
.map_err(|_| LockErrorKind::MultipleRootPackages { .map_err(|_| LockErrorKind::MultipleRootPackages {
@ -170,21 +178,30 @@ impl<'env> InstallTarget<'env> {
name: root_name.clone(), name: root_name.clone(),
})?; })?;
if dev.prod() { // Add the workspace package to the graph.
// Add the base package. let index = petgraph.add_node(if dev.prod() {
queue.push_back((root, None)); self.package_to_node(dist, tags, build_options, install_options)?
} else {
self.non_installable_node(dist, tags)?
});
inverse.insert(&dist.id, index);
// Add any extras. // Add an edge from the root.
petgraph.add_edge(root, index, Edge::Prod(MarkerTree::TRUE));
if dev.prod() {
// Push its dependencies on the queue.
queue.push_back((dist, None));
match extras { match extras {
ExtrasSpecification::None => {} ExtrasSpecification::None => {}
ExtrasSpecification::All => { ExtrasSpecification::All => {
for extra in root.optional_dependencies.keys() { for extra in dist.optional_dependencies.keys() {
queue.push_back((root, Some(extra))); queue.push_back((dist, Some(extra)));
} }
} }
ExtrasSpecification::Some(extras) => { ExtrasSpecification::Some(extras) => {
for extra in extras { for extra in extras {
queue.push_back((root, Some(extra))); queue.push_back((dist, Some(extra)));
} }
} }
} }
@ -192,16 +209,44 @@ impl<'env> InstallTarget<'env> {
// Add any dev dependencies. // Add any dev dependencies.
for group in dev.iter() { for group in dev.iter() {
for dep in root.dependency_groups.get(group).into_iter().flatten() { for dep in dist.dependency_groups.get(group).into_iter().flatten() {
if dep.complexified_marker.evaluate(marker_env, &[]) { if !dep.complexified_marker.evaluate(marker_env, &[]) {
let dep_dist = self.lock().find_by_id(&dep.package_id); continue;
if seen.insert((&dep.package_id, None)) { }
queue.push_back((dep_dist, None));
let dep_dist = self.lock().find_by_id(&dep.package_id);
// Add the dependency to the graph.
let dep_index = match inverse.entry(&dep.package_id) {
Entry::Vacant(entry) => {
let index = petgraph.add_node(self.package_to_node(
dep_dist,
tags,
build_options,
install_options,
)?);
entry.insert(index);
index
} }
for extra in &dep.extra { Entry::Occupied(entry) => {
if seen.insert((&dep.package_id, Some(extra))) { let index = *entry.get();
queue.push_back((dep_dist, Some(extra))); index
} }
};
petgraph.add_edge(
index,
dep_index,
Edge::Dev(group.clone(), dep.complexified_marker.clone()),
);
// Push its dependencies on the queue.
if seen.insert((&dep.package_id, None)) {
queue.push_back((dep_dist, None));
}
for extra in &dep.extra {
if seen.insert((&dep.package_id, Some(extra))) {
queue.push_back((dep_dist, Some(extra)));
} }
} }
} }
@ -215,70 +260,173 @@ impl<'env> InstallTarget<'env> {
.map_err(|err| LockErrorKind::DependencyGroup { err })?; .map_err(|err| LockErrorKind::DependencyGroup { err })?;
for group in dev.iter() { for group in dev.iter() {
for dependency in groups.get(group).into_iter().flatten() { for dependency in groups.get(group).into_iter().flatten() {
if dependency.marker.evaluate(marker_env, &[]) { if !dependency.marker.evaluate(marker_env, &[]) {
let root_name = &dependency.name; continue;
let root = self }
.lock()
.find_by_markers(root_name, marker_env)
.map_err(|_| LockErrorKind::MultipleRootPackages {
name: root_name.clone(),
})?
.ok_or_else(|| LockErrorKind::MissingRootPackage {
name: root_name.clone(),
})?;
// Add the base package. let root_name = &dependency.name;
queue.push_back((root, None)); let dist = self
.lock()
.find_by_markers(root_name, marker_env)
.map_err(|_| LockErrorKind::MultipleRootPackages {
name: root_name.clone(),
})?
.ok_or_else(|| LockErrorKind::MissingRootPackage {
name: root_name.clone(),
})?;
// Add any extras. // Add the workspace package to the graph.
for extra in &dependency.extras { let index = match inverse.entry(&dist.id) {
queue.push_back((root, Some(extra))); Entry::Vacant(entry) => {
let index = petgraph.add_node(self.package_to_node(
dist,
tags,
build_options,
install_options,
)?);
entry.insert(index);
index
} }
Entry::Occupied(entry) => {
let index = *entry.get();
index
}
};
// Add an edge from the root.
petgraph.add_edge(
root,
index,
Edge::Dev(group.clone(), dependency.marker.clone()),
);
// Push its dependencies on the queue.
queue.push_back((dist, None));
for extra in &dependency.extras {
queue.push_back((dist, Some(extra)));
} }
} }
} }
let mut map = BTreeMap::default(); while let Some((package, extra)) = queue.pop_front() {
let mut hashes = BTreeMap::default();
while let Some((dist, extra)) = queue.pop_front() {
let deps = if let Some(extra) = extra { let deps = if let Some(extra) = extra {
Either::Left(dist.optional_dependencies.get(extra).into_iter().flatten()) Either::Left(
package
.optional_dependencies
.get(extra)
.into_iter()
.flatten(),
)
} else { } else {
Either::Right(dist.dependencies.iter()) Either::Right(package.dependencies.iter())
}; };
for dep in deps { for dep in deps {
if dep.complexified_marker.evaluate(marker_env, &[]) { if !dep.complexified_marker.evaluate(marker_env, &[]) {
let dep_dist = self.lock().find_by_id(&dep.package_id); continue;
if seen.insert((&dep.package_id, None)) { }
queue.push_back((dep_dist, None));
let dep_dist = self.lock().find_by_id(&dep.package_id);
// Add the dependency to the graph.
let dep_index = match inverse.entry(&dep.package_id) {
Entry::Vacant(entry) => {
let index = petgraph.add_node(self.package_to_node(
dep_dist,
tags,
build_options,
install_options,
)?);
entry.insert(index);
index
} }
for extra in &dep.extra { Entry::Occupied(entry) => {
if seen.insert((&dep.package_id, Some(extra))) { let index = *entry.get();
queue.push_back((dep_dist, Some(extra))); index
} }
};
// Add the edge.
let index = inverse[&package.id];
petgraph.add_edge(
index,
dep_index,
if let Some(extra) = extra {
Edge::Optional(extra.clone(), dep.complexified_marker.clone())
} else {
Edge::Prod(dep.complexified_marker.clone())
},
);
// Push its dependencies on the queue.
if seen.insert((&dep.package_id, None)) {
queue.push_back((dep_dist, None));
}
for extra in &dep.extra {
if seen.insert((&dep.package_id, Some(extra))) {
queue.push_back((dep_dist, Some(extra)));
} }
} }
} }
if install_options.include_package(
&dist.id.name,
self.project_name(),
self.lock().members(),
) {
map.insert(
dist.id.name.clone(),
ResolvedDist::Installable {
dist: dist.to_dist(
self.workspace().install_path(),
TagPolicy::Required(tags),
build_options,
)?,
version: dist.id.version.clone(),
},
);
hashes.insert(dist.id.name.clone(), dist.hashes());
}
} }
let diagnostics = vec![];
Ok(Resolution::new(map, hashes, diagnostics)) Ok(Resolution::new(petgraph))
}
/// Create an installable [`Node`] from a [`Package`].
fn installable_node(
&self,
package: &Package,
tags: &Tags,
build_options: &BuildOptions,
) -> Result<Node, LockError> {
let dist = package.to_dist(
self.workspace().install_path(),
TagPolicy::Required(tags),
build_options,
)?;
let version = package.version().clone();
let dist = ResolvedDist::Installable { dist, version };
let hashes = package.hashes();
Ok(Node::Dist {
dist,
hashes,
install: true,
})
}
/// Create a non-installable [`Node`] from a [`Package`].
fn non_installable_node(&self, package: &Package, tags: &Tags) -> Result<Node, LockError> {
let dist = package.to_dist(
self.workspace().install_path(),
TagPolicy::Preferred(tags),
&BuildOptions::default(),
)?;
let version = package.version().clone();
let dist = ResolvedDist::Installable { dist, version };
let hashes = package.hashes();
Ok(Node::Dist {
dist,
hashes,
install: false,
})
}
/// Convert a lockfile entry to a graph [`Node`].
fn package_to_node(
&self,
package: &Package,
tags: &Tags,
build_options: &BuildOptions,
install_options: &InstallOptions,
) -> Result<Node, LockError> {
if install_options.include_package(
package.name(),
self.project_name(),
self.lock().members(),
) {
self.installable_node(package, tags, build_options)
} else {
self.non_installable_node(package, tags)
}
} }
} }

View File

@ -2,12 +2,10 @@ use std::collections::BTreeSet;
use owo_colors::OwoColorize; use owo_colors::OwoColorize;
use petgraph::visit::EdgeRef; use petgraph::visit::EdgeRef;
use petgraph::Direction; use petgraph::{Directed, Direction, Graph};
use rustc_hash::{FxBuildHasher, FxHashMap}; use rustc_hash::{FxBuildHasher, FxHashMap};
use uv_distribution_types::{ use uv_distribution_types::{DistributionMetadata, Name, SourceAnnotation, SourceAnnotations};
DistributionMetadata, Name, SourceAnnotation, SourceAnnotations, VersionId,
};
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep508::MarkerTree; use uv_pep508::MarkerTree;
@ -305,11 +303,9 @@ pub enum AnnotationStyle {
} }
/// We don't need the edge markers anymore since we switched to propagated markers. /// We don't need the edge markers anymore since we switched to propagated markers.
type IntermediatePetGraph<'dist> = type IntermediatePetGraph<'dist> = Graph<DisplayResolutionGraphNode<'dist>, (), Directed>;
petgraph::graph::Graph<DisplayResolutionGraphNode<'dist>, (), petgraph::Directed>;
type RequirementsTxtGraph<'dist> = type RequirementsTxtGraph<'dist> = Graph<RequirementsTxtDist<'dist>, (), Directed>;
petgraph::graph::Graph<RequirementsTxtDist<'dist>, (), petgraph::Directed>;
/// Reduce the graph, such that all nodes for a single package are combined, regardless of /// Reduce the graph, such that all nodes for a single package are combined, regardless of
/// the extras, as long as they have the same version and markers. /// the extras, as long as they have the same version and markers.
@ -324,8 +320,10 @@ type RequirementsTxtGraph<'dist> =
/// We also remove the root node, to simplify the graph structure. /// We also remove the root node, to simplify the graph structure.
fn combine_extras<'dist>(graph: &IntermediatePetGraph<'dist>) -> RequirementsTxtGraph<'dist> { fn combine_extras<'dist>(graph: &IntermediatePetGraph<'dist>) -> RequirementsTxtGraph<'dist> {
/// Return the key for a node. /// Return the key for a node.
fn version_marker(dist: &RequirementsTxtDist) -> (VersionId, MarkerTree) { fn version_marker<'dist>(
(dist.version_id(), dist.markers.clone()) dist: &'dist RequirementsTxtDist,
) -> (&'dist PackageName, &'dist MarkerTree) {
(dist.name(), dist.markers)
} }
let mut next = RequirementsTxtGraph::with_capacity(graph.node_count(), graph.edge_count()); let mut next = RequirementsTxtGraph::with_capacity(graph.node_count(), graph.edge_count());

View File

@ -10,8 +10,8 @@ use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use uv_configuration::{Constraints, Overrides}; use uv_configuration::{Constraints, Overrides};
use uv_distribution::Metadata; use uv_distribution::Metadata;
use uv_distribution_types::{ use uv_distribution_types::{
Dist, DistributionMetadata, IndexUrl, Name, ResolutionDiagnostic, ResolvedDist, VersionId, Dist, DistributionMetadata, Edge, IndexUrl, Name, Node, ResolutionDiagnostic, ResolvedDist,
VersionOrUrlRef, VersionId, VersionOrUrlRef,
}; };
use uv_git::GitResolver; use uv_git::GitResolver;
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
@ -275,7 +275,7 @@ impl ResolverOutput {
} }
fn add_edge( fn add_edge(
petgraph: &mut Graph<ResolutionGraphNode, MarkerTree>, graph: &mut Graph<ResolutionGraphNode, MarkerTree>,
inverse: &mut FxHashMap<PackageRef<'_>, NodeIndex>, inverse: &mut FxHashMap<PackageRef<'_>, NodeIndex>,
root_index: NodeIndex, root_index: NodeIndex,
edge: &ResolutionDependencyEdge, edge: &ResolutionDependencyEdge,
@ -306,20 +306,20 @@ impl ResolverOutput {
edge_marker edge_marker
}; };
if let Some(marker) = petgraph if let Some(marker) = graph
.find_edge(from_index, to_index) .find_edge(from_index, to_index)
.and_then(|edge| petgraph.edge_weight_mut(edge)) .and_then(|edge| graph.edge_weight_mut(edge))
{ {
// If either the existing marker or new marker is `true`, then the dependency is // If either the existing marker or new marker is `true`, then the dependency is
// included unconditionally, and so the combined marker is `true`. // included unconditionally, and so the combined marker is `true`.
marker.or(edge_marker); marker.or(edge_marker);
} else { } else {
petgraph.update_edge(from_index, to_index, edge_marker); graph.update_edge(from_index, to_index, edge_marker);
} }
} }
fn add_version<'a>( fn add_version<'a>(
petgraph: &mut Graph<ResolutionGraphNode, MarkerTree>, graph: &mut Graph<ResolutionGraphNode, MarkerTree>,
inverse: &mut FxHashMap<PackageRef<'a>, NodeIndex>, inverse: &mut FxHashMap<PackageRef<'a>, NodeIndex>,
diagnostics: &mut Vec<ResolutionDiagnostic>, diagnostics: &mut Vec<ResolutionDiagnostic>,
preferences: &Preferences, preferences: &Preferences,
@ -372,7 +372,7 @@ impl ResolverOutput {
} }
// Add the distribution to the graph. // Add the distribution to the graph.
let node = petgraph.add_node(ResolutionGraphNode::Dist(AnnotatedDist { let node = graph.add_node(ResolutionGraphNode::Dist(AnnotatedDist {
dist, dist,
name: name.clone(), name: name.clone(),
version: version.clone(), version: version.clone(),
@ -814,29 +814,98 @@ impl Display for ConflictingDistributionError {
} }
} }
/// Convert a [`ResolverOutput`] into a [`uv_distribution_types::Resolution`].
///
/// This involves converting [`ResolutionGraphNode`]s into [`Node`]s, which in turn involves
/// dropping any extras and dependency groups from the graph nodes. Instead, each package is
/// collapsed into a single node, with extras and dependency groups annotating the _edges_, rather
/// than being represented as separate nodes. This is a more natural representation, but a further
/// departure from the PubGrub model.
///
/// For simplicity, this transformation makes the assumption that the resolution only applies to a
/// subset of markers, i.e., it shouldn't be called on universal resolutions, and expects only a
/// single version of each package to be present in the graph.
impl From<ResolverOutput> for uv_distribution_types::Resolution { impl From<ResolverOutput> for uv_distribution_types::Resolution {
fn from(graph: ResolverOutput) -> Self { fn from(output: ResolverOutput) -> Self {
Self::new( let ResolverOutput {
graph graph,
.dists() diagnostics,
.map(|node| (node.name().clone(), node.dist.clone())) fork_markers,
.collect(), ..
graph } = output;
.dists()
.map(|node| (node.name().clone(), node.hashes.clone())) assert!(
.collect(), fork_markers.is_empty(),
graph.diagnostics, "universal resolutions are not supported"
) );
let mut transformed = Graph::with_capacity(graph.node_count(), graph.edge_count());
let mut inverse = FxHashMap::with_capacity_and_hasher(graph.node_count(), FxBuildHasher);
// Create the root node.
let root = transformed.add_node(Node::Root);
// Re-add the nodes to the reduced graph.
for index in graph.node_indices() {
let ResolutionGraphNode::Dist(dist) = &graph[index] else {
continue;
};
if dist.is_base() {
inverse.insert(
&dist.name,
transformed.add_node(Node::Dist {
dist: dist.dist.clone(),
hashes: dist.hashes.clone(),
install: true,
}),
);
}
}
// Re-add the edges to the reduced graph.
for edge in graph.edge_indices() {
let (source, target) = graph.edge_endpoints(edge).unwrap();
let marker = graph[edge].clone();
match (&graph[source], &graph[target]) {
(ResolutionGraphNode::Root, ResolutionGraphNode::Dist(target_dist)) => {
let target = inverse[&target_dist.name()];
transformed.update_edge(root, target, Edge::Prod(marker));
}
(
ResolutionGraphNode::Dist(source_dist),
ResolutionGraphNode::Dist(target_dist),
) => {
let source = inverse[&source_dist.name()];
let target = inverse[&target_dist.name()];
let edge = if let Some(extra) = source_dist.extra.as_ref() {
Edge::Optional(extra.clone(), marker)
} else if let Some(dev) = source_dist.dev.as_ref() {
Edge::Dev(dev.clone(), marker)
} else {
Edge::Prod(marker)
};
transformed.add_edge(source, target, edge);
}
_ => {
unreachable!("root should not contain incoming edges");
}
}
}
uv_distribution_types::Resolution::new(transformed).with_diagnostics(diagnostics)
} }
} }
/// Find any packages that don't have any lower bound on them when in resolution-lowest mode. /// Find any packages that don't have any lower bound on them when in resolution-lowest mode.
fn report_missing_lower_bounds( fn report_missing_lower_bounds(
petgraph: &Graph<ResolutionGraphNode, MarkerTree>, graph: &Graph<ResolutionGraphNode, MarkerTree>,
diagnostics: &mut Vec<ResolutionDiagnostic>, diagnostics: &mut Vec<ResolutionDiagnostic>,
) { ) {
for node_index in petgraph.node_indices() { for node_index in graph.node_indices() {
let ResolutionGraphNode::Dist(dist) = petgraph.node_weight(node_index).unwrap() else { let ResolutionGraphNode::Dist(dist) = graph.node_weight(node_index).unwrap() else {
// Ignore the root package. // Ignore the root package.
continue; continue;
}; };
@ -847,7 +916,7 @@ fn report_missing_lower_bounds(
// have to drop. // have to drop.
continue; continue;
} }
if !has_lower_bound(node_index, dist.name(), petgraph) { if !has_lower_bound(node_index, dist.name(), graph) {
diagnostics.push(ResolutionDiagnostic::MissingLowerBound { diagnostics.push(ResolutionDiagnostic::MissingLowerBound {
package_name: dist.name().clone(), package_name: dist.name().clone(),
}); });
@ -859,10 +928,10 @@ fn report_missing_lower_bounds(
fn has_lower_bound( fn has_lower_bound(
node_index: NodeIndex, node_index: NodeIndex,
package_name: &PackageName, package_name: &PackageName,
petgraph: &Graph<ResolutionGraphNode, MarkerTree>, graph: &Graph<ResolutionGraphNode, MarkerTree>,
) -> bool { ) -> bool {
for neighbor_index in petgraph.neighbors_directed(node_index, Direction::Incoming) { for neighbor_index in graph.neighbors_directed(node_index, Direction::Incoming) {
let neighbor_dist = match petgraph.node_weight(neighbor_index).unwrap() { let neighbor_dist = match graph.node_weight(neighbor_index).unwrap() {
ResolutionGraphNode::Root => { ResolutionGraphNode::Root => {
// We already handled direct dependencies with a missing constraint // We already handled direct dependencies with a missing constraint
// separately. // separately.

View File

@ -274,8 +274,7 @@ impl HashStrategy {
) -> Result<Self, HashStrategyError> { ) -> Result<Self, HashStrategyError> {
let mut hashes = FxHashMap::<VersionId, Vec<HashDigest>>::default(); let mut hashes = FxHashMap::<VersionId, Vec<HashDigest>>::default();
for dist in resolution.distributions() { for (dist, digests) in resolution.hashes() {
let digests = resolution.get_hashes(dist.name());
if digests.is_empty() { if digests.is_empty() {
// Under `--require-hashes`, every requirement must include a hash. // Under `--require-hashes`, every requirement must include a hash.
if mode.is_require() { if mode.is_require() {

View File

@ -733,8 +733,8 @@ pub(crate) fn diagnose_environment(
for diagnostic in site_packages.diagnostics(markers)? { for diagnostic in site_packages.diagnostics(markers)? {
// Only surface diagnostics that are "relevant" to the current resolution. // Only surface diagnostics that are "relevant" to the current resolution.
if resolution if resolution
.packages() .distributions()
.any(|package| diagnostic.includes(package)) .any(|dist| diagnostic.includes(dist.name()))
{ {
writeln!( writeln!(
printer.stderr(), printer.stderr(),

View File

@ -510,19 +510,19 @@ fn apply_editable_mode(
version, version,
} = dist } = dist
else { else {
return dist; return None;
}; };
ResolvedDist::Installable { Some(ResolvedDist::Installable {
dist: Dist::Source(SourceDist::Directory(DirectorySourceDist { dist: Dist::Source(SourceDist::Directory(DirectorySourceDist {
name, name: name.clone(),
install_path, install_path: install_path.clone(),
editable: false, editable: false,
r#virtual: false, r#virtual: false,
url, url: url.clone(),
})), })),
version, version: version.clone(),
} })
}), }),
} }
} }