mirror of https://github.com/astral-sh/uv
Allow registries to pre-provide core metadata (#15644)
## Summary This PR adds support for the `application/vnd.pyx.simple.v1` content type, similar to `application/vnd.pypi.simple.v1` with the exception that it can also include core metadata for package-versions directly.
This commit is contained in:
parent
f88aaa8740
commit
b57ad179b6
|
|
@ -1002,7 +1002,7 @@ impl CacheBucket {
|
||||||
Self::Interpreter => "interpreter-v4",
|
Self::Interpreter => "interpreter-v4",
|
||||||
// Note that when bumping this, you'll also need to bump it
|
// Note that when bumping this, you'll also need to bump it
|
||||||
// in `crates/uv/tests/it/cache_clean.rs`.
|
// in `crates/uv/tests/it/cache_clean.rs`.
|
||||||
Self::Simple => "simple-v16",
|
Self::Simple => "simple-v17",
|
||||||
// Note that when bumping this, you'll also need to bump it
|
// Note that when bumping this, you'll also need to bump it
|
||||||
// in `crates/uv/tests/it/cache_prune.rs`.
|
// in `crates/uv/tests/it/cache_prune.rs`.
|
||||||
Self::Wheels => "wheels-v5",
|
Self::Wheels => "wheels-v5",
|
||||||
|
|
|
||||||
|
|
@ -75,6 +75,11 @@ impl Error {
|
||||||
ErrorKind::BadHtml { source: err, url }.into()
|
ErrorKind::BadHtml { source: err, url }.into()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create a new error from a `MessagePack` parsing error.
|
||||||
|
pub(crate) fn from_msgpack_err(err: rmp_serde::decode::Error, url: DisplaySafeUrl) -> Self {
|
||||||
|
ErrorKind::BadMessagePack { source: err, url }.into()
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns `true` if this error corresponds to an offline error.
|
/// Returns `true` if this error corresponds to an offline error.
|
||||||
pub(crate) fn is_offline(&self) -> bool {
|
pub(crate) fn is_offline(&self) -> bool {
|
||||||
matches!(&*self.kind, ErrorKind::Offline(_))
|
matches!(&*self.kind, ErrorKind::Offline(_))
|
||||||
|
|
@ -251,6 +256,12 @@ pub enum ErrorKind {
|
||||||
url: DisplaySafeUrl,
|
url: DisplaySafeUrl,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
#[error("Received some unexpected MessagePack from {}", url)]
|
||||||
|
BadMessagePack {
|
||||||
|
source: rmp_serde::decode::Error,
|
||||||
|
url: DisplaySafeUrl,
|
||||||
|
},
|
||||||
|
|
||||||
#[error("Failed to read zip with range requests: `{0}`")]
|
#[error("Failed to read zip with range requests: `{0}`")]
|
||||||
AsyncHttpRangeReader(DisplaySafeUrl, #[source] AsyncHttpRangeReaderError),
|
AsyncHttpRangeReader(DisplaySafeUrl, #[source] AsyncHttpRangeReaderError),
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -204,7 +204,7 @@ impl<'a> FlatIndexClient<'a> {
|
||||||
let unarchived: Vec<File> = files
|
let unarchived: Vec<File> = files
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|file| {
|
.filter_map(|file| {
|
||||||
match File::try_from(file, &base) {
|
match File::try_from_pypi(file, &base) {
|
||||||
Ok(file) => Some(file),
|
Ok(file) => Some(file),
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
// Ignore files with unparsable version specifiers.
|
// Ignore files with unparsable version specifiers.
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ use tokio::sync::{Mutex, Semaphore};
|
||||||
use tracing::{Instrument, debug, info_span, instrument, trace, warn};
|
use tracing::{Instrument, debug, info_span, instrument, trace, warn};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use uv_auth::Indexes;
|
use uv_auth::{Indexes, PyxTokenStore};
|
||||||
use uv_cache::{Cache, CacheBucket, CacheEntry, WheelCache};
|
use uv_cache::{Cache, CacheBucket, CacheEntry, WheelCache};
|
||||||
use uv_configuration::IndexStrategy;
|
use uv_configuration::IndexStrategy;
|
||||||
use uv_configuration::KeyringProviderType;
|
use uv_configuration::KeyringProviderType;
|
||||||
|
|
@ -29,7 +29,7 @@ use uv_normalize::PackageName;
|
||||||
use uv_pep440::Version;
|
use uv_pep440::Version;
|
||||||
use uv_pep508::MarkerEnvironment;
|
use uv_pep508::MarkerEnvironment;
|
||||||
use uv_platform_tags::Platform;
|
use uv_platform_tags::Platform;
|
||||||
use uv_pypi_types::{PypiSimpleDetail, ResolutionMetadata};
|
use uv_pypi_types::{PypiSimpleDetail, PyxSimpleDetail, ResolutionMetadata};
|
||||||
use uv_redacted::DisplaySafeUrl;
|
use uv_redacted::DisplaySafeUrl;
|
||||||
use uv_small_str::SmallString;
|
use uv_small_str::SmallString;
|
||||||
use uv_torch::TorchStrategy;
|
use uv_torch::TorchStrategy;
|
||||||
|
|
@ -173,6 +173,7 @@ impl<'a> RegistryClientBuilder<'a> {
|
||||||
client,
|
client,
|
||||||
timeout,
|
timeout,
|
||||||
flat_indexes: Arc::default(),
|
flat_indexes: Arc::default(),
|
||||||
|
pyx_token_store: PyxTokenStore::from_settings().ok(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -202,6 +203,7 @@ impl<'a> RegistryClientBuilder<'a> {
|
||||||
client,
|
client,
|
||||||
timeout,
|
timeout,
|
||||||
flat_indexes: Arc::default(),
|
flat_indexes: Arc::default(),
|
||||||
|
pyx_token_store: PyxTokenStore::from_settings().ok(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -225,6 +227,9 @@ pub struct RegistryClient {
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
/// The flat index entries for each `--find-links`-style index URL.
|
/// The flat index entries for each `--find-links`-style index URL.
|
||||||
flat_indexes: Arc<Mutex<FlatIndexCache>>,
|
flat_indexes: Arc<Mutex<FlatIndexCache>>,
|
||||||
|
/// The pyx token store to use for persistent credentials.
|
||||||
|
// TODO(charlie): The token store is only needed for `is_known_url`; can we avoid storing it here?
|
||||||
|
pyx_token_store: Option<PyxTokenStore>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The format of the package metadata returned by querying an index.
|
/// The format of the package metadata returned by querying an index.
|
||||||
|
|
@ -512,7 +517,7 @@ impl RegistryClient {
|
||||||
let result = if matches!(index, IndexUrl::Path(_)) {
|
let result = if matches!(index, IndexUrl::Path(_)) {
|
||||||
self.fetch_local_index(package_name, &url).await
|
self.fetch_local_index(package_name, &url).await
|
||||||
} else {
|
} else {
|
||||||
self.fetch_remote_index(package_name, &url, &cache_entry, cache_control)
|
self.fetch_remote_index(package_name, &url, index, &cache_entry, cache_control)
|
||||||
.await
|
.await
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -553,14 +558,27 @@ impl RegistryClient {
|
||||||
&self,
|
&self,
|
||||||
package_name: &PackageName,
|
package_name: &PackageName,
|
||||||
url: &DisplaySafeUrl,
|
url: &DisplaySafeUrl,
|
||||||
|
index: &IndexUrl,
|
||||||
cache_entry: &CacheEntry,
|
cache_entry: &CacheEntry,
|
||||||
cache_control: CacheControl<'_>,
|
cache_control: CacheControl<'_>,
|
||||||
) -> Result<OwnedArchive<SimpleMetadata>, Error> {
|
) -> Result<OwnedArchive<SimpleMetadata>, Error> {
|
||||||
|
// In theory, we should be able to pass `MediaType::all()` to all registries, and as
|
||||||
|
// unsupported media types should be ignored by the server. For now, we implement this
|
||||||
|
// defensively to avoid issues with misconfigured servers.
|
||||||
|
let accept = if self
|
||||||
|
.pyx_token_store
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|token_store| token_store.is_known_url(index.url()))
|
||||||
|
{
|
||||||
|
MediaType::all()
|
||||||
|
} else {
|
||||||
|
MediaType::pypi()
|
||||||
|
};
|
||||||
let simple_request = self
|
let simple_request = self
|
||||||
.uncached_client(url)
|
.uncached_client(url)
|
||||||
.get(Url::from(url.clone()))
|
.get(Url::from(url.clone()))
|
||||||
.header("Accept-Encoding", "gzip, deflate, zstd")
|
.header("Accept-Encoding", "gzip, deflate, zstd")
|
||||||
.header("Accept", MediaType::accepts())
|
.header("Accept", accept)
|
||||||
.build()
|
.build()
|
||||||
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
|
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
|
||||||
let parse_simple_response = |response: Response| {
|
let parse_simple_response = |response: Response| {
|
||||||
|
|
@ -585,17 +603,48 @@ impl RegistryClient {
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let unarchived = match media_type {
|
let unarchived = match media_type {
|
||||||
MediaType::Json => {
|
MediaType::PyxV1Msgpack => {
|
||||||
let bytes = response
|
let bytes = response
|
||||||
.bytes()
|
.bytes()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
|
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
|
||||||
|
let data: PyxSimpleDetail = rmp_serde::from_slice(bytes.as_ref())
|
||||||
|
.map_err(|err| Error::from_msgpack_err(err, url.clone()))?;
|
||||||
|
|
||||||
|
SimpleMetadata::from_pyx_files(
|
||||||
|
data.files,
|
||||||
|
data.core_metadata,
|
||||||
|
package_name,
|
||||||
|
&url,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
MediaType::PyxV1Json => {
|
||||||
|
let bytes = response
|
||||||
|
.bytes()
|
||||||
|
.await
|
||||||
|
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
|
||||||
|
let data: PyxSimpleDetail = serde_json::from_slice(bytes.as_ref())
|
||||||
|
.map_err(|err| Error::from_json_err(err, url.clone()))?;
|
||||||
|
|
||||||
|
SimpleMetadata::from_pyx_files(
|
||||||
|
data.files,
|
||||||
|
data.core_metadata,
|
||||||
|
package_name,
|
||||||
|
&url,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
MediaType::PypiV1Json => {
|
||||||
|
let bytes = response
|
||||||
|
.bytes()
|
||||||
|
.await
|
||||||
|
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
|
||||||
|
|
||||||
let data: PypiSimpleDetail = serde_json::from_slice(bytes.as_ref())
|
let data: PypiSimpleDetail = serde_json::from_slice(bytes.as_ref())
|
||||||
.map_err(|err| Error::from_json_err(err, url.clone()))?;
|
.map_err(|err| Error::from_json_err(err, url.clone()))?;
|
||||||
|
|
||||||
SimpleMetadata::from_pypi_files(data.files, package_name, &url)
|
SimpleMetadata::from_pypi_files(data.files, package_name, &url)
|
||||||
}
|
}
|
||||||
MediaType::Html => {
|
MediaType::PypiV1Html | MediaType::TextHtml => {
|
||||||
let text = response
|
let text = response
|
||||||
.text()
|
.text()
|
||||||
.await
|
.await
|
||||||
|
|
@ -1089,6 +1138,7 @@ pub struct SimpleMetadata(Vec<SimpleMetadatum>);
|
||||||
pub struct SimpleMetadatum {
|
pub struct SimpleMetadatum {
|
||||||
pub version: Version,
|
pub version: Version,
|
||||||
pub files: VersionFiles,
|
pub files: VersionFiles,
|
||||||
|
pub metadata: Option<ResolutionMetadata>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SimpleMetadata {
|
impl SimpleMetadata {
|
||||||
|
|
@ -1101,7 +1151,7 @@ impl SimpleMetadata {
|
||||||
package_name: &PackageName,
|
package_name: &PackageName,
|
||||||
base: &Url,
|
base: &Url,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let mut map: BTreeMap<Version, VersionFiles> = BTreeMap::default();
|
let mut version_map: BTreeMap<Version, VersionFiles> = BTreeMap::default();
|
||||||
|
|
||||||
// Convert to a reference-counted string.
|
// Convert to a reference-counted string.
|
||||||
let base = SmallString::from(base.as_str());
|
let base = SmallString::from(base.as_str());
|
||||||
|
|
@ -1113,11 +1163,7 @@ impl SimpleMetadata {
|
||||||
warn!("Skipping file for {package_name}: {}", file.filename);
|
warn!("Skipping file for {package_name}: {}", file.filename);
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let version = match filename {
|
let file = match File::try_from_pypi(file, &base) {
|
||||||
DistFilename::SourceDistFilename(ref inner) => &inner.version,
|
|
||||||
DistFilename::WheelFilename(ref inner) => &inner.version,
|
|
||||||
};
|
|
||||||
let file = match File::try_from(file, &base) {
|
|
||||||
Ok(file) => file,
|
Ok(file) => file,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
// Ignore files with unparsable version specifiers.
|
// Ignore files with unparsable version specifiers.
|
||||||
|
|
@ -1125,7 +1171,7 @@ impl SimpleMetadata {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
match map.entry(version.clone()) {
|
match version_map.entry(filename.version().clone()) {
|
||||||
std::collections::btree_map::Entry::Occupied(mut entry) => {
|
std::collections::btree_map::Entry::Occupied(mut entry) => {
|
||||||
entry.get_mut().push(filename, file);
|
entry.get_mut().push(filename, file);
|
||||||
}
|
}
|
||||||
|
|
@ -1136,9 +1182,78 @@ impl SimpleMetadata {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Self(
|
Self(
|
||||||
map.into_iter()
|
version_map
|
||||||
.map(|(version, files)| SimpleMetadatum { version, files })
|
.into_iter()
|
||||||
|
.map(|(version, files)| SimpleMetadatum {
|
||||||
|
version,
|
||||||
|
files,
|
||||||
|
metadata: None,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_pyx_files(
|
||||||
|
files: Vec<uv_pypi_types::PyxFile>,
|
||||||
|
mut core_metadata: FxHashMap<Version, uv_pypi_types::CoreMetadatum>,
|
||||||
|
package_name: &PackageName,
|
||||||
|
base: &Url,
|
||||||
|
) -> Self {
|
||||||
|
let mut version_map: BTreeMap<Version, VersionFiles> = BTreeMap::default();
|
||||||
|
|
||||||
|
// Convert to a reference-counted string.
|
||||||
|
let base = SmallString::from(base.as_str());
|
||||||
|
|
||||||
|
// Group the distributions by version and kind
|
||||||
|
for file in files {
|
||||||
|
let file = match File::try_from_pyx(file, &base) {
|
||||||
|
Ok(file) => file,
|
||||||
|
Err(err) => {
|
||||||
|
// Ignore files with unparsable version specifiers.
|
||||||
|
warn!("Skipping file for {package_name}: {err}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let Some(filename) = DistFilename::try_from_filename(&file.filename, package_name)
|
||||||
|
else {
|
||||||
|
warn!("Skipping file for {package_name}: {}", file.filename);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
match version_map.entry(filename.version().clone()) {
|
||||||
|
std::collections::btree_map::Entry::Occupied(mut entry) => {
|
||||||
|
entry.get_mut().push(filename, file);
|
||||||
|
}
|
||||||
|
std::collections::btree_map::Entry::Vacant(entry) => {
|
||||||
|
let mut files = VersionFiles::default();
|
||||||
|
files.push(filename, file);
|
||||||
|
entry.insert(files);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Self(
|
||||||
|
version_map
|
||||||
|
.into_iter()
|
||||||
|
.map(|(version, files)| {
|
||||||
|
let metadata =
|
||||||
|
core_metadata
|
||||||
|
.remove(&version)
|
||||||
|
.map(|metadata| ResolutionMetadata {
|
||||||
|
name: package_name.clone(),
|
||||||
|
version: version.clone(),
|
||||||
|
requires_dist: metadata.requires_dist,
|
||||||
|
requires_python: metadata.requires_python,
|
||||||
|
provides_extras: metadata.provides_extras,
|
||||||
|
dynamic: false,
|
||||||
|
});
|
||||||
|
SimpleMetadatum {
|
||||||
|
version,
|
||||||
|
files,
|
||||||
|
metadata,
|
||||||
|
}
|
||||||
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
@ -1177,26 +1292,51 @@ impl ArchivedSimpleMetadata {
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
enum MediaType {
|
enum MediaType {
|
||||||
Json,
|
PyxV1Msgpack,
|
||||||
Html,
|
PyxV1Json,
|
||||||
|
PypiV1Json,
|
||||||
|
PypiV1Html,
|
||||||
|
TextHtml,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MediaType {
|
impl MediaType {
|
||||||
/// Parse a media type from a string, returning `None` if the media type is not supported.
|
/// Parse a media type from a string, returning `None` if the media type is not supported.
|
||||||
fn from_str(s: &str) -> Option<Self> {
|
fn from_str(s: &str) -> Option<Self> {
|
||||||
match s {
|
match s {
|
||||||
"application/vnd.pypi.simple.v1+json" => Some(Self::Json),
|
"application/vnd.pyx.simple.v1+msgpack" => Some(Self::PyxV1Msgpack),
|
||||||
"application/vnd.pypi.simple.v1+html" | "text/html" => Some(Self::Html),
|
"application/vnd.pyx.simple.v1+json" => Some(Self::PyxV1Json),
|
||||||
|
"application/vnd.pypi.simple.v1+json" => Some(Self::PypiV1Json),
|
||||||
|
"application/vnd.pypi.simple.v1+html" => Some(Self::PypiV1Html),
|
||||||
|
"text/html" => Some(Self::TextHtml),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the `Accept` header value for all supported media types.
|
/// Return the `Accept` header value for all PyPI media types.
|
||||||
#[inline]
|
#[inline]
|
||||||
const fn accepts() -> &'static str {
|
const fn pypi() -> &'static str {
|
||||||
// See: https://peps.python.org/pep-0691/#version-format-selection
|
// See: https://peps.python.org/pep-0691/#version-format-selection
|
||||||
"application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
|
"application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return the `Accept` header value for all supported media types.
|
||||||
|
#[inline]
|
||||||
|
const fn all() -> &'static str {
|
||||||
|
// See: https://peps.python.org/pep-0691/#version-format-selection
|
||||||
|
"application/vnd.pyx.simple.v1+msgpack, application/vnd.pyx.simple.v1+json;q=0.9, application/vnd.pypi.simple.v1+json;q=0.8, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for MediaType {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::PyxV1Msgpack => write!(f, "application/vnd.pyx.simple.v1+msgpack"),
|
||||||
|
Self::PyxV1Json => write!(f, "application/vnd.pyx.simple.v1+json"),
|
||||||
|
Self::PypiV1Json => write!(f, "application/vnd.pypi.simple.v1+json"),
|
||||||
|
Self::PypiV1Html => write!(f, "application/vnd.pypi.simple.v1+html"),
|
||||||
|
Self::TextHtml => write!(f, "text/html"),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)]
|
#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)]
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,10 @@ pub enum FileConversionError {
|
||||||
RequiresPython(String, #[source] VersionSpecifiersParseError),
|
RequiresPython(String, #[source] VersionSpecifiersParseError),
|
||||||
#[error("Failed to parse URL: {0}")]
|
#[error("Failed to parse URL: {0}")]
|
||||||
Url(String, #[source] url::ParseError),
|
Url(String, #[source] url::ParseError),
|
||||||
|
#[error("Failed to parse filename from URL: {0}")]
|
||||||
|
MissingPathSegments(String),
|
||||||
|
#[error(transparent)]
|
||||||
|
Utf8(#[from] std::str::Utf8Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Internal analog to [`uv_pypi_types::PypiFile`].
|
/// Internal analog to [`uv_pypi_types::PypiFile`].
|
||||||
|
|
@ -40,7 +44,7 @@ pub struct File {
|
||||||
|
|
||||||
impl File {
|
impl File {
|
||||||
/// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers
|
/// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers
|
||||||
pub fn try_from(
|
pub fn try_from_pypi(
|
||||||
file: uv_pypi_types::PypiFile,
|
file: uv_pypi_types::PypiFile,
|
||||||
base: &SmallString,
|
base: &SmallString,
|
||||||
) -> Result<Self, FileConversionError> {
|
) -> Result<Self, FileConversionError> {
|
||||||
|
|
@ -61,6 +65,51 @@ impl File {
|
||||||
yanked: file.yanked,
|
yanked: file.yanked,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn try_from_pyx(
|
||||||
|
file: uv_pypi_types::PyxFile,
|
||||||
|
base: &SmallString,
|
||||||
|
) -> Result<Self, FileConversionError> {
|
||||||
|
let filename = if let Some(filename) = file.filename {
|
||||||
|
filename
|
||||||
|
} else {
|
||||||
|
// Remove any query parameters or fragments from the URL to get the filename.
|
||||||
|
let base_url = file
|
||||||
|
.url
|
||||||
|
.as_ref()
|
||||||
|
.split_once('?')
|
||||||
|
.or_else(|| file.url.as_ref().split_once('#'))
|
||||||
|
.map(|(path, _)| path)
|
||||||
|
.unwrap_or(file.url.as_ref());
|
||||||
|
|
||||||
|
// Take the last segment, stripping any query or fragment.
|
||||||
|
let last = base_url
|
||||||
|
.split('/')
|
||||||
|
.next_back()
|
||||||
|
.ok_or_else(|| FileConversionError::MissingPathSegments(file.url.to_string()))?;
|
||||||
|
|
||||||
|
// Decode the filename, which may be percent-encoded.
|
||||||
|
let filename = percent_encoding::percent_decode_str(last).decode_utf8()?;
|
||||||
|
|
||||||
|
SmallString::from(filename)
|
||||||
|
};
|
||||||
|
Ok(Self {
|
||||||
|
filename,
|
||||||
|
dist_info_metadata: file
|
||||||
|
.core_metadata
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(CoreMetadata::is_available),
|
||||||
|
hashes: HashDigests::from(file.hashes),
|
||||||
|
requires_python: file
|
||||||
|
.requires_python
|
||||||
|
.transpose()
|
||||||
|
.map_err(|err| FileConversionError::RequiresPython(err.line().clone(), err))?,
|
||||||
|
size: file.size,
|
||||||
|
upload_time_utc_ms: file.upload_time.map(Timestamp::as_millisecond),
|
||||||
|
url: FileLocation::new(file.url, base),
|
||||||
|
yanked: file.yanked,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// While a registry file is generally a remote URL, it can also be a file if it comes from a directory flat indexes.
|
/// While a registry file is generally a remote URL, it can also be a file if it comes from a directory flat indexes.
|
||||||
|
|
|
||||||
|
|
@ -141,6 +141,15 @@ impl ResolvedDistRef<'_> {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the [`IndexUrl`], if the distribution is from a registry.
|
||||||
|
pub fn index(&self) -> Option<&IndexUrl> {
|
||||||
|
match self {
|
||||||
|
Self::InstallableRegistrySourceDist { sdist, .. } => Some(&sdist.index),
|
||||||
|
Self::InstallableRegistryBuiltDist { wheel, .. } => Some(&wheel.index),
|
||||||
|
Self::Installed { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Display for ResolvedDistRef<'_> {
|
impl Display for ResolvedDistRef<'_> {
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,15 @@ use std::borrow::Cow;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
use jiff::Timestamp;
|
use jiff::Timestamp;
|
||||||
|
use rustc_hash::FxHashMap;
|
||||||
use serde::{Deserialize, Deserializer, Serialize};
|
use serde::{Deserialize, Deserializer, Serialize};
|
||||||
|
|
||||||
use uv_pep440::{VersionSpecifiers, VersionSpecifiersParseError};
|
use uv_normalize::ExtraName;
|
||||||
|
use uv_pep440::{Version, VersionSpecifiers, VersionSpecifiersParseError};
|
||||||
|
use uv_pep508::Requirement;
|
||||||
use uv_small_str::SmallString;
|
use uv_small_str::SmallString;
|
||||||
|
|
||||||
|
use crate::VerbatimParsedUrl;
|
||||||
use crate::lenient_requirement::LenientVersionSpecifiers;
|
use crate::lenient_requirement::LenientVersionSpecifiers;
|
||||||
|
|
||||||
/// A collection of "files" from `PyPI`'s JSON API for a single package, as served by the
|
/// A collection of "files" from `PyPI`'s JSON API for a single package, as served by the
|
||||||
|
|
@ -123,6 +127,114 @@ impl<'de> Deserialize<'de> for PypiFile {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A collection of "files" from the Simple API.
|
||||||
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub struct PyxSimpleDetail {
|
||||||
|
/// The list of [`PyxFile`]s available for download sorted by filename.
|
||||||
|
pub files: Vec<PyxFile>,
|
||||||
|
/// The core metadata for the project, keyed by version.
|
||||||
|
#[serde(default)]
|
||||||
|
pub core_metadata: FxHashMap<Version, CoreMetadatum>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A single (remote) file belonging to a package, either a wheel or a source distribution,
|
||||||
|
/// as served by the Simple API.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct PyxFile {
|
||||||
|
pub core_metadata: Option<CoreMetadata>,
|
||||||
|
pub filename: Option<SmallString>,
|
||||||
|
pub hashes: Hashes,
|
||||||
|
pub requires_python: Option<Result<VersionSpecifiers, VersionSpecifiersParseError>>,
|
||||||
|
pub size: Option<u64>,
|
||||||
|
pub upload_time: Option<Timestamp>,
|
||||||
|
pub url: SmallString,
|
||||||
|
pub yanked: Option<Box<Yanked>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> Deserialize<'de> for PyxFile {
|
||||||
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
struct FileVisitor;
|
||||||
|
|
||||||
|
impl<'de> serde::de::Visitor<'de> for FileVisitor {
|
||||||
|
type Value = PyxFile;
|
||||||
|
|
||||||
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
|
formatter.write_str("a map containing file metadata")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_map<M>(self, mut access: M) -> Result<Self::Value, M::Error>
|
||||||
|
where
|
||||||
|
M: serde::de::MapAccess<'de>,
|
||||||
|
{
|
||||||
|
let mut core_metadata = None;
|
||||||
|
let mut filename = None;
|
||||||
|
let mut hashes = None;
|
||||||
|
let mut requires_python = None;
|
||||||
|
let mut size = None;
|
||||||
|
let mut upload_time = None;
|
||||||
|
let mut url = None;
|
||||||
|
let mut yanked = None;
|
||||||
|
|
||||||
|
while let Some(key) = access.next_key::<String>()? {
|
||||||
|
match key.as_str() {
|
||||||
|
"core-metadata" | "dist-info-metadata" | "data-dist-info-metadata" => {
|
||||||
|
if core_metadata.is_none() {
|
||||||
|
core_metadata = access.next_value()?;
|
||||||
|
} else {
|
||||||
|
let _: serde::de::IgnoredAny = access.next_value()?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"filename" => filename = Some(access.next_value()?),
|
||||||
|
"hashes" => hashes = Some(access.next_value()?),
|
||||||
|
"requires-python" => {
|
||||||
|
requires_python =
|
||||||
|
access.next_value::<Option<Cow<'_, str>>>()?.map(|s| {
|
||||||
|
LenientVersionSpecifiers::from_str(s.as_ref())
|
||||||
|
.map(VersionSpecifiers::from)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
"size" => size = Some(access.next_value()?),
|
||||||
|
"upload-time" => upload_time = Some(access.next_value()?),
|
||||||
|
"url" => url = Some(access.next_value()?),
|
||||||
|
"yanked" => yanked = Some(access.next_value()?),
|
||||||
|
_ => {
|
||||||
|
let _: serde::de::IgnoredAny = access.next_value()?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(PyxFile {
|
||||||
|
core_metadata,
|
||||||
|
filename,
|
||||||
|
hashes: hashes.ok_or_else(|| serde::de::Error::missing_field("hashes"))?,
|
||||||
|
requires_python,
|
||||||
|
size,
|
||||||
|
upload_time,
|
||||||
|
url: url.ok_or_else(|| serde::de::Error::missing_field("url"))?,
|
||||||
|
yanked,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
deserializer.deserialize_map(FileVisitor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub struct CoreMetadatum {
|
||||||
|
#[serde(default)]
|
||||||
|
pub requires_python: Option<VersionSpecifiers>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub requires_dist: Box<[Requirement<VerbatimParsedUrl>]>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub provides_extras: Box<[ExtraName]>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum CoreMetadata {
|
pub enum CoreMetadata {
|
||||||
Bool(bool),
|
Bool(bool),
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ use tokio_stream::wrappers::ReceiverStream;
|
||||||
use tracing::{Level, debug, info, instrument, trace, warn};
|
use tracing::{Level, debug, info, instrument, trace, warn};
|
||||||
|
|
||||||
use uv_configuration::{Constraints, Overrides};
|
use uv_configuration::{Constraints, Overrides};
|
||||||
use uv_distribution::DistributionDatabase;
|
use uv_distribution::{ArchiveMetadata, DistributionDatabase};
|
||||||
use uv_distribution_types::{
|
use uv_distribution_types::{
|
||||||
BuiltDist, CompatibleDist, DerivationChain, Dist, DistErrorKind, DistributionMetadata,
|
BuiltDist, CompatibleDist, DerivationChain, Dist, DistErrorKind, DistributionMetadata,
|
||||||
IncompatibleDist, IncompatibleSource, IncompatibleWheel, IndexCapabilities, IndexLocations,
|
IncompatibleDist, IncompatibleSource, IncompatibleWheel, IndexCapabilities, IndexLocations,
|
||||||
|
|
@ -2372,6 +2372,53 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
||||||
|
|
||||||
// Fetch distribution metadata from the distribution database.
|
// Fetch distribution metadata from the distribution database.
|
||||||
Request::Dist(dist) => {
|
Request::Dist(dist) => {
|
||||||
|
if let Some(version) = dist.version() {
|
||||||
|
if let Some(index) = dist.index() {
|
||||||
|
// Check the implicit indexes for pre-provided metadata.
|
||||||
|
let versions_response = self.index.implicit().get(dist.name());
|
||||||
|
if let Some(VersionsResponse::Found(version_maps)) =
|
||||||
|
versions_response.as_deref()
|
||||||
|
{
|
||||||
|
for version_map in version_maps {
|
||||||
|
if version_map.index() == Some(index) {
|
||||||
|
let Some(metadata) = version_map.get_metadata(version) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
debug!("Found registry-provided metadata for: {dist}");
|
||||||
|
return Ok(Some(Response::Dist {
|
||||||
|
dist,
|
||||||
|
metadata: MetadataResponse::Found(
|
||||||
|
ArchiveMetadata::from_metadata23(metadata.clone()),
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check the explicit indexes for pre-provided metadata.
|
||||||
|
let versions_response = self
|
||||||
|
.index
|
||||||
|
.explicit()
|
||||||
|
.get(&(dist.name().clone(), index.clone()));
|
||||||
|
if let Some(VersionsResponse::Found(version_maps)) =
|
||||||
|
versions_response.as_deref()
|
||||||
|
{
|
||||||
|
for version_map in version_maps {
|
||||||
|
let Some(metadata) = version_map.get_metadata(version) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
debug!("Found registry-provided metadata for: {dist}");
|
||||||
|
return Ok(Some(Response::Dist {
|
||||||
|
dist,
|
||||||
|
metadata: MetadataResponse::Found(
|
||||||
|
ArchiveMetadata::from_metadata23(metadata.clone()),
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let metadata = provider
|
let metadata = provider
|
||||||
.get_or_build_wheel_metadata(&dist)
|
.get_or_build_wheel_metadata(&dist)
|
||||||
.boxed_local()
|
.boxed_local()
|
||||||
|
|
@ -2464,6 +2511,42 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// If the registry provided metadata for this distribution, use it.
|
||||||
|
for version_map in version_map {
|
||||||
|
if let Some(metadata) = version_map.get_metadata(candidate.version()) {
|
||||||
|
let dist = dist.for_resolution();
|
||||||
|
if version_map.index() == dist.index() {
|
||||||
|
debug!("Found registry-provided metadata for: {dist}");
|
||||||
|
|
||||||
|
let metadata = MetadataResponse::Found(
|
||||||
|
ArchiveMetadata::from_metadata23(metadata.clone()),
|
||||||
|
);
|
||||||
|
|
||||||
|
let dist = dist.to_owned();
|
||||||
|
if &package_name != dist.name() {
|
||||||
|
return Err(ResolveError::MismatchedPackageName {
|
||||||
|
request: "distribution",
|
||||||
|
expected: package_name,
|
||||||
|
actual: dist.name().clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = match dist {
|
||||||
|
ResolvedDist::Installable { dist, .. } => Response::Dist {
|
||||||
|
dist: (*dist).clone(),
|
||||||
|
metadata,
|
||||||
|
},
|
||||||
|
ResolvedDist::Installed { dist } => Response::Installed {
|
||||||
|
dist: (*dist).clone(),
|
||||||
|
metadata,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return Ok(Some(response));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Avoid prefetching source distributions with unbounded lower-bound ranges. This
|
// Avoid prefetching source distributions with unbounded lower-bound ranges. This
|
||||||
// often leads to failed attempts to build legacy versions of packages that are
|
// often leads to failed attempts to build legacy versions of packages that are
|
||||||
// incompatible with modern build tools.
|
// incompatible with modern build tools.
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ use std::ops::RangeBounds;
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
use pubgrub::Ranges;
|
use pubgrub::Ranges;
|
||||||
|
use rustc_hash::FxHashMap;
|
||||||
use tracing::instrument;
|
use tracing::instrument;
|
||||||
|
|
||||||
use uv_client::{FlatIndexEntry, OwnedArchive, SimpleMetadata, VersionFiles};
|
use uv_client::{FlatIndexEntry, OwnedArchive, SimpleMetadata, VersionFiles};
|
||||||
|
|
@ -17,7 +18,7 @@ use uv_distribution_types::{
|
||||||
use uv_normalize::PackageName;
|
use uv_normalize::PackageName;
|
||||||
use uv_pep440::Version;
|
use uv_pep440::Version;
|
||||||
use uv_platform_tags::{IncompatibleTag, TagCompatibility, Tags};
|
use uv_platform_tags::{IncompatibleTag, TagCompatibility, Tags};
|
||||||
use uv_pypi_types::{HashDigest, Yanked};
|
use uv_pypi_types::{HashDigest, ResolutionMetadata, Yanked};
|
||||||
use uv_types::HashStrategy;
|
use uv_types::HashStrategy;
|
||||||
use uv_warnings::warn_user_once;
|
use uv_warnings::warn_user_once;
|
||||||
|
|
||||||
|
|
@ -57,12 +58,25 @@ impl VersionMap {
|
||||||
let mut stable = false;
|
let mut stable = false;
|
||||||
let mut local = false;
|
let mut local = false;
|
||||||
let mut map = BTreeMap::new();
|
let mut map = BTreeMap::new();
|
||||||
|
let mut core_metadata = FxHashMap::default();
|
||||||
// Create stubs for each entry in simple metadata. The full conversion
|
// Create stubs for each entry in simple metadata. The full conversion
|
||||||
// from a `VersionFiles` to a PrioritizedDist for each version
|
// from a `VersionFiles` to a PrioritizedDist for each version
|
||||||
// isn't done until that specific version is requested.
|
// isn't done until that specific version is requested.
|
||||||
for (datum_index, datum) in simple_metadata.iter().enumerate() {
|
for (datum_index, datum) in simple_metadata.iter().enumerate() {
|
||||||
|
// Deserialize the version.
|
||||||
let version = rkyv::deserialize::<Version, rkyv::rancor::Error>(&datum.version)
|
let version = rkyv::deserialize::<Version, rkyv::rancor::Error>(&datum.version)
|
||||||
.expect("archived version always deserializes");
|
.expect("archived version always deserializes");
|
||||||
|
|
||||||
|
// Deserialize the metadata.
|
||||||
|
let core_metadatum =
|
||||||
|
rkyv::deserialize::<Option<ResolutionMetadata>, rkyv::rancor::Error>(
|
||||||
|
&datum.metadata,
|
||||||
|
)
|
||||||
|
.expect("archived metadata always deserializes");
|
||||||
|
if let Some(core_metadatum) = core_metadatum {
|
||||||
|
core_metadata.insert(version.clone(), core_metadatum);
|
||||||
|
}
|
||||||
|
|
||||||
stable |= version.is_stable();
|
stable |= version.is_stable();
|
||||||
local |= version.is_local();
|
local |= version.is_local();
|
||||||
map.insert(
|
map.insert(
|
||||||
|
|
@ -104,6 +118,7 @@ impl VersionMap {
|
||||||
map,
|
map,
|
||||||
stable,
|
stable,
|
||||||
local,
|
local,
|
||||||
|
core_metadata,
|
||||||
simple_metadata,
|
simple_metadata,
|
||||||
no_binary: build_options.no_binary_package(package_name),
|
no_binary: build_options.no_binary_package(package_name),
|
||||||
no_build: build_options.no_build_package(package_name),
|
no_build: build_options.no_build_package(package_name),
|
||||||
|
|
@ -141,6 +156,14 @@ impl VersionMap {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return the [`ResolutionMetadata`] for the given version, if any.
|
||||||
|
pub fn get_metadata(&self, version: &Version) -> Option<&ResolutionMetadata> {
|
||||||
|
match self.inner {
|
||||||
|
VersionMapInner::Eager(_) => None,
|
||||||
|
VersionMapInner::Lazy(ref lazy) => lazy.core_metadata.get(version),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Return the [`DistFile`] for the given version, if any.
|
/// Return the [`DistFile`] for the given version, if any.
|
||||||
pub(crate) fn get(&self, version: &Version) -> Option<&PrioritizedDist> {
|
pub(crate) fn get(&self, version: &Version) -> Option<&PrioritizedDist> {
|
||||||
match self.inner {
|
match self.inner {
|
||||||
|
|
@ -352,6 +375,8 @@ struct VersionMapLazy {
|
||||||
stable: bool,
|
stable: bool,
|
||||||
/// Whether the version map contains at least one local version.
|
/// Whether the version map contains at least one local version.
|
||||||
local: bool,
|
local: bool,
|
||||||
|
/// The pre-populated metadata for each version.
|
||||||
|
core_metadata: FxHashMap<Version, ResolutionMetadata>,
|
||||||
/// The raw simple metadata from which `PrioritizedDist`s should
|
/// The raw simple metadata from which `PrioritizedDist`s should
|
||||||
/// be constructed.
|
/// be constructed.
|
||||||
simple_metadata: OwnedArchive<SimpleMetadata>,
|
simple_metadata: OwnedArchive<SimpleMetadata>,
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,7 @@ fn clean_package_pypi() -> Result<()> {
|
||||||
// Assert that the `.rkyv` file is created for `iniconfig`.
|
// Assert that the `.rkyv` file is created for `iniconfig`.
|
||||||
let rkyv = context
|
let rkyv = context
|
||||||
.cache_dir
|
.cache_dir
|
||||||
.child("simple-v16")
|
.child("simple-v17")
|
||||||
.child("pypi")
|
.child("pypi")
|
||||||
.child("iniconfig.rkyv");
|
.child("iniconfig.rkyv");
|
||||||
assert!(
|
assert!(
|
||||||
|
|
@ -125,7 +125,7 @@ fn clean_package_index() -> Result<()> {
|
||||||
// Assert that the `.rkyv` file is created for `iniconfig`.
|
// Assert that the `.rkyv` file is created for `iniconfig`.
|
||||||
let rkyv = context
|
let rkyv = context
|
||||||
.cache_dir
|
.cache_dir
|
||||||
.child("simple-v16")
|
.child("simple-v17")
|
||||||
.child("index")
|
.child("index")
|
||||||
.child("e8208120cae3ba69")
|
.child("e8208120cae3ba69")
|
||||||
.child("iniconfig.rkyv");
|
.child("iniconfig.rkyv");
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue