Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into zb/site-packages-link

This commit is contained in:
Charlie Marsh 2024-04-11 17:26:04 -04:00
parent 192b214dc6 8507ba872f
commit 375f76e701
194 changed files with 17549 additions and 7273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/renovate.json5 vendored
View File

@ -29,6 +29,18 @@
matchManagers: ["pre-commit"],
description: "Weekly update of pre-commit dependencies",
},
{
groupName: "Rust dev-dependencies",
matchManagers: ["cargo"],
matchDepTypes: ["devDependencies"],
description: "Weekly update of Rust development dependencies",
},
{
groupName: "pyo3",
matchManagers: ["cargo"],
matchPackagePatterns: ["pyo3"],
description: "Weekly update of pyo3 dependencies",
},
],
vulnerabilityAlerts: {
commitMessageSuffix: "",

76
.github/workflows/ci.yml vendored
View File

@ -72,33 +72,23 @@ jobs:
name: "cargo test | ${{ matrix.os }}"
steps:
- uses: actions/checkout@v4
- if: ${{ matrix.os == 'macos' }}
name: "Install bootstrap dependencies"
run: brew install coreutils
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: "Install required Python versions"
run: |
python -m pip install "zstandard==0.22.0"
python scripts/bootstrap/install.py
- name: "Install Rust toolchain"
run: rustup show
- name: "Install cargo nextest"
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- if: ${{ matrix.os != 'windows' }}
uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2
- name: "Install required Python versions"
run: |
cargo run -p uv-dev -- fetch-python
- name: "Install cargo nextest"
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- name: "Cargo test"
run: |
cargo nextest run --workspace --status-level skip --failure-output immediate-final --no-fail-fast -j 12 --final-status-level slow
@ -224,6 +214,42 @@ jobs:
path: ./target/debug/uv.exe
retention-days: 1
ecosystem-test:
needs: build-binary-linux
name: "ecosystem test | ${{ matrix.repo }}"
runs-on: ubuntu-latest
strategy:
matrix:
include:
- repo: "prefecthq/prefect"
command: "uv pip install -e '.[dev]'"
python: "3.9"
- repo: "pallets/flask"
command: "uv pip install -r requirements/dev.txt"
python: "3.12"
fail-fast: false
steps:
- uses: actions/checkout@v4
with:
repository: ${{ matrix.repo }}
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: "Download binary"
uses: actions/download-artifact@v4
with:
name: uv-linux-${{ github.sha }}
- name: "Prepare binary"
run: chmod +x ./uv
- name: "Test"
run: |
./uv venv
./${{ matrix.command }}
cache-test-ubuntu:
needs: build-binary-linux
name: "check cache | ubuntu"
@ -274,12 +300,12 @@ jobs:
needs: build-binary-linux
name: "check system | python on debian"
runs-on: ubuntu-latest
container: debian:bullseye
container: debian:bookworm
steps:
- uses: actions/checkout@v4
- name: "Install Python"
run: apt-get update && apt-get install -y python3.9 python3-pip python3.9-venv
run: apt-get update && apt-get install -y python3.11 python3-pip python3.11-venv
- name: "Download binary"
uses: actions/download-artifact@v4
@ -290,16 +316,16 @@ jobs:
run: chmod +x ./uv
- name: "Print Python path"
run: echo $(which python3.9)
run: echo $(which python3.11)
- name: "Validate global Python install"
run: python3.9 scripts/check_system_python.py --uv ./uv
run: python3.11 scripts/check_system_python.py --uv ./uv --externally-managed
system-test-fedora:
needs: build-binary-linux
name: "check system | python on fedora"
runs-on: ubuntu-latest
container: fedora:39
container: fedora:41
steps:
- uses: actions/checkout@v4
@ -346,6 +372,8 @@ jobs:
run: python scripts/check_system_python.py --uv ./uv
system-test-centos:
# https://github.com/astral-sh/uv/issues/2915
if: false
needs: build-binary-linux
name: "check system | python on centos"
runs-on: ubuntu-latest

6
.github/workflows/release.yml vendored
View File

@ -65,7 +65,7 @@ jobs:
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0-prerelease.3/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0/cargo-dist-installer.sh | sh"
# sure would be cool if github gave us proper conditionals...
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
# functionality based on whether this is a pull_request, and whether it's from a fork.
@ -107,7 +107,7 @@ jobs:
submodules: recursive
- name: Install cargo-dist
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0-prerelease.3/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0/cargo-dist-installer.sh | sh"
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
- name: Fetch local artifacts
uses: actions/download-artifact@v4
@ -151,7 +151,7 @@ jobs:
with:
submodules: recursive
- name: Install cargo-dist
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0-prerelease.3/cargo-dist-installer.sh | sh"
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0/cargo-dist-installer.sh | sh"
# Fetch artifacts from scratch-storage
- name: Fetch artifacts
uses: actions/download-artifact@v4

2
.gitignore vendored
View File

@ -8,7 +8,7 @@ target/
target-alpine/
# Bootstrapped Python versions
bin/
/bin/
# These are backup files generated by rustfmt
**/*.rs.bk

4
.pre-commit-config.yaml
View File

@ -12,7 +12,7 @@ repos:
- id: validate-pyproject
- repo: https://github.com/crate-ci/typos
rev: v1.18.2
rev: v1.20.4
hooks:
- id: typos
@ -32,7 +32,7 @@ repos:
types_or: [yaml, json5]
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.2.1
rev: v0.3.5
hooks:
- id: ruff-format
- id: ruff

49
CHANGELOG.md
View File

@ -1,5 +1,54 @@
# Changelog
## 0.1.31
### Bug fixes
- Ignore direct URL distributions in prefetcher ([#2943](https://github.com/astral-sh/uv/pull/2943))
## 0.1.30
### Enhancements
- Show resolution diagnostics after `pip install` ([#2829](https://github.com/astral-sh/uv/pull/2829))
### Performance
- Speed up cold-cache `urllib3`-`boto3`-`botocore` performance with batched prefetching ([#2452](https://github.com/astral-sh/uv/pull/2452))
### Bug fixes
- Backtrack on distributions with invalid metadata ([#2834](https://github.com/astral-sh/uv/pull/2834))
- Include LICENSE files in source distribution ([#2855](https://github.com/astral-sh/uv/pull/2855))
- Respect `--no-build` and `--no-binary` in `--find-links` ([#2826](https://github.com/astral-sh/uv/pull/2826))
- Respect cached local `--find-links` in install plan ([#2907](https://github.com/astral-sh/uv/pull/2907))
- Avoid panic with multiple confirmation handlers ([#2903](https://github.com/astral-sh/uv/pull/2903))
- Use scheme parsing to determine absolute vs. relative URLs ([#2904](https://github.com/astral-sh/uv/pull/2904))
- Remove additional 'because' in resolution failure messages ([#2849](https://github.com/astral-sh/uv/pull/2849))
- Use `miette` when printing `pip sync` resolution failures ([#2848](https://github.com/astral-sh/uv/pull/2848))
## 0.1.29
### Enhancements
- Allow conflicting Git URLs that refer to the same commit SHA ([#2769](https://github.com/astral-sh/uv/pull/2769))
- Allow package lookups across multiple indexes via explicit opt-in (`--index-strategy unsafe-any-match`) ([#2815](https://github.com/astral-sh/uv/pull/2815))
- Allow no-op `--no-compile` flag on CLI ([#2816](https://github.com/astral-sh/uv/pull/2816))
- Upgrade `rs-async-zip` to support data descriptors ([#2809](https://github.com/astral-sh/uv/pull/2809))
### Bug fixes
- Avoid unused extras check in `pip install` for source trees ([#2811](https://github.com/astral-sh/uv/pull/2811))
- Deduplicate editables during install commands ([#2820](https://github.com/astral-sh/uv/pull/2820))
- Fix windows lock race: lock exclusive after all try lock errors ([#2800](https://github.com/astral-sh/uv/pull/2800))
- Preserve `.git` suffixes and casing in Git dependencies ([#2789](https://github.com/astral-sh/uv/pull/2789))
- Respect Git tags and branches that look like short commits ([#2795](https://github.com/astral-sh/uv/pull/2795))
- Enable virtualenv creation on Windows with cpython-x86 ([#2707](https://github.com/astral-sh/uv/pull/2707))
### Documentation
- Document that uv is safe to run concurrently ([#2818](https://github.com/astral-sh/uv/pull/2818))
## 0.1.28
### Enhancements

12
CONTRIBUTING.md
View File

@ -22,12 +22,6 @@ CMake may be installed with Homebrew:
brew install cmake
```
The Python bootstrapping script requires `coreutils` and `zstd`; we recommend installing them with Homebrew:
```shell
brew install coreutils zstd
```
See the [Python](#python) section for instructions on installing the Python versions.
### Windows
@ -45,13 +39,13 @@ Testing uv requires multiple specific Python versions. You can install them into
`<project root>/bin` via our bootstrapping script:
```shell
pipx run scripts/bootstrap/install.py
cargo run -p uv-dev -- fetch-python
```
Alternatively, you can install `zstandard` from PyPI, then run:
You may need to add the versions to your `PATH`:
```shell
python3.12 scripts/bootstrap/install.py
source .env
```
You can configure the bootstrapping directory with `UV_BOOTSTRAP_DIR`.

787
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

44
Cargo.toml
View File

@ -44,21 +44,23 @@ uv-normalize = { path = "crates/uv-normalize" }
uv-requirements = { path = "crates/uv-requirements" }
uv-resolver = { path = "crates/uv-resolver" }
uv-types = { path = "crates/uv-types" }
uv-configuration = { path = "crates/uv-configuration" }
uv-trampoline = { path = "crates/uv-trampoline" }
uv-version = { path = "crates/uv-version" }
uv-virtualenv = { path = "crates/uv-virtualenv" }
uv-warnings = { path = "crates/uv-warnings" }
uv-toolchain = { path = "crates/uv-toolchain" }
anstream = { version = "0.6.13" }
anyhow = { version = "1.0.80" }
async-channel = { version = "2.2.0" }
async-compression = { version = "0.4.6" }
async-trait = { version = "0.1.78" }
async_http_range_reader = { version = "0.7.0" }
async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "d76801da0943de985254fc6255c0e476b57c5836", features = ["deflate"] }
axoupdater = { version = "0.3.1", default-features = false }
async_http_range_reader = { version = "0.7.1" }
async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "1dcb40cfe1bf5325a6fd4bfcf9894db40241f585", features = ["deflate"] }
axoupdater = { version = "0.4.0", default-features = false }
backoff = { version = "0.4.0" }
base64 = { version = "0.21.7" }
base64 = { version = "0.22.0" }
cachedir = { version = "0.3.1" }
cargo-util = { version = "0.2.8" }
chrono = { version = "0.4.31" }
@ -85,13 +87,14 @@ hex = { version = "0.4.3" }
hmac = { version = "0.12.1" }
home = { version = "0.5.9" }
html-escape = { version = "0.2.13" }
http = { version = "0.2.12" }
http = { version = "1.1.0" }
indexmap = { version = "2.2.5" }
indicatif = { version = "0.17.7" }
indoc = { version = "2.0.4" }
itertools = { version = "0.12.1" }
junction = { version = "1.0.0" }
mailparse = { version = "0.14.0" }
md-5 = { version = "0.10.6" }
miette = { version = "7.2.0" }
nanoid = { version = "0.4.0" }
once_cell = { version = "1.19.0" }
@ -106,9 +109,9 @@ rand = { version = "0.8.5" }
rayon = { version = "1.8.0" }
reflink-copy = { version = "0.1.15" }
regex = { version = "1.10.2" }
reqwest = { version = "0.11.23", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls", "rustls-tls-native-roots"] }
reqwest-middleware = { version = "0.2.4" }
reqwest-retry = { version = "0.3.0" }
reqwest = { version = "0.12.3", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls", "rustls-tls-native-roots"] }
reqwest-middleware = { version = "0.3.0" }
reqwest-retry = { version = "0.5.0" }
rkyv = { version = "0.7.43", features = ["strict", "validation"] }
rmp-serde = { version = "1.1.2" }
rust-netrc = { version = "0.1.1" }
@ -120,7 +123,6 @@ serde_json = { version = "1.0.114" }
sha1 = { version = "0.10.6" }
sha2 = { version = "0.10.8" }
sys-info = { version = "0.9.1" }
task-local-extensions = { version = "0.1.4" }
tempfile = { version = "3.9.0" }
textwrap = { version = "0.16.1" }
thiserror = { version = "1.0.56" }
@ -133,7 +135,7 @@ toml = { version = "0.8.12" }
tracing = { version = "0.1.40" }
tracing-durations-export = { version = "0.2.0", features = ["plot"] }
tracing-indicatif = { version = "0.3.6" }
tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] }
tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json", "registry"] }
tracing-tree = { version = "0.3.0" }
unicode-width = { version = "0.1.11" }
unscanny = { version = "0.1.0" }
@ -197,7 +199,7 @@ lto = "thin"
# Config for 'cargo dist'
[workspace.metadata.dist]
# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
cargo-dist-version = "0.13.0-prerelease.3"
cargo-dist-version = "0.13.0"
# CI backends to support
ci = ["github"]
# The installers to generate for each app
@ -207,8 +209,24 @@ windows-archive = ".zip"
# The archive format to use for non-windows builds (defaults .tar.xz)
unix-archive = ".tar.gz"
# Target platforms to build apps for (Rust target-triple syntax)
targets = ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", "i686-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin", "aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl", "i686-unknown-linux-musl", "x86_64-pc-windows-msvc", "i686-pc-windows-msvc", "armv7-unknown-linux-gnueabihf", "powerpc64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "s390x-unknown-linux-gnu", "armv7-unknown-linux-musleabihf", "arm-unknown-linux-musleabihf"]
# Whether to auto-include files like READMEs and CHANGELOGs (default true)
targets = [
"aarch64-apple-darwin",
"aarch64-unknown-linux-gnu",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-musleabihf",
"armv7-unknown-linux-gnueabihf",
"armv7-unknown-linux-musleabihf",
"i686-pc-windows-msvc",
"i686-unknown-linux-gnu",
"i686-unknown-linux-musl",
"powerpc64-unknown-linux-gnu",
"powerpc64le-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-pc-windows-msvc",
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
]# Whether to auto-include files like READMEs and CHANGELOGs (default true)
auto-includes = false
# Whether cargo-dist should create a Github Release or use an existing draft
create-release = true

20
PIP_COMPATIBILITY.md
View File

@ -30,7 +30,7 @@ drawbacks:
target tool the user is expecting to use.
4. It prevents uv from introducing any settings or configuration that don't exist in the target
tool, since otherwise `pip.conf` (or similar) would no longer be usable with `pip`.
5. It can lead user confusion, since uv would be reading settings that don't actually affect its
5. It can lead to user confusion, since uv would be reading settings that don't actually affect its
behavior, and many users may _not_ expect uv to read configuration files intended for other
tools.
@ -107,7 +107,7 @@ the available versions of a given package. However, uv and `pip` differ in how t
packages that exist on multiple indexes.
For example, imagine that a company publishes an internal version of `requests` on a private index
(`--extra-index-url`), but also allow installing packages from PyPI by default. In this case, the
(`--extra-index-url`), but also allows installing packages from PyPI by default. In this case, the
private `requests` would conflict with the public [`requests`](https://pypi.org/project/requests/)
on PyPI.
@ -117,7 +117,7 @@ finds a match. This means that if a package exists on multiple indexes, uv will
candidate versions to those present in the first index that contains the package.
`pip`, meanwhile, will combine the candidate versions from all indexes, and select the best
version from the combined set., though it makes [no guarantees around the order](https://github.com/pypa/pip/issues/5045#issuecomment-369521345)
version from the combined set, though it makes [no guarantees around the order](https://github.com/pypa/pip/issues/5045#issuecomment-369521345)
in which it searches indexes, and expects that packages are unique up to name and version, even
across indexes.
@ -128,6 +128,12 @@ internal package, thus causing the malicious package to be installed instead of
package. See, for example, [the `torchtriton` attack](https://pytorch.org/blog/compromised-nightly-dependency/)
from December 2022.
As of v0.1.29, users can opt in to `pip`-style behavior for multiple indexes via the
`--index-strategy unsafe-any-match` command-line option, or the `UV_INDEX_STRATEGY` environment
variable. When enabled, uv will search for each package across all indexes, and consider all
available versions when resolving dependencies, prioritizing the `--extra-index-url` indexes over
the default index URL. (Versions that are duplicated _across_ indexes will be ignored.)
In the future, uv will support pinning packages to dedicated indexes (see: [#171](https://github.com/astral-sh/uv/issues/171)).
Additionally, [PEP 708](https://peps.python.org/pep-0708/) is a provisional standard that aims to
address the "dependency confusion" issue across package registries and installers.
@ -253,14 +259,6 @@ When uv resolutions differ from `pip` in undesirable ways, it's often a sign tha
are too loose, and that the user should consider tightening them. For example, in the case of
`starlette` and `fastapi`, the user could require `fastapi>=0.110.0`.
## Hash-checking mode
While uv will include hashes via `uv pip compile --generate-hashes`, it does not support
hash-checking mode, which is a feature of `pip` that allows users to verify the integrity of
downloaded packages by checking their hashes against those provided in the `requirements.txt` file.
In the future, uv will support hash-checking mode. For more, see [#474](https://github.com/astral-sh/uv/issues/474).
## `pip check`
At present, `uv pip check` will surface the following diagnostics:

30
README.md
View File

@ -261,8 +261,18 @@ The specifics of uv's caching semantics vary based on the nature of the dependen
- **For Git dependencies**, uv caches based on the fully-resolved Git commit hash. As such,
`uv pip compile` will pin Git dependencies to a specific commit hash when writing the resolved
dependency set.
- **For local dependencies**, uv caches based on the last-modified time of the `setup.py` or
`pyproject.toml` file.
- **For local dependencies**, uv caches based on the last-modified time of the source archive (i.e.,
the local `.whl` or `.tar.gz` file). For directories, uv caches based on the last-modified time of
the `pyproject.toml`, `setup.py`, or `setup.cfg` file.
It's safe to run multiple `uv` commands concurrently, even against the same virtual environment.
uv's cache is designed to be thread-safe and append-only, and thus robust to multiple concurrent
readers and writers. uv applies a file-based lock to the target virtual environment when installing,
to avoid concurrent modifications across processes.
Note that it's _not_ safe to modify the uv cache directly (e.g., `uv cache clean`) while other `uv`
commands are running, and _never_ safe to modify the cache directly (e.g., by removing a file or
directory).
If you're running into caching issues, uv includes a few escape hatches:
@ -443,12 +453,22 @@ uv accepts the following command-line arguments as environment variables:
`lowest-direct`, uv will install the lowest compatible versions of all direct dependencies.
- `UV_PRERELEASE`: Equivalent to the `--prerelease` command-line argument. For example, if set to
`allow`, uv will allow pre-release versions for all dependencies.
- `UV_SYSTEM_PYTHON`: Equivalent to the `--system` command-line argument. If set to `true`, uv
- `UV_SYSTEM_PYTHON`: Equivalent to the `--system` command-line argument. If set to `true`, uv
will use the first Python interpreter found in the system `PATH`.
WARNING: `UV_SYSTEM_PYTHON=true` is intended for use in continuous integration (CI) environments and
should be used with caution, as it can modify the system Python installation.
WARNING: `UV_SYSTEM_PYTHON=true` is intended for use in continuous integration (CI) or
containerized environments and should be used with caution, as modifying the system Python
can lead to unexpected behavior.
- `UV_BREAK_SYSTEM_PACKAGES`: Equivalent to the `--break-system-packages` command-line argument. If
set to `true`, uv will allow the installation of packages that conflict with system-installed
packages.
WARNING: `UV_BREAK_SYSTEM_PACKAGES=true` is intended for use in continuous integration (CI) or
containerized environments and should be used with caution, as modifying the system Python
can lead to unexpected behavior.
- `UV_NATIVE_TLS`: Equivalent to the `--native-tls` command-line argument. If set to `true`, uv
will use the system's trust store instead of the bundled `webpki-roots` crate.
- `UV_INDEX_STRATEGY`: Equivalent to the `--index-strategy` command-line argument. For example, if
set to `unsafe-any-match`, uv will consider versions of a given package available across all
index URLs, rather than limiting its search to the first index URL that contains the package.
In each case, the corresponding command-line argument takes precedence over an environment variable.

10
crates/distribution-types/src/buildable.rs
View File

@ -1,6 +1,7 @@
use std::borrow::Cow;
use std::path::Path;
use pep440_rs::Version;
use url::Url;
use uv_normalize::PackageName;
@ -28,6 +29,15 @@ impl BuildableSource<'_> {
}
}
/// Return the [`Version`] of the source, if available.
pub fn version(&self) -> Option<&Version> {
match self {
Self::Dist(SourceDist::Registry(dist)) => Some(&dist.filename.version),
Self::Dist(_) => None,
Self::Url(_) => None,
}
}
/// Return the [`BuildableSource`] as a [`SourceDist`], if it is a distribution.
pub fn as_dist(&self) -> Option<&SourceDist> {
match self {

48
crates/distribution-types/src/cached.rs
View File

@ -4,9 +4,11 @@ use anyhow::Result;
use distribution_filename::WheelFilename;
use pep508_rs::VerbatimUrl;
use pypi_types::HashDigest;
use uv_normalize::PackageName;
use crate::direct_url::{DirectUrl, LocalFileUrl};
use crate::hash::Hashed;
use crate::{
BuiltDist, Dist, DistributionMetadata, InstalledMetadata, InstalledVersion, Name, SourceDist,
VersionOrUrl,
@ -25,6 +27,7 @@ pub enum CachedDist {
pub struct CachedRegistryDist {
pub filename: WheelFilename,
pub path: PathBuf,
pub hashes: Vec<HashDigest>,
}
#[derive(Debug, Clone)]
@ -33,45 +36,60 @@ pub struct CachedDirectUrlDist {
pub url: VerbatimUrl,
pub path: PathBuf,
pub editable: bool,
pub hashes: Vec<HashDigest>,
}
impl CachedDist {
/// Initialize a [`CachedDist`] from a [`Dist`].
pub fn from_remote(remote: Dist, filename: WheelFilename, path: PathBuf) -> Self {
pub fn from_remote(
remote: Dist,
filename: WheelFilename,
hashes: Vec<HashDigest>,
path: PathBuf,
) -> Self {
match remote {
Dist::Built(BuiltDist::Registry(_dist)) => {
Self::Registry(CachedRegistryDist { filename, path })
}
Dist::Built(BuiltDist::Registry(_dist)) => Self::Registry(CachedRegistryDist {
filename,
path,
hashes,
}),
Dist::Built(BuiltDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Built(BuiltDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Source(SourceDist::Registry(_dist)) => {
Self::Registry(CachedRegistryDist { filename, path })
}
Dist::Source(SourceDist::Registry(_dist)) => Self::Registry(CachedRegistryDist {
filename,
path,
hashes,
}),
Dist::Source(SourceDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Source(SourceDist::Git(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Source(SourceDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: dist.editable,
}),
@ -104,6 +122,7 @@ impl CachedDist {
}
}
/// Returns `true` if the distribution is editable.
pub fn editable(&self) -> bool {
match self {
Self::Registry(_) => false,
@ -111,6 +130,7 @@ impl CachedDist {
}
}
/// Returns the [`WheelFilename`] of the distribution.
pub fn filename(&self) -> &WheelFilename {
match self {
Self::Registry(dist) => &dist.filename,
@ -119,12 +139,24 @@ impl CachedDist {
}
}
impl Hashed for CachedRegistryDist {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}
impl CachedDirectUrlDist {
/// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`url::Url`], and [`Path`].
pub fn from_url(filename: WheelFilename, url: VerbatimUrl, path: PathBuf) -> Self {
pub fn from_url(
filename: WheelFilename,
url: VerbatimUrl,
hashes: Vec<HashDigest>,
path: PathBuf,
) -> Self {
Self {
filename,
url,
hashes,
path,
editable: false,
}

52
crates/distribution-types/src/editable.rs
View File

@ -1,4 +1,6 @@
use std::borrow::Cow;
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;
use std::path::PathBuf;
use url::Url;
@ -41,3 +43,53 @@ impl std::fmt::Display for LocalEditable {
std::fmt::Display::fmt(&self.url, f)
}
}
/// A collection of [`LocalEditable`]s.
#[derive(Debug, Clone)]
pub struct LocalEditables(Vec<LocalEditable>);
impl LocalEditables {
/// Merge and dedupe a list of [`LocalEditable`]s.
///
/// This function will deduplicate any editables that point to identical paths, merging their
/// extras.
pub fn from_editables(editables: impl Iterator<Item = LocalEditable>) -> Self {
let mut map = BTreeMap::new();
for editable in editables {
match map.entry(editable.path.clone()) {
Entry::Vacant(entry) => {
entry.insert(editable);
}
Entry::Occupied(mut entry) => {
let existing = entry.get_mut();
existing.extras.extend(editable.extras);
}
}
}
Self(map.into_values().collect())
}
/// Return the number of editables.
pub fn len(&self) -> usize {
self.0.len()
}
/// Return whether the editables are empty.
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Return the editables as a vector.
pub fn into_vec(self) -> Vec<LocalEditable> {
self.0
}
}
impl IntoIterator for LocalEditables {
type Item = LocalEditable;
type IntoIter = std::vec::IntoIter<LocalEditable>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}

24
crates/distribution-types/src/file.rs
View File

@ -6,7 +6,8 @@ use thiserror::Error;
use url::Url;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
use pypi_types::{DistInfoMetadata, Hashes, Yanked};
use pep508_rs::split_scheme;
use pypi_types::{DistInfoMetadata, HashDigest, Yanked};
/// Error converting [`pypi_types::File`] to [`distribution_type::File`].
#[derive(Debug, Error)]
@ -24,9 +25,9 @@ pub enum FileConversionError {
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
pub struct File {
pub dist_info_metadata: Option<DistInfoMetadata>,
pub dist_info_metadata: bool,
pub filename: String,
pub hashes: Hashes,
pub hashes: Vec<HashDigest>,
pub requires_python: Option<VersionSpecifiers>,
pub size: Option<u64>,
// N.B. We don't use a chrono DateTime<Utc> here because it's a little
@ -42,19 +43,24 @@ impl File {
/// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers
pub fn try_from(file: pypi_types::File, base: &Url) -> Result<Self, FileConversionError> {
Ok(Self {
dist_info_metadata: file.dist_info_metadata,
dist_info_metadata: file
.dist_info_metadata
.as_ref()
.is_some_and(DistInfoMetadata::is_available),
filename: file.filename,
hashes: file.hashes,
hashes: file.hashes.into_digests(),
requires_python: file
.requires_python
.transpose()
.map_err(|err| FileConversionError::RequiresPython(err.line().clone(), err))?,
size: file.size,
upload_time_utc_ms: file.upload_time.map(|dt| dt.timestamp_millis()),
url: if file.url.contains("://") {
FileLocation::AbsoluteUrl(file.url)
} else {
FileLocation::RelativeUrl(base.to_string(), file.url)
url: {
if split_scheme(&file.url).is_some() {
FileLocation::AbsoluteUrl(file.url)
} else {
FileLocation::RelativeUrl(base.to_string(), file.url)
}
},
yanked: file.yanked,
})

84
crates/distribution-types/src/hash.rs Normal file
View File

@ -0,0 +1,84 @@
use pypi_types::{HashAlgorithm, HashDigest};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HashPolicy<'a> {
/// No hash policy is specified.
None,
/// Hashes should be generated (specifically, a SHA-256 hash), but not validated.
Generate,
/// Hashes should be validated against a pre-defined list of hashes. If necessary, hashes should
/// be generated so as to ensure that the archive is valid.
Validate(&'a [HashDigest]),
}
impl<'a> HashPolicy<'a> {
/// Returns `true` if the hash policy is `None`.
pub fn is_none(&self) -> bool {
matches!(self, Self::None)
}
/// Returns `true` if the hash policy is `Generate`.
pub fn is_generate(&self) -> bool {
matches!(self, Self::Generate)
}
/// Returns `true` if the hash policy is `Validate`.
pub fn is_validate(&self) -> bool {
matches!(self, Self::Validate(_))
}
/// Return the algorithms used in the hash policy.
pub fn algorithms(&self) -> Vec<HashAlgorithm> {
match self {
Self::None => vec![],
Self::Generate => vec![HashAlgorithm::Sha256],
Self::Validate(hashes) => {
let mut algorithms = hashes.iter().map(HashDigest::algorithm).collect::<Vec<_>>();
algorithms.sort();
algorithms.dedup();
algorithms
}
}
}
/// Return the digests used in the hash policy.
pub fn digests(&self) -> &[HashDigest] {
match self {
Self::None => &[],
Self::Generate => &[],
Self::Validate(hashes) => hashes,
}
}
}
pub trait Hashed {
/// Return the [`HashDigest`]s for the archive.
fn hashes(&self) -> &[HashDigest];
/// Returns `true` if the archive satisfies the given hash policy.
fn satisfies(&self, hashes: HashPolicy) -> bool {
match hashes {
HashPolicy::None => true,
HashPolicy::Generate => self
.hashes()
.iter()
.any(|hash| hash.algorithm == HashAlgorithm::Sha256),
HashPolicy::Validate(hashes) => self.hashes().iter().any(|hash| hashes.contains(hash)),
}
}
/// Returns `true` if the archive includes a hash for at least one of the given algorithms.
fn has_digests(&self, hashes: HashPolicy) -> bool {
match hashes {
HashPolicy::None => true,
HashPolicy::Generate => self
.hashes()
.iter()
.any(|hash| hash.algorithm == HashAlgorithm::Sha256),
HashPolicy::Validate(hashes) => hashes
.iter()
.map(HashDigest::algorithm)
.any(|algorithm| self.hashes().iter().any(|hash| hash.algorithm == algorithm)),
}
}
}

94
crates/distribution-types/src/id.rs
View File

@ -1,30 +1,66 @@
use std::fmt::{Display, Formatter};
use std::path::PathBuf;
use cache_key::{CanonicalUrl, RepositoryUrl};
use url::Url;
use pep440_rs::Version;
use pypi_types::HashDigest;
use uv_normalize::PackageName;
/// A unique identifier for a package (e.g., `black==23.10.0`).
/// A unique identifier for a package. A package can either be identified by a name (e.g., `black`)
/// or a URL (e.g., `git+https://github.com/psf/black`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum PackageId {
NameVersion(PackageName, Version),
Url(String),
/// The identifier consists of a package name.
Name(PackageName),
/// The identifier consists of a URL.
Url(CanonicalUrl),
}
impl PackageId {
/// Create a new [`PackageId`] from a package name and version.
pub fn from_registry(name: PackageName, version: Version) -> Self {
Self::NameVersion(name, version)
pub fn from_registry(name: PackageName) -> Self {
Self::Name(name)
}
/// Create a new [`PackageId`] from a URL.
pub fn from_url(url: &Url) -> Self {
Self::Url(cache_key::digest(&cache_key::CanonicalUrl::new(url)))
Self::Url(CanonicalUrl::new(url))
}
}
impl Display for PackageId {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Name(name) => write!(f, "{name}"),
Self::Url(url) => write!(f, "{url}"),
}
}
}
/// A unique identifier for a package at a specific version (e.g., `black==23.10.0`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum VersionId {
/// The identifier consists of a package name and version.
NameVersion(PackageName, Version),
/// The identifier consists of a URL.
Url(CanonicalUrl),
}
impl VersionId {
/// Create a new [`VersionId`] from a package name and version.
pub fn from_registry(name: PackageName, version: Version) -> Self {
Self::NameVersion(name, version)
}
/// Create a new [`VersionId`] from a URL.
pub fn from_url(url: &Url) -> Self {
Self::Url(CanonicalUrl::new(url))
}
}
impl Display for VersionId {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::NameVersion(name, version) => write!(f, "{name}-{version}"),
@ -33,33 +69,39 @@ impl Display for PackageId {
}
}
/// A unique identifier for a distribution (e.g., `black-23.10.0-py3-none-any.whl`).
/// A unique resource identifier for the distribution, like a SHA-256 hash of the distribution's
/// contents.
///
/// A distribution is a specific archive of a package at a specific version. For a given package
/// version, there may be multiple distributions, e.g., source distribution, along with
/// multiple binary distributions (wheels) for different platforms. As a concrete example,
/// `black-23.10.0-py3-none-any.whl` would represent a (binary) distribution of the `black` package
/// at version `23.10.0`.
///
/// The distribution ID is used to uniquely identify a distribution. Ideally, the distribution
/// ID should be a hash of the distribution's contents, though in practice, it's only required
/// that the ID is unique within a single invocation of the resolver (and so, e.g., a hash of
/// the URL would also be sufficient).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DistributionId(String);
impl DistributionId {
pub fn new(id: impl Into<String>) -> Self {
Self(id.into())
}
}
impl DistributionId {
pub fn as_str(&self) -> &str {
&self.0
}
pub enum DistributionId {
Url(CanonicalUrl),
PathBuf(PathBuf),
Digest(HashDigest),
AbsoluteUrl(String),
RelativeUrl(String, String),
}
/// A unique identifier for a resource, like a URL or a Git repository.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ResourceId(String);
impl ResourceId {
pub fn new(id: impl Into<String>) -> Self {
Self(id.into())
}
pub enum ResourceId {
Url(RepositoryUrl),
PathBuf(PathBuf),
Digest(HashDigest),
AbsoluteUrl(String),
RelativeUrl(String, String),
}
impl From<&Self> for PackageId {
impl From<&Self> for VersionId {
/// Required for `WaitMap::wait`.
fn from(value: &Self) -> Self {
value.clone()

20
crates/distribution-types/src/index_url.rs
View File

@ -24,6 +24,7 @@ static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
pub enum IndexUrl {
Pypi(VerbatimUrl),
Url(VerbatimUrl),
Path(VerbatimUrl),
}
impl IndexUrl {
@ -32,6 +33,7 @@ impl IndexUrl {
match self {
Self::Pypi(url) => url.raw(),
Self::Url(url) => url.raw(),
Self::Path(url) => url.raw(),
}
}
}
@ -41,6 +43,7 @@ impl Display for IndexUrl {
match self {
Self::Pypi(url) => Display::fmt(url, f),
Self::Url(url) => Display::fmt(url, f),
Self::Path(url) => Display::fmt(url, f),
}
}
}
@ -50,6 +53,7 @@ impl Verbatim for IndexUrl {
match self {
Self::Pypi(url) => url.verbatim(),
Self::Url(url) => url.verbatim(),
Self::Path(url) => url.verbatim(),
}
}
}
@ -83,6 +87,7 @@ impl From<IndexUrl> for Url {
match index {
IndexUrl::Pypi(url) => url.to_url(),
IndexUrl::Url(url) => url.to_url(),
IndexUrl::Path(url) => url.to_url(),
}
}
}
@ -94,6 +99,7 @@ impl Deref for IndexUrl {
match &self {
Self::Pypi(url) => url,
Self::Url(url) => url,
Self::Path(url) => url,
}
}
}
@ -165,15 +171,7 @@ impl Display for FlatIndexLocation {
}
}
/// The index locations to use for fetching packages.
///
/// By default, uses the PyPI index.
///
/// "pip treats all package sources equally" (<https://github.com/pypa/pip/issues/8606#issuecomment-788754817>),
/// and so do we, i.e., you can't rely that on any particular order of querying indices.
///
/// If the fields are none and empty, ignore the package index, instead rely on local archives and
/// caches.
/// The index locations to use for fetching packages. By default, uses the PyPI index.
///
/// From a pip perspective, this type merges `--index-url`, `--extra-index-url`, and `--find-links`.
#[derive(Debug, Clone)]
@ -338,7 +336,9 @@ impl<'a> IndexUrls {
}
}
/// Return an iterator over all [`IndexUrl`] entries.
/// Return an iterator over all [`IndexUrl`] entries in order.
///
/// Prioritizes the extra indexes over the main index.
///
/// If `no_index` was enabled, then this always returns an empty
/// iterator.

97
crates/distribution-types/src/lib.rs
View File

@ -51,6 +51,7 @@ pub use crate::direct_url::*;
pub use crate::editable::*;
pub use crate::error::*;
pub use crate::file::*;
pub use crate::hash::*;
pub use crate::id::*;
pub use crate::index_url::*;
pub use crate::installed::*;
@ -66,6 +67,7 @@ mod direct_url;
mod editable;
mod error;
mod file;
mod hash;
mod id;
mod index_url;
mod installed;
@ -371,6 +373,14 @@ impl Dist {
}
}
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Built(dist) => dist.index(),
Self::Source(dist) => dist.index(),
}
}
/// Returns the [`File`] instance, if this dist is from a registry with simple json api support
pub fn file(&self) -> Option<&File> {
match self {
@ -388,7 +398,16 @@ impl Dist {
}
impl BuiltDist {
/// Returns the [`File`] instance, if this dist is from a registry with simple json api support
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Registry(registry) => Some(&registry.index),
Self::DirectUrl(_) => None,
Self::Path(_) => None,
}
}
/// Returns the [`File`] instance, if this distribution is from a registry.
pub fn file(&self) -> Option<&File> {
match self {
Self::Registry(registry) => Some(&registry.file),
@ -406,6 +425,14 @@ impl BuiltDist {
}
impl SourceDist {
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Registry(registry) => Some(&registry.index),
Self::DirectUrl(_) | Self::Git(_) | Self::Path(_) => None,
}
}
/// Returns the [`File`] instance, if this dist is from a registry with simple json api support
pub fn file(&self) -> Option<&File> {
match self {
@ -764,26 +791,26 @@ impl RemoteSource for Dist {
impl Identifier for Url {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&cache_key::CanonicalUrl::new(self)))
DistributionId::Url(cache_key::CanonicalUrl::new(self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&cache_key::RepositoryUrl::new(self)))
ResourceId::Url(cache_key::RepositoryUrl::new(self))
}
}
impl Identifier for File {
fn distribution_id(&self) -> DistributionId {
if let Some(hash) = self.hashes.as_str() {
DistributionId::new(hash)
if let Some(hash) = self.hashes.first() {
DistributionId::Digest(hash.clone())
} else {
self.url.distribution_id()
}
}
fn resource_id(&self) -> ResourceId {
if let Some(hash) = self.hashes.as_str() {
ResourceId::new(hash)
if let Some(hash) = self.hashes.first() {
ResourceId::Digest(hash.clone())
} else {
self.url.resource_id()
}
@ -792,67 +819,31 @@ impl Identifier for File {
impl Identifier for Path {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
DistributionId::PathBuf(self.to_path_buf())
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for String {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for &str {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for (&str, &str) {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for (&Url, &str) {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
ResourceId::PathBuf(self.to_path_buf())
}
}
impl Identifier for FileLocation {
fn distribution_id(&self) -> DistributionId {
match self {
Self::RelativeUrl(base, url) => (base.as_str(), url.as_str()).distribution_id(),
Self::AbsoluteUrl(url) => url.distribution_id(),
Self::RelativeUrl(base, url) => {
DistributionId::RelativeUrl(base.to_string(), url.to_string())
}
Self::AbsoluteUrl(url) => DistributionId::AbsoluteUrl(url.to_string()),
Self::Path(path) => path.distribution_id(),
}
}
fn resource_id(&self) -> ResourceId {
match self {
Self::RelativeUrl(base, url) => (base.as_str(), url.as_str()).resource_id(),
Self::AbsoluteUrl(url) => url.resource_id(),
Self::RelativeUrl(base, url) => {
ResourceId::RelativeUrl(base.to_string(), url.to_string())
}
Self::AbsoluteUrl(url) => ResourceId::AbsoluteUrl(url.to_string()),
Self::Path(path) => path.resource_id(),
}
}

131
crates/distribution-types/src/prioritized_distribution.rs
View File

@ -1,8 +1,8 @@
use std::fmt::{Display, Formatter};
use pep440_rs::VersionSpecifiers;
use platform_tags::{IncompatibleTag, TagCompatibility, TagPriority};
use pypi_types::{Hashes, Yanked};
use platform_tags::{IncompatibleTag, TagPriority};
use pypi_types::{HashDigest, Yanked};
use crate::{Dist, InstalledDist, ResolvedDistRef};
@ -18,7 +18,7 @@ struct PrioritizedDistInner {
/// The highest-priority wheel.
wheel: Option<(Dist, WheelCompatibility)>,
/// The hashes for each distribution.
hashes: Vec<Hashes>,
hashes: Vec<HashDigest>,
}
/// A distribution that can be used for both resolution and installation.
@ -113,7 +113,7 @@ impl Display for IncompatibleDist {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WheelCompatibility {
Incompatible(IncompatibleWheel),
Compatible(TagPriority),
Compatible(Hash, TagPriority),
}
#[derive(Debug, PartialEq, Eq, Clone)]
@ -128,7 +128,7 @@ pub enum IncompatibleWheel {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SourceDistCompatibility {
Incompatible(IncompatibleSource),
Compatible,
Compatible(Hash),
}
#[derive(Debug, PartialEq, Eq, Clone)]
@ -139,26 +139,40 @@ pub enum IncompatibleSource {
NoBuild,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum Hash {
/// The hash is present, but does not match the expected value.
Mismatched,
/// The hash is missing.
Missing,
/// The hash matches the expected value.
Matched,
}
impl PrioritizedDist {
/// Create a new [`PrioritizedDist`] from the given wheel distribution.
pub fn from_built(dist: Dist, hash: Option<Hashes>, compatibility: WheelCompatibility) -> Self {
pub fn from_built(
dist: Dist,
hashes: Vec<HashDigest>,
compatibility: WheelCompatibility,
) -> Self {
Self(Box::new(PrioritizedDistInner {
wheel: Some((dist, compatibility)),
source: None,
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(),
hashes,
}))
}
/// Create a new [`PrioritizedDist`] from the given source distribution.
pub fn from_source(
dist: Dist,
hash: Option<Hashes>,
hashes: Vec<HashDigest>,
compatibility: SourceDistCompatibility,
) -> Self {
Self(Box::new(PrioritizedDistInner {
wheel: None,
source: Some((dist, compatibility)),
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(),
hashes,
}))
}
@ -166,7 +180,7 @@ impl PrioritizedDist {
pub fn insert_built(
&mut self,
dist: Dist,
hash: Option<Hashes>,
hashes: Vec<HashDigest>,
compatibility: WheelCompatibility,
) {
// Track the highest-priority wheel.
@ -178,16 +192,14 @@ impl PrioritizedDist {
self.0.wheel = Some((dist, compatibility));
}
if let Some(hash) = hash {
self.0.hashes.push(hash);
}
self.0.hashes.extend(hashes);
}
/// Insert the given source distribution into the [`PrioritizedDist`].
pub fn insert_source(
&mut self,
dist: Dist,
hash: Option<Hashes>,
hashes: Vec<HashDigest>,
compatibility: SourceDistCompatibility,
) {
// Track the highest-priority source.
@ -199,16 +211,25 @@ impl PrioritizedDist {
self.0.source = Some((dist, compatibility));
}
if let Some(hash) = hash {
self.0.hashes.push(hash);
}
self.0.hashes.extend(hashes);
}
/// Return the highest-priority distribution for the package version, if any.
pub fn get(&self) -> Option<CompatibleDist> {
match (&self.0.wheel, &self.0.source) {
// If both are compatible, break ties based on the hash.
(
Some((wheel, WheelCompatibility::Compatible(wheel_hash, tag_priority))),
Some((source_dist, SourceDistCompatibility::Compatible(source_hash))),
) => {
if source_hash > wheel_hash {
Some(CompatibleDist::SourceDist(source_dist))
} else {
Some(CompatibleDist::CompatibleWheel(wheel, *tag_priority))
}
}
// Prefer the highest-priority, platform-compatible wheel.
(Some((wheel, WheelCompatibility::Compatible(tag_priority))), _) => {
(Some((wheel, WheelCompatibility::Compatible(_, tag_priority))), _) => {
Some(CompatibleDist::CompatibleWheel(wheel, *tag_priority))
}
// If we have a compatible source distribution and an incompatible wheel, return the
@ -217,64 +238,42 @@ impl PrioritizedDist {
// using the wheel is faster.
(
Some((wheel, WheelCompatibility::Incompatible(_))),
Some((source_dist, SourceDistCompatibility::Compatible)),
Some((source_dist, SourceDistCompatibility::Compatible(_))),
) => Some(CompatibleDist::IncompatibleWheel { source_dist, wheel }),
// Otherwise, if we have a source distribution, return it.
(None, Some((source_dist, SourceDistCompatibility::Compatible))) => {
(None, Some((source_dist, SourceDistCompatibility::Compatible(_)))) => {
Some(CompatibleDist::SourceDist(source_dist))
}
_ => None,
}
}
/// Return the compatible source distribution, if any.
pub fn compatible_source(&self) -> Option<&Dist> {
self.0
.source
.as_ref()
.and_then(|(dist, compatibility)| match compatibility {
SourceDistCompatibility::Compatible => Some(dist),
SourceDistCompatibility::Incompatible(_) => None,
})
}
/// Return the incompatible source distribution, if any.
pub fn incompatible_source(&self) -> Option<(&Dist, &IncompatibleSource)> {
self.0
.source
.as_ref()
.and_then(|(dist, compatibility)| match compatibility {
SourceDistCompatibility::Compatible => None,
SourceDistCompatibility::Compatible(_) => None,
SourceDistCompatibility::Incompatible(incompatibility) => {
Some((dist, incompatibility))
}
})
}
/// Return the compatible built distribution, if any.
pub fn compatible_wheel(&self) -> Option<(&Dist, TagPriority)> {
self.0
.wheel
.as_ref()
.and_then(|(dist, compatibility)| match compatibility {
WheelCompatibility::Compatible(priority) => Some((dist, *priority)),
WheelCompatibility::Incompatible(_) => None,
})
}
/// Return the incompatible built distribution, if any.
pub fn incompatible_wheel(&self) -> Option<(&Dist, &IncompatibleWheel)> {
self.0
.wheel
.as_ref()
.and_then(|(dist, compatibility)| match compatibility {
WheelCompatibility::Compatible(_) => None,
WheelCompatibility::Compatible(_, _) => None,
WheelCompatibility::Incompatible(incompatibility) => Some((dist, incompatibility)),
})
}
/// Return the hashes for each distribution.
pub fn hashes(&self) -> &[Hashes] {
pub fn hashes(&self) -> &[HashDigest] {
&self.0.hashes
}
@ -311,11 +310,23 @@ impl<'a> CompatibleDist<'a> {
} => ResolvedDistRef::Installable(source_dist),
}
}
/// Returns whether the distribution is a source distribution.
///
/// Avoid building source distributions we don't need.
pub fn prefetchable(&self) -> bool {
match *self {
CompatibleDist::SourceDist(_) => false,
CompatibleDist::InstalledDist(_)
| CompatibleDist::CompatibleWheel(_, _)
| CompatibleDist::IncompatibleWheel { .. } => true,
}
}
}
impl WheelCompatibility {
pub fn is_compatible(&self) -> bool {
matches!(self, Self::Compatible(_))
matches!(self, Self::Compatible(_, _))
}
/// Return `true` if the current compatibility is more compatible than another.
@ -324,11 +335,12 @@ impl WheelCompatibility {
/// Compatible wheel ordering is determined by tag priority.
pub fn is_more_compatible(&self, other: &Self) -> bool {
match (self, other) {
(Self::Compatible(_), Self::Incompatible(_)) => true,
(Self::Compatible(tag_priority), Self::Compatible(other_tag_priority)) => {
tag_priority > other_tag_priority
}
(Self::Incompatible(_), Self::Compatible(_)) => false,
(Self::Compatible(_, _), Self::Incompatible(_)) => true,
(
Self::Compatible(hash, tag_priority),
Self::Compatible(other_hash, other_tag_priority),
) => (hash, tag_priority) > (other_hash, other_tag_priority),
(Self::Incompatible(_), Self::Compatible(_, _)) => false,
(Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => {
incompatibility.is_more_compatible(other_incompatibility)
}
@ -344,9 +356,11 @@ impl SourceDistCompatibility {
/// Incompatible source distribution priority selects a source distribution that was "closest" to being usable.
pub fn is_more_compatible(&self, other: &Self) -> bool {
match (self, other) {
(Self::Compatible, Self::Incompatible(_)) => true,
(Self::Compatible, Self::Compatible) => false, // Arbitrary
(Self::Incompatible(_), Self::Compatible) => false,
(Self::Compatible(_), Self::Incompatible(_)) => true,
(Self::Compatible(compatibility), Self::Compatible(other_compatibility)) => {
compatibility > other_compatibility
}
(Self::Incompatible(_), Self::Compatible(_)) => false,
(Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => {
incompatibility.is_more_compatible(other_incompatibility)
}
@ -354,15 +368,6 @@ impl SourceDistCompatibility {
}
}
impl From<TagCompatibility> for WheelCompatibility {
fn from(value: TagCompatibility) -> Self {
match value {
TagCompatibility::Compatible(priority) => Self::Compatible(priority),
TagCompatibility::Incompatible(tag) => Self::Incompatible(IncompatibleWheel::Tag(tag)),
}
}
}
impl IncompatibleSource {
fn is_more_compatible(&self, other: &Self) -> bool {
match self {

23
crates/distribution-types/src/resolved.rs
View File

@ -1,10 +1,10 @@
use std::fmt::Display;
use std::fmt::{Display, Formatter};
use pep508_rs::PackageName;
use crate::{
Dist, DistributionId, DistributionMetadata, Identifier, InstalledDist, Name, ResourceId,
VersionOrUrl,
Dist, DistributionId, DistributionMetadata, Identifier, IndexUrl, InstalledDist, Name,
ResourceId, VersionOrUrl,
};
/// A distribution that can be used for resolution and installation.
@ -31,6 +31,14 @@ impl ResolvedDist {
Self::Installed(dist) => dist.is_editable(),
}
}
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Installable(dist) => dist.index(),
Self::Installed(_) => None,
}
}
}
impl ResolvedDistRef<'_> {
@ -42,6 +50,15 @@ impl ResolvedDistRef<'_> {
}
}
impl Display for ResolvedDistRef<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Installable(dist) => Display::fmt(dist, f),
Self::Installed(dist) => Display::fmt(dist, f),
}
}
}
impl Name for ResolvedDistRef<'_> {
fn name(&self) -> &PackageName {
match self {

35
crates/distribution-types/src/traits.rs
View File

@ -10,7 +10,8 @@ use crate::{
BuiltDist, CachedDirectUrlDist, CachedDist, CachedRegistryDist, DirectUrlBuiltDist,
DirectUrlSourceDist, Dist, DistributionId, GitSourceDist, InstalledDirectUrlDist,
InstalledDist, InstalledRegistryDist, InstalledVersion, LocalDist, PackageId, PathBuiltDist,
PathSourceDist, RegistryBuiltDist, RegistrySourceDist, ResourceId, SourceDist, VersionOrUrl,
PathSourceDist, RegistryBuiltDist, RegistrySourceDist, ResourceId, SourceDist, VersionId,
VersionOrUrl,
};
pub trait Name {
@ -25,16 +26,29 @@ pub trait DistributionMetadata: Name {
/// for URL-based distributions.
fn version_or_url(&self) -> VersionOrUrl;
/// Returns a unique identifier for the package.
/// Returns a unique identifier for the package at the given version (e.g., `black==23.10.0`).
///
/// Note that this is not equivalent to a unique identifier for the _distribution_, as multiple
/// registry-based distributions (e.g., different wheels for the same package and version)
/// will return the same package ID, but different distribution IDs.
fn package_id(&self) -> PackageId {
/// will return the same version ID, but different distribution IDs.
fn version_id(&self) -> VersionId {
match self.version_or_url() {
VersionOrUrl::Version(version) => {
PackageId::from_registry(self.name().clone(), version.clone())
VersionId::from_registry(self.name().clone(), version.clone())
}
VersionOrUrl::Url(url) => VersionId::from_url(url),
}
}
/// Returns a unique identifier for a package. A package can either be identified by a name
/// (e.g., `black`) or a URL (e.g., `git+https://github.com/psf/black`).
///
/// Note that this is not equivalent to a unique identifier for the _distribution_, as multiple
/// registry-based distributions (e.g., different wheels for the same package and version)
/// will return the same version ID, but different distribution IDs.
fn package_id(&self) -> PackageId {
match self.version_or_url() {
VersionOrUrl::Version(_) => PackageId::from_registry(self.name().clone()),
VersionOrUrl::Url(url) => PackageId::from_url(url),
}
}
@ -57,6 +71,17 @@ pub trait RemoteSource {
pub trait Identifier {
/// Return a unique resource identifier for the distribution, like a SHA-256 hash of the
/// distribution's contents.
///
/// A distribution is a specific archive of a package at a specific version. For a given package
/// version, there may be multiple distributions, e.g., source distribution, along with
/// multiple binary distributions (wheels) for different platforms. As a concrete example,
/// `black-23.10.0-py3-none-any.whl` would represent a (binary) distribution of the `black` package
/// at version `23.10.0`.
///
/// The distribution ID is used to uniquely identify a distribution. Ideally, the distribution
/// ID should be a hash of the distribution's contents, though in practice, it's only required
/// that the ID is unique within a single invocation of the resolver (and so, e.g., a hash of
/// the URL would also be sufficient).
fn distribution_id(&self) -> DistributionId;
/// Return a unique resource identifier for the underlying resource backing the distribution.

2
crates/pep508-rs/src/lib.rs
View File

@ -72,7 +72,7 @@ pub enum Pep508ErrorSource {
String(String),
/// A URL parsing error.
#[error(transparent)]
UrlError(#[from] verbatim_url::VerbatimUrlError),
UrlError(#[from] VerbatimUrlError),
/// The version requirement is not supported.
#[error("{0}")]
UnsupportedRequirement(String),

42
crates/pypi-types/src/base_url.rs
View File

@ -1,37 +1,19 @@
use serde::{Deserialize, Serialize};
use url::Url;
/// Join a possibly relative URL to a base URL.
///
/// When `maybe_relative` is not relative, then it is parsed and returned with
/// `base` being ignored.
///
/// This is useful for parsing URLs that may be absolute or relative, with a
/// known base URL, and that doesn't require having already parsed a `BaseUrl`.
pub fn base_url_join_relative(base: &str, maybe_relative: &str) -> Result<Url, JoinRelativeError> {
match Url::parse(maybe_relative) {
Ok(absolute) => Ok(absolute),
Err(err) => {
if err == url::ParseError::RelativeUrlWithoutBase {
let base_url = Url::parse(base).map_err(|err| JoinRelativeError::ParseError {
original: base.to_string(),
source: err,
})?;
/// Join a relative URL to a base URL.
pub fn base_url_join_relative(base: &str, relative: &str) -> Result<Url, JoinRelativeError> {
let base_url = Url::parse(base).map_err(|err| JoinRelativeError::ParseError {
original: base.to_string(),
source: err,
})?;
base_url
.join(maybe_relative)
.map_err(|_| JoinRelativeError::ParseError {
original: format!("{base}/{maybe_relative}"),
source: err,
})
} else {
Err(JoinRelativeError::ParseError {
original: maybe_relative.to_string(),
source: err,
})
}
}
}
base_url
.join(relative)
.map_err(|err| JoinRelativeError::ParseError {
original: format!("{base}/{relative}"),
source: err,
})
}
/// An error that occurs when `base_url_join_relative` fails.

110
crates/pypi-types/src/metadata.rs
View File

@ -1,9 +1,8 @@
//! Derived from `pypi_types_crate`.
use indexmap::IndexMap;
use std::io;
use std::str::FromStr;
use indexmap::IndexMap;
use mailparse::{MailHeaderMap, MailParseError};
use serde::{Deserialize, Serialize};
use thiserror::Error;
@ -39,38 +38,17 @@ pub struct Metadata23 {
///
/// The error type
#[derive(Error, Debug)]
pub enum Error {
/// I/O error
#[error(transparent)]
Io(#[from] io::Error),
/// mail parse error
pub enum MetadataError {
#[error(transparent)]
MailParse(#[from] MailParseError),
/// TOML parse error
#[error(transparent)]
Toml(#[from] toml::de::Error),
/// Metadata field not found
#[error("metadata field {0} not found")]
FieldNotFound(&'static str),
/// Unknown distribution type
#[error("unknown distribution type")]
UnknownDistributionType,
/// Metadata file not found
#[error("metadata file not found")]
MetadataNotFound,
/// Invalid project URL (no comma)
#[error("Invalid Project-URL field (missing comma): '{0}'")]
InvalidProjectUrl(String),
/// Multiple metadata files found
#[error("found multiple metadata files: {0:?}")]
MultipleMetadataFiles(Vec<String>),
/// Invalid Version
#[error("invalid version: {0}")]
Pep440VersionError(VersionParseError),
/// Invalid VersionSpecifier
#[error(transparent)]
Pep440Error(#[from] VersionSpecifiersParseError),
/// Invalid Requirement
#[error(transparent)]
Pep508Error(#[from] Pep508Error),
#[error(transparent)]
@ -86,20 +64,20 @@ pub enum Error {
/// From <https://github.com/PyO3/python-pkginfo-rs/blob/d719988323a0cfea86d4737116d7917f30e819e2/src/metadata.rs#LL78C2-L91C26>
impl Metadata23 {
/// Parse the [`Metadata23`] from a `METADATA` file, as included in a built distribution (wheel).
pub fn parse_metadata(content: &[u8]) -> Result<Self, Error> {
pub fn parse_metadata(content: &[u8]) -> Result<Self, MetadataError> {
let headers = Headers::parse(content)?;
let name = PackageName::new(
headers
.get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?,
.ok_or(MetadataError::FieldNotFound("Name"))?,
)?;
let version = Version::from_str(
&headers
.get_first_value("Version")
.ok_or(Error::FieldNotFound("Version"))?,
.ok_or(MetadataError::FieldNotFound("Version"))?,
)
.map_err(Error::Pep440VersionError)?;
.map_err(MetadataError::Pep440VersionError)?;
let requires_dist = headers
.get_all_values("Requires-Dist")
.map(|requires_dist| {
@ -135,28 +113,28 @@ impl Metadata23 {
/// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2
/// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and
/// `Provides-Extra`) are marked as dynamic.
pub fn parse_pkg_info(content: &[u8]) -> Result<Self, Error> {
pub fn parse_pkg_info(content: &[u8]) -> Result<Self, MetadataError> {
let headers = Headers::parse(content)?;
// To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be
// present and set to a value of at least `2.2`.
let metadata_version = headers
.get_first_value("Metadata-Version")
.ok_or(Error::FieldNotFound("Metadata-Version"))?;
.ok_or(MetadataError::FieldNotFound("Metadata-Version"))?;
// Parse the version into (major, minor).
let (major, minor) = parse_version(&metadata_version)?;
if (major, minor) < (2, 2) || (major, minor) >= (3, 0) {
return Err(Error::UnsupportedMetadataVersion(metadata_version));
return Err(MetadataError::UnsupportedMetadataVersion(metadata_version));
}
// If any of the fields we need are marked as dynamic, we can't use the `PKG-INFO` file.
let dynamic = headers.get_all_values("Dynamic").collect::<Vec<_>>();
for field in dynamic {
match field.as_str() {
"Requires-Python" => return Err(Error::DynamicField("Requires-Python")),
"Requires-Dist" => return Err(Error::DynamicField("Requires-Dist")),
"Provides-Extra" => return Err(Error::DynamicField("Provides-Extra")),
"Requires-Python" => return Err(MetadataError::DynamicField("Requires-Python")),
"Requires-Dist" => return Err(MetadataError::DynamicField("Requires-Dist")),
"Provides-Extra" => return Err(MetadataError::DynamicField("Provides-Extra")),
_ => (),
}
}
@ -165,14 +143,14 @@ impl Metadata23 {
let name = PackageName::new(
headers
.get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?,
.ok_or(MetadataError::FieldNotFound("Name"))?,
)?;
let version = Version::from_str(
&headers
.get_first_value("Version")
.ok_or(Error::FieldNotFound("Version"))?,
.ok_or(MetadataError::FieldNotFound("Version"))?,
)
.map_err(Error::Pep440VersionError)?;
.map_err(MetadataError::Pep440VersionError)?;
// The remaining fields are required to be present.
let requires_dist = headers
@ -208,29 +186,31 @@ impl Metadata23 {
}
/// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621.
pub fn parse_pyproject_toml(contents: &str) -> Result<Self, Error> {
pub fn parse_pyproject_toml(contents: &str) -> Result<Self, MetadataError> {
let pyproject_toml: PyProjectToml = toml::from_str(contents)?;
let project = pyproject_toml
.project
.ok_or(Error::FieldNotFound("project"))?;
.ok_or(MetadataError::FieldNotFound("project"))?;
// If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file.
let dynamic = project.dynamic.unwrap_or_default();
for field in dynamic {
match field.as_str() {
"dependencies" => return Err(Error::DynamicField("dependencies")),
"dependencies" => return Err(MetadataError::DynamicField("dependencies")),
"optional-dependencies" => {
return Err(Error::DynamicField("optional-dependencies"))
return Err(MetadataError::DynamicField("optional-dependencies"))
}
"requires-python" => return Err(Error::DynamicField("requires-python")),
"version" => return Err(Error::DynamicField("version")),
"requires-python" => return Err(MetadataError::DynamicField("requires-python")),
"version" => return Err(MetadataError::DynamicField("version")),
_ => (),
}
}
let name = project.name;
let version = project.version.ok_or(Error::FieldNotFound("version"))?;
let version = project
.version
.ok_or(MetadataError::FieldNotFound("version"))?;
let requires_python = project.requires_python.map(VersionSpecifiers::from);
// Extract the requirements.
@ -309,28 +289,31 @@ pub struct Metadata10 {
impl Metadata10 {
/// Parse the [`Metadata10`] from a `PKG-INFO` file, as included in a source distribution.
pub fn parse_pkg_info(content: &[u8]) -> Result<Self, Error> {
pub fn parse_pkg_info(content: &[u8]) -> Result<Self, MetadataError> {
let headers = Headers::parse(content)?;
let name = PackageName::new(
headers
.get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?,
.ok_or(MetadataError::FieldNotFound("Name"))?,
)?;
Ok(Self { name })
}
}
/// Parse a `Metadata-Version` field into a (major, minor) tuple.
fn parse_version(metadata_version: &str) -> Result<(u8, u8), Error> {
let (major, minor) = metadata_version
.split_once('.')
.ok_or(Error::InvalidMetadataVersion(metadata_version.to_string()))?;
fn parse_version(metadata_version: &str) -> Result<(u8, u8), MetadataError> {
let (major, minor) =
metadata_version
.split_once('.')
.ok_or(MetadataError::InvalidMetadataVersion(
metadata_version.to_string(),
))?;
let major = major
.parse::<u8>()
.map_err(|_| Error::InvalidMetadataVersion(metadata_version.to_string()))?;
.map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?;
let minor = minor
.parse::<u8>()
.map_err(|_| Error::InvalidMetadataVersion(metadata_version.to_string()))?;
.map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?;
Ok((major, minor))
}
@ -373,7 +356,7 @@ mod tests {
use pep440_rs::Version;
use uv_normalize::PackageName;
use crate::Error;
use crate::MetadataError;
use super::Metadata23;
@ -381,11 +364,11 @@ mod tests {
fn test_parse_metadata() {
let s = "Metadata-Version: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Name"))));
assert!(matches!(meta, Err(MetadataError::FieldNotFound("Name"))));
let s = "Metadata-Version: 1.0\nName: asdf";
let meta = Metadata23::parse_metadata(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Version"))));
assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version"))));
let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap();
@ -404,22 +387,25 @@ mod tests {
let s = "Metadata-Version: 1.0\nName: =?utf-8?q?=C3=A4_space?= <x@y.org>\nVersion: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes());
assert!(matches!(meta, Err(Error::InvalidName(_))));
assert!(matches!(meta, Err(MetadataError::InvalidName(_))));
}
#[test]
fn test_parse_pkg_info() {
let s = "Metadata-Version: 2.1";
let meta = Metadata23::parse_pkg_info(s.as_bytes());
assert!(matches!(meta, Err(Error::UnsupportedMetadataVersion(_))));
assert!(matches!(
meta,
Err(MetadataError::UnsupportedMetadataVersion(_))
));
let s = "Metadata-Version: 2.2\nName: asdf";
let meta = Metadata23::parse_pkg_info(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Version"))));
assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version"))));
let s = "Metadata-Version: 2.3\nName: asdf";
let meta = Metadata23::parse_pkg_info(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Version"))));
assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version"))));
let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap();
@ -428,7 +414,7 @@ mod tests {
let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nDynamic: Requires-Dist";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap_err();
assert!(matches!(meta, Error::DynamicField("Requires-Dist")));
assert!(matches!(meta, MetadataError::DynamicField("Requires-Dist")));
let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap();
@ -444,7 +430,7 @@ mod tests {
name = "asdf"
"#;
let meta = Metadata23::parse_pyproject_toml(s);
assert!(matches!(meta, Err(Error::FieldNotFound("version"))));
assert!(matches!(meta, Err(MetadataError::FieldNotFound("version"))));
let s = r#"
[project]
@ -452,7 +438,7 @@ mod tests {
dynamic = ["version"]
"#;
let meta = Metadata23::parse_pyproject_toml(s);
assert!(matches!(meta, Err(Error::DynamicField("version"))));
assert!(matches!(meta, Err(MetadataError::DynamicField("version"))));
let s = r#"
[project]

189
crates/pypi-types/src/simple_json.rs
View File

@ -68,11 +68,7 @@ where
))
}
#[derive(
Debug, Clone, Serialize, Deserialize, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub enum DistInfoMetadata {
Bool(bool),
@ -125,23 +121,7 @@ impl Default for Yanked {
/// A dictionary mapping a hash name to a hex encoded digest of the file.
///
/// PEP 691 says multiple hashes can be included and the interpretation is left to the client.
#[derive(
Debug,
Clone,
Ord,
PartialOrd,
Eq,
PartialEq,
Hash,
Default,
Serialize,
Deserialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
#[derive(Debug, Clone, Eq, PartialEq, Default, Deserialize)]
pub struct Hashes {
pub md5: Option<Box<str>>,
pub sha256: Option<Box<str>>,
@ -150,31 +130,34 @@ pub struct Hashes {
}
impl Hashes {
/// Format as `<algorithm>:<hash>`.
pub fn to_string(&self) -> Option<String> {
self.sha512
.as_ref()
.map(|sha512| format!("sha512:{sha512}"))
.or_else(|| {
self.sha384
.as_ref()
.map(|sha384| format!("sha384:{sha384}"))
})
.or_else(|| {
self.sha256
.as_ref()
.map(|sha256| format!("sha256:{sha256}"))
})
.or_else(|| self.md5.as_ref().map(|md5| format!("md5:{md5}")))
}
/// Return the hash digest.
pub fn as_str(&self) -> Option<&str> {
self.sha512
.as_deref()
.or(self.sha384.as_deref())
.or(self.sha256.as_deref())
.or(self.md5.as_deref())
/// Convert a set of [`Hashes`] into a list of [`HashDigest`]s.
pub fn into_digests(self) -> Vec<HashDigest> {
let mut digests = Vec::new();
if let Some(sha512) = self.sha512 {
digests.push(HashDigest {
algorithm: HashAlgorithm::Sha512,
digest: sha512,
});
}
if let Some(sha384) = self.sha384 {
digests.push(HashDigest {
algorithm: HashAlgorithm::Sha384,
digest: sha384,
});
}
if let Some(sha256) = self.sha256 {
digests.push(HashDigest {
algorithm: HashAlgorithm::Sha256,
digest: sha256,
});
}
if let Some(md5) = self.md5 {
digests.push(HashDigest {
algorithm: HashAlgorithm::Md5,
digest: md5,
});
}
digests
}
}
@ -239,6 +222,118 @@ impl FromStr for Hashes {
}
}
#[derive(
Debug,
Clone,
Copy,
Ord,
PartialOrd,
Eq,
PartialEq,
Hash,
Serialize,
Deserialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
pub enum HashAlgorithm {
Md5,
Sha256,
Sha384,
Sha512,
}
impl FromStr for HashAlgorithm {
type Err = HashError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"md5" => Ok(Self::Md5),
"sha256" => Ok(Self::Sha256),
"sha384" => Ok(Self::Sha384),
"sha512" => Ok(Self::Sha512),
_ => Err(HashError::UnsupportedHashAlgorithm(s.to_string())),
}
}
}
impl std::fmt::Display for HashAlgorithm {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Md5 => write!(f, "md5"),
Self::Sha256 => write!(f, "sha256"),
Self::Sha384 => write!(f, "sha384"),
Self::Sha512 => write!(f, "sha512"),
}
}
}
/// A hash name and hex encoded digest of the file.
#[derive(
Debug,
Clone,
Ord,
PartialOrd,
Eq,
PartialEq,
Hash,
Serialize,
Deserialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
pub struct HashDigest {
pub algorithm: HashAlgorithm,
pub digest: Box<str>,
}
impl HashDigest {
/// Return the [`HashAlgorithm`] of the digest.
pub fn algorithm(&self) -> HashAlgorithm {
self.algorithm
}
}
impl std::fmt::Display for HashDigest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}:{}", self.algorithm, self.digest)
}
}
impl FromStr for HashDigest {
type Err = HashError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut parts = s.split(':');
// Extract the key and value.
let name = parts
.next()
.ok_or_else(|| HashError::InvalidStructure(s.to_string()))?;
let value = parts
.next()
.ok_or_else(|| HashError::InvalidStructure(s.to_string()))?;
// Ensure there are no more parts.
if parts.next().is_some() {
return Err(HashError::InvalidStructure(s.to_string()));
}
let algorithm = HashAlgorithm::from_str(name)?;
Ok(HashDigest {
algorithm,
digest: value.to_owned().into_boxed_str(),
})
}
}
#[derive(thiserror::Error, Debug)]
pub enum HashError {
#[error("Unexpected hash (expected `<algorithm>:<hash>`): {0}")]

2
crates/requirements-txt/Cargo.toml
View File

@ -17,7 +17,7 @@ pep508_rs = { workspace = true, features = ["rkyv", "serde", "non-pep508-extensi
uv-client = { workspace = true }
uv-fs = { workspace = true }
uv-normalize = { workspace = true }
uv-types = { workspace = true }
uv-configuration = { workspace = true }
uv-warnings = { workspace = true }
fs-err = { workspace = true }

12
crates/requirements-txt/src/lib.rs
View File

@ -52,9 +52,9 @@ use pep508_rs::{
#[cfg(feature = "http")]
use uv_client::BaseClient;
use uv_client::BaseClientBuilder;
use uv_configuration::{NoBinary, NoBuild, PackageNameSpecifier};
use uv_fs::{normalize_url_path, Simplified};
use uv_normalize::ExtraName;
use uv_types::{NoBinary, NoBuild, PackageNameSpecifier};
use uv_warnings::warn_user;
/// We emit one of those for each requirements.txt entry
@ -293,21 +293,16 @@ impl Display for EditableRequirement {
/// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in
/// the future also editable an similar information
#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Serialize)]
#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct RequirementEntry {
/// The actual PEP 508 requirement
pub requirement: RequirementsTxtRequirement,
/// Hashes of the downloadable packages
pub hashes: Vec<String>,
/// Editable installation, see e.g. <https://stackoverflow.com/q/35064426/3549270>
pub editable: bool,
}
impl Display for RequirementEntry {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if self.editable {
write!(f, "-e ")?;
}
write!(f, "{}", self.requirement)?;
for hash in &self.hashes {
write!(f, " --hash {hash}")?;
@ -670,7 +665,6 @@ fn parse_entry(
RequirementsTxtStatement::RequirementEntry(RequirementEntry {
requirement,
hashes,
editable: false,
})
} else if let Some(char) = s.peek() {
let (line, column) = calculate_row_column(content, s.cursor());
@ -1743,7 +1737,6 @@ mod test {
},
),
hashes: [],
editable: false,
},
],
constraints: [],
@ -1798,7 +1791,6 @@ mod test {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

6
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -77,7 +75,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -102,7 +99,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -127,7 +123,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -152,7 +147,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -57,7 +56,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -66,7 +64,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -99,7 +96,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -41,7 +40,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

5
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap
View File

@ -56,7 +56,6 @@ RequirementsTxt {
hashes: [
"sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -110,7 +109,6 @@ RequirementsTxt {
hashes: [
"sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -175,7 +173,6 @@ RequirementsTxt {
hashes: [
"sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -230,7 +227,6 @@ RequirementsTxt {
"sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5",
"sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -287,7 +283,6 @@ RequirementsTxt {
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e",
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a",
],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -57,7 +56,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

6
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -77,7 +75,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -102,7 +99,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -127,7 +123,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -152,7 +147,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -66,7 +64,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -99,7 +96,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -41,7 +40,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

5
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap
View File

@ -56,7 +56,6 @@ RequirementsTxt {
hashes: [
"sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -110,7 +109,6 @@ RequirementsTxt {
hashes: [
"sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -175,7 +173,6 @@ RequirementsTxt {
hashes: [
"sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -230,7 +227,6 @@ RequirementsTxt {
"sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5",
"sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a",
],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -287,7 +283,6 @@ RequirementsTxt {
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e",
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a",
],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

3
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-bare-url.txt.snap
View File

@ -28,7 +28,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Unnamed(
@ -58,7 +57,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Unnamed(
@ -84,7 +82,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Pep508(
@ -57,7 +56,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

3
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-bare-url.txt.snap
View File

@ -28,7 +28,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Unnamed(
@ -58,7 +57,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
RequirementEntry {
requirement: Unnamed(
@ -84,7 +82,6 @@ RequirementsTxt {
},
),
hashes: [],
editable: false,
},
],
constraints: [],

4
crates/uv-auth/Cargo.toml
View File

@ -7,15 +7,15 @@ edition = "2021"
async-trait = { workspace = true }
base64 = { workspace = true }
clap = { workspace = true, features = ["derive", "env"], optional = true }
http = { workspace = true }
once_cell = { workspace = true }
reqwest = { workspace = true }
reqwest-middleware = { workspace = true }
rust-netrc = { workspace = true }
task-local-extensions = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true }
once_cell = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }

2
crates/uv-auth/src/middleware.rs
View File

@ -1,9 +1,9 @@
use http::Extensions;
use std::path::Path;
use netrc::Netrc;
use reqwest::{header::HeaderValue, Request, Response};
use reqwest_middleware::{Middleware, Next};
use task_local_extensions::Extensions;
use tracing::{debug, warn};
use crate::{

1
crates/uv-build/Cargo.toml
View File

@ -20,6 +20,7 @@ pep508_rs = { workspace = true }
uv-fs = { workspace = true }
uv-interpreter = { workspace = true }
uv-types = { workspace = true, features = ["serde"] }
uv-configuration = { workspace = true, features = ["serde"] }
uv-virtualenv = { workspace = true }
anyhow = { workspace = true }

53
crates/uv-build/src/lib.rs
View File

@ -28,11 +28,10 @@ use tracing::{debug, info_span, instrument, Instrument};
use distribution_types::Resolution;
use pep440_rs::Version;
use pep508_rs::{PackageName, Requirement};
use uv_configuration::{BuildKind, ConfigSettings, SetupPyStrategy};
use uv_fs::{PythonExt, Simplified};
use uv_interpreter::{Interpreter, PythonEnvironment};
use uv_types::{
BuildContext, BuildIsolation, BuildKind, ConfigSettings, SetupPyStrategy, SourceBuildTrait,
};
use uv_types::{BuildContext, BuildIsolation, SourceBuildTrait};
/// e.g. `pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory`
static MISSING_HEADER_RE: Lazy<Regex> = Lazy::new(|| {
@ -113,7 +112,7 @@ pub enum MissingLibrary {
#[derive(Debug, Error)]
pub struct MissingHeaderCause {
missing_library: MissingLibrary,
package_id: String,
version_id: String,
}
impl Display for MissingHeaderCause {
@ -123,22 +122,22 @@ impl Display for MissingHeaderCause {
write!(
f,
"This error likely indicates that you need to install a library that provides \"{}\" for {}",
header, self.package_id
header, self.version_id
)
}
MissingLibrary::Linker(library) => {
write!(
f,
"This error likely indicates that you need to install the library that provides a shared library \
for {library} for {package_id} (e.g. lib{library}-dev)",
library = library, package_id = self.package_id
for {library} for {version_id} (e.g. lib{library}-dev)",
library = library, version_id = self.version_id
)
}
MissingLibrary::PythonPackage(package) => {
write!(
f,
"This error likely indicates that you need to `uv pip install {package}` into the build environment for {package_id}",
package = package, package_id = self.package_id
"This error likely indicates that you need to `uv pip install {package}` into the build environment for {version_id}",
package = package, version_id = self.version_id
)
}
}
@ -149,7 +148,7 @@ impl Error {
fn from_command_output(
message: String,
output: &Output,
package_id: impl Into<String>,
version_id: impl Into<String>,
) -> Self {
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
@ -179,7 +178,7 @@ impl Error {
stderr,
missing_header_cause: MissingHeaderCause {
missing_library,
package_id: package_id.into(),
version_id: version_id.into(),
},
};
}
@ -365,7 +364,7 @@ pub struct SourceBuild {
/// > it created.
metadata_directory: Option<PathBuf>,
/// Package id such as `foo-1.2.3`, for error reporting
package_id: String,
version_id: String,
/// Whether we do a regular PEP 517 build or an PEP 660 editable build
build_kind: BuildKind,
/// Modified PATH that contains the `venv_bin`, `user_path` and `system_path` variables in that order
@ -386,7 +385,7 @@ impl SourceBuild {
interpreter: &Interpreter,
build_context: &impl BuildContext,
source_build_context: SourceBuildContext,
package_id: String,
version_id: String,
setup_py: SetupPyStrategy,
config_settings: ConfigSettings,
build_isolation: BuildIsolation<'_>,
@ -478,7 +477,7 @@ impl SourceBuild {
&venv,
pep517_backend,
build_context,
&package_id,
&version_id,
build_kind,
&config_settings,
&environment_variables,
@ -498,7 +497,7 @@ impl SourceBuild {
build_kind,
config_settings,
metadata_directory: None,
package_id,
version_id,
environment_variables,
modified_path,
})
@ -696,7 +695,7 @@ impl SourceBuild {
return Err(Error::from_command_output(
"Build backend failed to determine metadata through `prepare_metadata_for_build_wheel`".to_string(),
&output,
&self.package_id,
&self.version_id,
));
}
@ -715,8 +714,8 @@ impl SourceBuild {
/// dir.
///
/// <https://packaging.python.org/en/latest/specifications/source-distribution-format/>
#[instrument(skip_all, fields(package_id = self.package_id))]
pub async fn build(&self, wheel_dir: &Path) -> Result<String, Error> {
#[instrument(skip_all, fields(version_id = self.version_id))]
pub async fn build_wheel(&self, wheel_dir: &Path) -> Result<String, Error> {
// The build scripts run with the extracted root as cwd, so they need the absolute path.
let wheel_dir = fs::canonicalize(wheel_dir)?;
@ -751,7 +750,7 @@ impl SourceBuild {
return Err(Error::from_command_output(
"Failed building wheel through setup.py".to_string(),
&output,
&self.package_id,
&self.version_id,
));
}
let dist = fs::read_dir(self.source_tree.join("dist"))?;
@ -762,7 +761,7 @@ impl SourceBuild {
"Expected exactly wheel in `dist/` after invoking setup.py, found {dist_dir:?}"
),
&output,
&self.package_id)
&self.version_id)
);
};
@ -832,7 +831,7 @@ impl SourceBuild {
self.build_kind
),
&output,
&self.package_id,
&self.version_id,
));
}
@ -844,7 +843,7 @@ impl SourceBuild {
self.build_kind
),
&output,
&self.package_id,
&self.version_id,
));
}
Ok(distribution_filename)
@ -857,7 +856,7 @@ impl SourceBuildTrait for SourceBuild {
}
async fn wheel<'a>(&'a self, wheel_dir: &'a Path) -> anyhow::Result<String> {
Ok(self.build(wheel_dir).await?)
Ok(self.build_wheel(wheel_dir).await?)
}
}
@ -874,7 +873,7 @@ async fn create_pep517_build_environment(
venv: &PythonEnvironment,
pep517_backend: &Pep517Backend,
build_context: &impl BuildContext,
package_id: &str,
version_id: &str,
build_kind: BuildKind,
config_settings: &ConfigSettings,
environment_variables: &FxHashMap<OsString, OsString>,
@ -928,7 +927,7 @@ async fn create_pep517_build_environment(
return Err(Error::from_command_output(
format!("Build backend failed to determine extra requires with `build_{build_kind}()`"),
&output,
package_id,
version_id,
));
}
@ -939,7 +938,7 @@ async fn create_pep517_build_environment(
"Build backend failed to read extra requires from `get_requires_for_build_{build_kind}`: {err}"
),
&output,
package_id,
version_id,
)
})?;
@ -950,7 +949,7 @@ async fn create_pep517_build_environment(
"Build backend failed to return extra requires with `get_requires_for_build_{build_kind}`: {err}"
),
&output,
package_id,
version_id,
)
})?;

24
crates/uv-cache/src/archive.rs Normal file
View File

@ -0,0 +1,24 @@
use std::path::Path;
/// A unique identifier for an archive (unzipped wheel) in the cache.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ArchiveId(String);
impl Default for ArchiveId {
fn default() -> Self {
Self::new()
}
}
impl ArchiveId {
/// Generate a new unique identifier for an archive.
pub fn new() -> Self {
Self(nanoid::nanoid!())
}
}
impl AsRef<Path> for ArchiveId {
fn as_ref(&self) -> &Path {
self.0.as_ref()
}
}

35
crates/uv-cache/src/lib.rs
View File

@ -23,7 +23,9 @@ use crate::removal::{rm_rf, Removal};
pub use crate::timestamp::Timestamp;
pub use crate::wheel::WheelCache;
use crate::wheel::WheelCacheKind;
pub use archive::ArchiveId;
mod archive;
mod by_timestamp;
#[cfg(feature = "clap")]
mod cli;
@ -71,6 +73,12 @@ impl CacheEntry {
}
}
impl AsRef<Path> for CacheEntry {
fn as_ref(&self) -> &Path {
&self.0
}
}
/// A subdirectory within the cache.
#[derive(Debug, Clone)]
pub struct CacheShard(PathBuf);
@ -167,6 +175,11 @@ impl Cache {
CacheEntry::new(self.bucket(cache_bucket).join(dir), file)
}
/// Return the path to an archive in the cache.
pub fn archive(&self, id: &ArchiveId) -> PathBuf {
self.bucket(CacheBucket::Archive).join(id)
}
/// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
pub fn must_revalidate(&self, package: &PackageName) -> bool {
match &self.refresh {
@ -208,18 +221,18 @@ impl Cache {
}
}
/// Persist a temporary directory to the artifact store.
/// Persist a temporary directory to the artifact store, returning its unique ID.
pub async fn persist(
&self,
temp_dir: impl AsRef<Path>,
path: impl AsRef<Path>,
) -> io::Result<PathBuf> {
) -> io::Result<ArchiveId> {
// Create a unique ID for the artifact.
// TODO(charlie): Support content-addressed persistence via SHAs.
let id = nanoid::nanoid!();
let id = ArchiveId::new();
// Move the temporary directory into the directory store.
let archive_entry = self.entry(CacheBucket::Archive, "", id);
let archive_entry = self.entry(CacheBucket::Archive, "", &id);
fs_err::create_dir_all(archive_entry.dir())?;
uv_fs::rename_with_retry(temp_dir.as_ref(), archive_entry.path()).await?;
@ -227,7 +240,7 @@ impl Cache {
fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
uv_fs::replace_symlink(archive_entry.path(), path.as_ref())?;
Ok(archive_entry.into_path_buf())
Ok(id)
}
/// Initialize a directory for use as a cache.
@ -594,12 +607,12 @@ pub enum CacheBucket {
impl CacheBucket {
fn to_str(self) -> &'static str {
match self {
Self::BuiltWheels => "built-wheels-v2",
Self::BuiltWheels => "built-wheels-v3",
Self::FlatIndex => "flat-index-v0",
Self::Git => "git-v0",
Self::Interpreter => "interpreter-v0",
Self::Simple => "simple-v6",
Self::Wheels => "wheels-v0",
Self::Simple => "simple-v7",
Self::Wheels => "wheels-v1",
Self::Archive => "archive-v0",
}
}
@ -795,6 +808,12 @@ impl ArchiveTimestamp {
}
}
/// Return the modification timestamp for a file.
pub fn from_file(path: impl AsRef<Path>) -> Result<Self, io::Error> {
let metadata = fs_err::metadata(path.as_ref())?;
Ok(Self::Exact(Timestamp::from_metadata(&metadata)))
}
/// Return the modification timestamp for an archive.
pub fn timestamp(&self) -> Timestamp {
match self {

17
crates/uv-client/Cargo.toml
View File

@ -11,13 +11,14 @@ install-wheel-rs = { workspace = true }
pep440_rs = { workspace = true }
pep508_rs = { workspace = true }
platform-tags = { workspace = true }
pypi-types = { workspace = true }
uv-auth = { workspace = true }
uv-cache = { workspace = true }
uv-configuration = { workspace = true }
uv-fs = { workspace = true, features = ["tokio"] }
uv-normalize = { workspace = true }
uv-version = { workspace = true }
uv-warnings = { workspace = true }
pypi-types = { workspace = true }
anyhow = { workspace = true }
async-trait = { workspace = true }
@ -33,11 +34,9 @@ reqwest-middleware = { workspace = true }
reqwest-retry = { workspace = true }
rkyv = { workspace = true }
rmp-serde = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sys-info = { workspace = true }
task-local-extensions = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tl = { workspace = true }
@ -47,14 +46,10 @@ tracing = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true }
# These must be kept in-sync with those used by `reqwest`.
rustls = { version = "0.21.10" }
rustls-native-certs = { version = "0.6.3" }
webpki-roots = { version = "0.25.4" }
[dev-dependencies]
anyhow = { workspace = true }
hyper = { version = "0.14.28", features = ["server", "http1"] }
insta = { version = "1.36.1" }
os_info = { version = "3.7.0", default-features = false }
http-body-util = { version = "0.1.0" }
hyper = { version = "1.2.0", features = ["server", "http1"] }
hyper-util = { version = "0.1.3", features = ["tokio"] }
insta = { version = "1.36.1" , features = ["filters", "json", "redactions"] }
tokio = { workspace = true, features = ["fs", "macros"] }

22
crates/uv-client/src/base_client.rs
View File

@ -16,10 +16,9 @@ use uv_warnings::warn_user_once;
use crate::linehaul::LineHaul;
use crate::middleware::OfflineMiddleware;
use crate::tls::Roots;
use crate::{tls, Connectivity};
use crate::Connectivity;
/// A builder for an [`RegistryClient`].
/// A builder for an [`BaseClient`].
#[derive(Debug, Clone)]
pub struct BaseClientBuilder<'a> {
keyring_provider: KeyringProvider,
@ -140,19 +139,20 @@ impl<'a> BaseClientBuilder<'a> {
}
path_exists
});
// Load the TLS configuration.
let tls = tls::load(if self.native_tls || ssl_cert_file_exists {
Roots::Native
} else {
Roots::Webpki
})
.expect("Failed to load TLS configuration.");
// Configure the builder.
let client_core = ClientBuilder::new()
.user_agent(user_agent_string)
.pool_max_idle_per_host(20)
.timeout(std::time::Duration::from_secs(timeout))
.use_preconfigured_tls(tls);
.tls_built_in_root_certs(false);
// Configure TLS.
let client_core = if self.native_tls || ssl_cert_file_exists {
client_core.tls_built_in_native_certs(true)
} else {
client_core.tls_built_in_webpki_certs(true)
};
client_core.build().expect("Failed to build HTTP client.")
});

28
crates/uv-client/src/cached_client.rs
View File

@ -299,6 +299,34 @@ impl CachedClient {
}
}
/// Make a request without checking whether the cache is fresh.
pub async fn skip_cache<
Payload: Serialize + DeserializeOwned + Send + 'static,
CallBackError,
Callback,
CallbackReturn,
>(
&self,
req: Request,
cache_entry: &CacheEntry,
response_callback: Callback,
) -> Result<Payload, CachedClientError<CallBackError>>
where
Callback: FnOnce(Response) -> CallbackReturn + Send,
CallbackReturn: Future<Output = Result<Payload, CallBackError>> + Send,
{
let (response, cache_policy) = self.fresh_request(req).await?;
let payload = self
.run_response_callback(cache_entry, cache_policy, response, move |resp| async {
let payload = response_callback(resp).await?;
Ok(SerdeCacheable { inner: payload })
})
.await?;
Ok(payload)
}
async fn resend_and_heal_cache<Payload: Cacheable, CallBackError, Callback, CallbackReturn>(
&self,
req: Request,

8
crates/uv-client/src/error.rs
View File

@ -132,7 +132,7 @@ pub enum ErrorKind {
/// Dist-info error
#[error(transparent)]
InstallWheel(#[from] install_wheel_rs::Error),
DistInfo(#[from] install_wheel_rs::Error),
#[error("{0} isn't available locally, but making network requests to registries was banned.")]
NoIndex(String),
@ -146,7 +146,11 @@ pub enum ErrorKind {
/// The metadata file could not be parsed.
#[error("Couldn't parse metadata of {0} from {1}")]
MetadataParseError(WheelFilename, String, #[source] Box<pypi_types::Error>),
MetadataParseError(
WheelFilename,
String,
#[source] Box<pypi_types::MetadataError>,
),
/// The metadata file was not found in the wheel.
#[error("Metadata file `{0}` was not found in {1}")]

153
crates/uv-client/src/flat_index.rs
View File

@ -1,24 +1,14 @@
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;
use std::path::PathBuf;
use futures::{FutureExt, StreamExt};
use reqwest::Response;
use rustc_hash::FxHashMap;
use tracing::{debug, info_span, instrument, warn, Instrument};
use tracing::{debug, info_span, warn, Instrument};
use url::Url;
use distribution_filename::DistFilename;
use distribution_types::{
BuiltDist, Dist, File, FileLocation, FlatIndexLocation, IndexUrl, PrioritizedDist,
RegistryBuiltDist, RegistrySourceDist, SourceDist, SourceDistCompatibility,
};
use pep440_rs::Version;
use distribution_types::{File, FileLocation, FlatIndexLocation, IndexUrl};
use pep508_rs::VerbatimUrl;
use platform_tags::Tags;
use pypi_types::Hashes;
use uv_cache::{Cache, CacheBucket};
use uv_normalize::PackageName;
use crate::cached_client::{CacheControl, CachedClientError};
use crate::html::SimpleHtml;
@ -36,10 +26,10 @@ pub enum FlatIndexError {
#[derive(Debug, Default, Clone)]
pub struct FlatIndexEntries {
/// The list of `--find-links` entries.
entries: Vec<(DistFilename, File, IndexUrl)>,
pub entries: Vec<(DistFilename, File, IndexUrl)>,
/// Whether any `--find-links` entries could not be resolved due to a lack of network
/// connectivity.
offline: bool,
pub offline: bool,
}
impl FlatIndexEntries {
@ -215,7 +205,7 @@ impl<'a> FlatIndexClient<'a> {
fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
// Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?;
let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path));
let index_url = IndexUrl::Path(VerbatimUrl::from_path(&path));
let mut dists = Vec::new();
for entry in fs_err::read_dir(path)? {
@ -234,9 +224,9 @@ impl<'a> FlatIndexClient<'a> {
};
let file = File {
dist_info_metadata: None,
dist_info_metadata: false,
filename: filename.to_string(),
hashes: Hashes::default(),
hashes: Vec::new(),
requires_python: None,
size: None,
upload_time_utc_ms: None,
@ -256,132 +246,3 @@ impl<'a> FlatIndexClient<'a> {
Ok(FlatIndexEntries::from_entries(dists))
}
}
/// A set of [`PrioritizedDist`] from a `--find-links` entry, indexed by [`PackageName`]
/// and [`Version`].
#[derive(Debug, Clone, Default)]
pub struct FlatIndex {
/// The list of [`FlatDistributions`] from the `--find-links` entries, indexed by package name.
index: FxHashMap<PackageName, FlatDistributions>,
/// Whether any `--find-links` entries could not be resolved due to a lack of network
/// connectivity.
offline: bool,
}
impl FlatIndex {
/// Collect all files from a `--find-links` target into a [`FlatIndex`].
#[instrument(skip_all)]
pub fn from_entries(entries: FlatIndexEntries, tags: &Tags) -> Self {
// Collect compatible distributions.
let mut index = FxHashMap::default();
for (filename, file, url) in entries.entries {
let distributions = index.entry(filename.name().clone()).or_default();
Self::add_file(distributions, file, filename, tags, url);
}
// Collect offline entries.
let offline = entries.offline;
Self { index, offline }
}
fn add_file(
distributions: &mut FlatDistributions,
file: File,
filename: DistFilename,
tags: &Tags,
index: IndexUrl,
) {
// No `requires-python` here: for source distributions, we don't have that information;
// for wheels, we read it lazily only when selected.
match filename {
DistFilename::WheelFilename(filename) => {
let compatibility = filename.compatibility(tags);
let version = filename.version.clone();
let dist = Dist::Built(BuiltDist::Registry(RegistryBuiltDist {
filename,
file: Box::new(file),
index,
}));
match distributions.0.entry(version) {
Entry::Occupied(mut entry) => {
entry
.get_mut()
.insert_built(dist, None, compatibility.into());
}
Entry::Vacant(entry) => {
entry.insert(PrioritizedDist::from_built(
dist,
None,
compatibility.into(),
));
}
}
}
DistFilename::SourceDistFilename(filename) => {
let dist = Dist::Source(SourceDist::Registry(RegistrySourceDist {
filename: filename.clone(),
file: Box::new(file),
index,
}));
match distributions.0.entry(filename.version) {
Entry::Occupied(mut entry) => {
entry.get_mut().insert_source(
dist,
None,
SourceDistCompatibility::Compatible,
);
}
Entry::Vacant(entry) => {
entry.insert(PrioritizedDist::from_source(
dist,
None,
SourceDistCompatibility::Compatible,
));
}
}
}
}
}
/// Get the [`FlatDistributions`] for the given package name.
pub fn get(&self, package_name: &PackageName) -> Option<&FlatDistributions> {
self.index.get(package_name)
}
/// Returns `true` if there are any offline `--find-links` entries.
pub fn offline(&self) -> bool {
self.offline
}
}
/// A set of [`PrioritizedDist`] from a `--find-links` entry for a single package, indexed
/// by [`Version`].
#[derive(Debug, Clone, Default)]
pub struct FlatDistributions(BTreeMap<Version, PrioritizedDist>);
impl FlatDistributions {
pub fn iter(&self) -> impl Iterator<Item = (&Version, &PrioritizedDist)> {
self.0.iter()
}
pub fn remove(&mut self, version: &Version) -> Option<PrioritizedDist> {
self.0.remove(version)
}
}
impl IntoIterator for FlatDistributions {
type Item = (Version, PrioritizedDist);
type IntoIter = std::collections::btree_map::IntoIter<Version, PrioritizedDist>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}
impl From<FlatDistributions> for BTreeMap<Version, PrioritizedDist> {
fn from(distributions: FlatDistributions) -> Self {
distributions.0
}
}

57
crates/uv-client/src/html.rs
View File

@ -164,6 +164,9 @@ impl SimpleHtml {
.last()
.ok_or_else(|| Error::MissingFilename(href.to_string()))?;
// Strip any query string from the filename.
let filename = filename.split('?').next().unwrap_or(filename);
// Unquote the filename.
let filename = urlencoding::decode(filename)
.map_err(|_| Error::UnsupportedFilename(filename.to_string()))?;
@ -681,6 +684,60 @@ mod tests {
"###);
}
#[test]
fn parse_query_string() {
let text = r#"
<!DOCTYPE html>
<html>
<body>
<h1>Links for jinja2</h1>
<a href="/whl/Jinja2-3.1.2-py3-none-any.whl?project=legacy">Jinja2-3.1.2-py3-none-any.whl</a><br/>
</body>
</html>
<!--TIMESTAMP 1703347410-->
"#;
let base = Url::parse("https://download.pytorch.org/whl/jinja2/").unwrap();
let result = SimpleHtml::parse(text, &base).unwrap();
insta::assert_debug_snapshot!(result, @r###"
SimpleHtml {
base: BaseUrl(
Url {
scheme: "https",
cannot_be_a_base: false,
username: "",
password: None,
host: Some(
Domain(
"download.pytorch.org",
),
),
port: None,
path: "/whl/jinja2/",
query: None,
fragment: None,
},
),
files: [
File {
dist_info_metadata: None,
filename: "Jinja2-3.1.2-py3-none-any.whl",
hashes: Hashes {
md5: None,
sha256: None,
sha384: None,
sha512: None,
},
requires_python: None,
size: None,
upload_time: None,
url: "/whl/Jinja2-3.1.2-py3-none-any.whl?project=legacy",
yanked: None,
},
],
}
"###);
}
#[test]
fn parse_missing_hash_value() {
let text = r#"

3
crates/uv-client/src/lib.rs
View File

@ -1,7 +1,7 @@
pub use base_client::{BaseClient, BaseClientBuilder};
pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy};
pub use error::{BetterReqwestError, Error, ErrorKind};
pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError};
pub use flat_index::{FlatIndexClient, FlatIndexEntries, FlatIndexError};
pub use linehaul::LineHaul;
pub use registry_client::{
Connectivity, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum,
@ -20,4 +20,3 @@ mod middleware;
mod registry_client;
mod remote_metadata;
mod rkyvutil;
mod tls;

2
crates/uv-client/src/middleware.rs
View File

@ -1,8 +1,8 @@
use http::Extensions;
use std::fmt::Debug;
use reqwest::{Request, Response};
use reqwest_middleware::{Middleware, Next};
use task_local_extensions::Extensions;
use url::Url;
/// A custom error type for the offline middleware.

61
crates/uv-client/src/registry_client.rs
View File

@ -9,7 +9,7 @@ use http::HeaderMap;
use reqwest::{Client, Response, StatusCode};
use serde::{Deserialize, Serialize};
use tokio::io::AsyncReadExt;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
use tracing::{info_span, instrument, trace, warn, Instrument};
use url::Url;
@ -22,6 +22,7 @@ use platform_tags::Platform;
use pypi_types::{Metadata23, SimpleJson};
use uv_auth::KeyringProvider;
use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_configuration::IndexStrategy;
use uv_normalize::PackageName;
use crate::base_client::{BaseClient, BaseClientBuilder};
@ -35,6 +36,7 @@ use crate::{CachedClient, CachedClientError, Error, ErrorKind};
#[derive(Debug, Clone)]
pub struct RegistryClientBuilder<'a> {
index_urls: IndexUrls,
index_strategy: IndexStrategy,
keyring_provider: KeyringProvider,
native_tls: bool,
retries: u32,
@ -49,6 +51,7 @@ impl RegistryClientBuilder<'_> {
pub fn new(cache: Cache) -> Self {
Self {
index_urls: IndexUrls::default(),
index_strategy: IndexStrategy::default(),
keyring_provider: KeyringProvider::default(),
native_tls: false,
cache,
@ -68,6 +71,12 @@ impl<'a> RegistryClientBuilder<'a> {
self
}
#[must_use]
pub fn index_strategy(mut self, index_strategy: IndexStrategy) -> Self {
self.index_strategy = index_strategy;
self
}
#[must_use]
pub fn keyring_provider(mut self, keyring_provider: KeyringProvider) -> Self {
self.keyring_provider = keyring_provider;
@ -147,6 +156,7 @@ impl<'a> RegistryClientBuilder<'a> {
RegistryClient {
index_urls: self.index_urls,
index_strategy: self.index_strategy,
cache: self.cache,
connectivity,
client,
@ -160,6 +170,8 @@ impl<'a> RegistryClientBuilder<'a> {
pub struct RegistryClient {
/// The index URLs to use for fetching packages.
index_urls: IndexUrls,
/// The strategy to use when fetching across multiple indexes.
index_strategy: IndexStrategy,
/// The underlying HTTP client.
client: CachedClient,
/// Used for the remote wheel METADATA cache.
@ -206,17 +218,23 @@ impl RegistryClient {
pub async fn simple(
&self,
package_name: &PackageName,
) -> Result<(IndexUrl, OwnedArchive<SimpleMetadata>), Error> {
) -> Result<Vec<(IndexUrl, OwnedArchive<SimpleMetadata>)>, Error> {
let mut it = self.index_urls.indexes().peekable();
if it.peek().is_none() {
return Err(ErrorKind::NoIndex(package_name.as_ref().to_string()).into());
}
let mut results = Vec::new();
for index in it {
let result = self.simple_single_index(package_name, index).await?;
match self.simple_single_index(package_name, index).await? {
Ok(metadata) => {
results.push((index.clone(), metadata));
return match result {
Ok(metadata) => Ok((index.clone(), metadata)),
// If we're only using the first match, we can stop here.
if self.index_strategy == IndexStrategy::FirstMatch {
break;
}
}
Err(CachedClientError::Client(err)) => match err.into_kind() {
ErrorKind::Offline(_) => continue,
ErrorKind::ReqwestError(err) => {
@ -225,20 +243,24 @@ impl RegistryClient {
{
continue;
}
Err(ErrorKind::from(err).into())
return Err(ErrorKind::from(err).into());
}
other => Err(other.into()),
other => return Err(other.into()),
},
Err(CachedClientError::Callback(err)) => Err(err),
Err(CachedClientError::Callback(err)) => return Err(err),
};
}
match self.connectivity {
Connectivity::Online => {
Err(ErrorKind::PackageNotFound(package_name.to_string()).into())
}
Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()),
if results.is_empty() {
return match self.connectivity {
Connectivity::Online => {
Err(ErrorKind::PackageNotFound(package_name.to_string()).into())
}
Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()),
};
}
Ok(results)
}
async fn simple_single_index(
@ -263,6 +285,7 @@ impl RegistryClient {
Path::new(&match index {
IndexUrl::Pypi(_) => "pypi".to_string(),
IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
IndexUrl::Path(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
}),
format!("{package_name}.rkyv"),
);
@ -402,11 +425,7 @@ impl RegistryClient {
) -> Result<Metadata23, Error> {
// If the metadata file is available at its own url (PEP 658), download it from there.
let filename = WheelFilename::from_str(&file.filename).map_err(ErrorKind::WheelFilename)?;
if file
.dist_info_metadata
.as_ref()
.is_some_and(pypi_types::DistInfoMetadata::is_available)
{
if file.dist_info_metadata {
let mut url = url.clone();
url.set_path(&format!("{}.metadata", url.path()));
@ -596,7 +615,8 @@ async fn read_metadata_async_seek(
debug_source: String,
reader: impl tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin,
) -> Result<Metadata23, Error> {
let mut zip_reader = async_zip::tokio::read::seek::ZipFileReader::with_tokio(reader)
let reader = futures::io::BufReader::new(reader.compat());
let mut zip_reader = async_zip::base::read::seek::ZipFileReader::new(reader)
.await
.map_err(|err| ErrorKind::Zip(filename.clone(), err))?;
@ -609,7 +629,7 @@ async fn read_metadata_async_seek(
.enumerate()
.filter_map(|(index, entry)| Some((index, entry.filename().as_str().ok()?))),
)
.map_err(ErrorKind::InstallWheel)?;
.map_err(ErrorKind::DistInfo)?;
// Read the contents of the `METADATA` file.
let mut contents = Vec::new();
@ -633,6 +653,7 @@ async fn read_metadata_async_stream<R: futures::AsyncRead + Unpin>(
debug_source: String,
reader: R,
) -> Result<Metadata23, Error> {
let reader = futures::io::BufReader::with_capacity(128 * 1024, reader);
let mut zip = async_zip::base::read::stream::ZipFileReader::new(reader);
while let Some(mut entry) = zip

8
crates/uv-client/src/remote_metadata.rs
View File

@ -1,5 +1,5 @@
use async_http_range_reader::AsyncHttpRangeReader;
use async_zip::tokio::read::seek::ZipFileReader;
use futures::io::BufReader;
use tokio_util::compat::TokioAsyncReadCompatExt;
use distribution_filename::WheelFilename;
@ -61,7 +61,8 @@ pub(crate) async fn wheel_metadata_from_remote_zip(
.await;
// Construct a zip reader to uses the stream.
let mut reader = ZipFileReader::new(reader.compat())
let buf = BufReader::new(reader.compat());
let mut reader = async_zip::base::read::seek::ZipFileReader::new(buf)
.await
.map_err(|err| ErrorKind::Zip(filename.clone(), err))?;
@ -74,7 +75,7 @@ pub(crate) async fn wheel_metadata_from_remote_zip(
.enumerate()
.filter_map(|(idx, e)| Some(((idx, e), e.filename().as_str().ok()?))),
)
.map_err(ErrorKind::InstallWheel)?;
.map_err(ErrorKind::DistInfo)?;
let offset = metadata_entry.header_offset();
let size = metadata_entry.compressed_size()
@ -90,6 +91,7 @@ pub(crate) async fn wheel_metadata_from_remote_zip(
reader
.inner_mut()
.get_mut()
.get_mut()
.prefetch(offset..offset + size)
.await;

102
crates/uv-client/src/tls.rs
View File

@ -1,102 +0,0 @@
use rustls::ClientConfig;
use tracing::warn;
#[derive(thiserror::Error, Debug)]
pub(crate) enum TlsError {
#[error(transparent)]
Rustls(#[from] rustls::Error),
#[error("zero valid certificates found in native root store")]
ZeroCertificates,
#[error("failed to load native root certificates")]
NativeCertificates(#[source] std::io::Error),
}
#[derive(Debug, Clone, Copy)]
pub(crate) enum Roots {
/// Use reqwest's `rustls-tls-webpki-roots` behavior for loading root certificates.
Webpki,
/// Use reqwest's `rustls-tls-native-roots` behavior for loading root certificates.
Native,
}
/// Initialize a TLS configuration for the client.
///
/// This is equivalent to the TLS initialization `reqwest` when `rustls-tls` is enabled,
/// with two notable changes:
///
/// 1. It enables _either_ the `webpki-roots` or the `native-certs` feature, but not both.
/// 2. It assumes the following builder settings (which match the defaults):
/// - `root_certs: vec![]`
/// - `min_tls_version: None`
/// - `max_tls_version: None`
/// - `identity: None`
/// - `certs_verification: false`
/// - `tls_sni: true`
/// - `http_version_pref: HttpVersionPref::All`
///
/// See: <https://github.com/seanmonstar/reqwest/blob/e3192638518d577759dd89da489175b8f992b12f/src/async_impl/client.rs#L498>
pub(crate) fn load(roots: Roots) -> Result<ClientConfig, TlsError> {
// Set root certificates.
let mut root_cert_store = rustls::RootCertStore::empty();
match roots {
Roots::Webpki => {
// Use `rustls-tls-webpki-roots`
use rustls::OwnedTrustAnchor;
let trust_anchors = webpki_roots::TLS_SERVER_ROOTS.iter().map(|trust_anchor| {
OwnedTrustAnchor::from_subject_spki_name_constraints(
trust_anchor.subject,
trust_anchor.spki,
trust_anchor.name_constraints,
)
});
root_cert_store.add_trust_anchors(trust_anchors);
}
Roots::Native => {
// Use: `rustls-tls-native-roots`
let mut valid_count = 0;
let mut invalid_count = 0;
for cert in
rustls_native_certs::load_native_certs().map_err(TlsError::NativeCertificates)?
{
let cert = rustls::Certificate(cert.0);
// Continue on parsing errors, as native stores often include ancient or syntactically
// invalid certificates, like root certificates without any X509 extensions.
// Inspiration: https://github.com/rustls/rustls/blob/633bf4ba9d9521a95f68766d04c22e2b01e68318/rustls/src/anchors.rs#L105-L112
match root_cert_store.add(&cert) {
Ok(_) => valid_count += 1,
Err(err) => {
invalid_count += 1;
warn!(
"rustls failed to parse DER certificate {:?} {:?}",
&err, &cert
);
}
}
}
if valid_count == 0 && invalid_count > 0 {
return Err(TlsError::ZeroCertificates);
}
}
}
// Build TLS config
let config_builder = ClientConfig::builder()
.with_safe_default_cipher_suites()
.with_safe_default_kx_groups()
.with_protocol_versions(rustls::ALL_VERSIONS)?
.with_root_certificates(root_cert_store);
// Finalize TLS config
let mut tls = config_builder.with_no_client_auth();
// Enable SNI
tls.enable_sni = true;
// ALPN protocol
tls.alpn_protocols = vec!["h2".into(), "http/1.1".into()];
Ok(tls)
}

30
crates/uv-client/tests/netrc_auth.rs
View File

@ -3,10 +3,13 @@ use std::io::Write;
use anyhow::Result;
use futures::future;
use hyper::header::AUTHORIZATION;
use hyper::server::conn::Http;
use http::header::AUTHORIZATION;
use http_body_util::Full;
use hyper::body::Bytes;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Body, Request, Response};
use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use tempfile::NamedTempFile;
use tokio::net::TcpListener;
@ -21,7 +24,7 @@ async fn test_client_with_netrc_credentials() -> Result<()> {
// Spawn the server loop in a background task
tokio::spawn(async move {
let svc = service_fn(move |req: Request<Body>| {
let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
// Get User Agent Header and send it back in the response
let auth = req
.headers()
@ -29,16 +32,19 @@ async fn test_client_with_netrc_credentials() -> Result<()> {
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.unwrap_or_default(); // Empty Default
future::ok::<_, hyper::Error>(Response::new(Body::from(auth)))
future::ok::<_, hyper::Error>(Response::new(Full::new(Bytes::from(auth))))
});
// Start Hyper Server
// Start Server (not wrapped in loop {} since we want a single response server)
// If you want server to accept multiple connections, wrap it in loop {}
let (socket, _) = listener.accept().await.unwrap();
Http::new()
.http1_keep_alive(false)
.serve_connection(socket, svc)
.with_upgrades()
.await
.expect("Server Started");
let socket = TokioIo::new(socket);
tokio::task::spawn(async move {
http1::Builder::new()
.serve_connection(socket, svc)
.with_upgrades()
.await
.expect("Server Started");
});
});
// Create a netrc file

190
crates/uv-client/tests/user_agent_version.rs
View File

@ -1,9 +1,13 @@
use anyhow::Result;
use futures::future;
use http_body_util::Full;
use hyper::body::Bytes;
use hyper::header::USER_AGENT;
use hyper::server::conn::Http;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Body, Request, Response};
use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use insta::{assert_json_snapshot, assert_snapshot, with_settings};
use pep508_rs::{MarkerEnvironment, StringVersion};
use platform_tags::{Arch, Os, Platform};
use tokio::net::TcpListener;
@ -19,8 +23,8 @@ async fn test_user_agent_has_version() -> Result<()> {
let addr = listener.local_addr()?;
// Spawn the server loop in a background task
tokio::spawn(async move {
let svc = service_fn(move |req: Request<Body>| {
let server_task = tokio::spawn(async move {
let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
// Get User Agent Header and send it back in the response
let user_agent = req
.headers()
@ -28,16 +32,19 @@ async fn test_user_agent_has_version() -> Result<()> {
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.unwrap_or_default(); // Empty Default
future::ok::<_, hyper::Error>(Response::new(Body::from(user_agent)))
future::ok::<_, hyper::Error>(Response::new(Full::new(Bytes::from(user_agent))))
});
// Start Hyper Server
// Start Server (not wrapped in loop {} since we want a single response server)
// If you want server to accept multiple connections, wrap it in loop {}
let (socket, _) = listener.accept().await.unwrap();
Http::new()
.http1_keep_alive(false)
.serve_connection(socket, svc)
.with_upgrades()
.await
.expect("Server Started");
let socket = TokioIo::new(socket);
tokio::task::spawn(async move {
http1::Builder::new()
.serve_connection(socket, svc)
.with_upgrades()
.await
.expect("Server Started");
});
});
// Initialize uv-client
@ -46,7 +53,8 @@ async fn test_user_agent_has_version() -> Result<()> {
// Send request to our dummy server
let res = client
.uncached_client()
.cached_client()
.uncached()
.get(format!("http://{addr}"))
.send()
.await?;
@ -60,6 +68,9 @@ async fn test_user_agent_has_version() -> Result<()> {
// Verify body matches regex
assert_eq!(body, format!("uv/{}", version()));
// Wait for the server task to complete, to be a good citizen.
server_task.await?;
Ok(())
}
@ -70,8 +81,8 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
let addr = listener.local_addr()?;
// Spawn the server loop in a background task
tokio::spawn(async move {
let svc = service_fn(move |req: Request<Body>| {
let server_task = tokio::spawn(async move {
let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
// Get User Agent Header and send it back in the response
let user_agent = req
.headers()
@ -79,16 +90,19 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.unwrap_or_default(); // Empty Default
future::ok::<_, hyper::Error>(Response::new(Body::from(user_agent)))
future::ok::<_, hyper::Error>(Response::new(Full::new(Bytes::from(user_agent))))
});
// Start Hyper Server
// Start Server (not wrapped in loop {} since we want a single response server)
// If you want server to accept multiple connections, wrap it in loop {}
let (socket, _) = listener.accept().await.unwrap();
Http::new()
.http1_keep_alive(false)
.serve_connection(socket, svc)
.with_upgrades()
.await
.expect("Server Started");
let socket = TokioIo::new(socket);
tokio::task::spawn(async move {
http1::Builder::new()
.serve_connection(socket, svc)
.with_upgrades()
.await
.expect("Server Started");
});
});
// Add some representative markers for an Ubuntu CI runner
@ -142,7 +156,8 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
// Send request to our dummy server
let res = client
.uncached_client()
.cached_client()
.uncached()
.get(format!("http://{addr}"))
.send()
.await?;
@ -153,6 +168,9 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
// Check User Agent
let body = res.text().await?;
// Wait for the server task to complete, to be a good citizen.
server_task.await?;
// Unpack User-Agent with linehaul
let (uv_version, uv_linehaul) = body
.split_once(' ')
@ -161,62 +179,82 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
// Deserializing Linehaul
let linehaul: LineHaul = serde_json::from_str(uv_linehaul)?;
// Assert uv version
assert_eq!(uv_version, format!("uv/{}", version()));
// Assert linehaul
let installer_info = linehaul.installer.unwrap();
let system_info = linehaul.system.unwrap();
let impl_info = linehaul.implementation.unwrap();
assert_eq!(installer_info.name.unwrap(), "uv".to_string());
assert_eq!(installer_info.version.unwrap(), version());
assert_eq!(system_info.name.unwrap(), markers.platform_system);
assert_eq!(system_info.release.unwrap(), markers.platform_release);
assert_eq!(
impl_info.name.unwrap(),
markers.platform_python_implementation
);
assert_eq!(
impl_info.version.unwrap(),
markers.python_full_version.version.to_string()
);
assert_eq!(
linehaul.python.unwrap(),
markers.python_full_version.version.to_string()
);
assert_eq!(linehaul.cpu.unwrap(), markers.platform_machine);
assert_eq!(linehaul.openssl_version, None);
assert_eq!(linehaul.setuptools_version, None);
assert_eq!(linehaul.rustc_version, None);
// Assert linehaul user agent
let filters = vec![(version(), "[VERSION]")];
with_settings!({
filters => filters
}, {
// Assert uv version
assert_snapshot!(uv_version, @"uv/[VERSION]");
// Assert linehaul json
assert_json_snapshot!(&linehaul, {
".distro" => "[distro]",
".ci" => "[ci]"
}, @r###"
{
"installer": {
"name": "uv",
"version": "[VERSION]"
},
"python": "3.12.2",
"implementation": {
"name": "CPython",
"version": "3.12.2"
},
"distro": "[distro]",
"system": {
"name": "Linux",
"release": "6.5.0-1016-azure"
},
"cpu": "x86_64",
"openssl_version": null,
"setuptools_version": null,
"rustc_version": null,
"ci": "[ci]"
}
"###);
});
// Assert distro
if cfg!(windows) {
assert_eq!(linehaul.distro, None);
assert_json_snapshot!(&linehaul.distro, @"null");
} else if cfg!(target_os = "linux") {
// Using `os_info` to confirm our values are as expected in Linux
let info = os_info::get();
let Some(distro_info) = linehaul.distro else {
panic!("got no distro, but expected one in linehaul")
};
assert_eq!(distro_info.id.as_deref(), info.codename());
if let Some(ref name) = distro_info.name {
assert_eq!(name, &info.os_type().to_string());
}
if let Some(ref version) = distro_info.version {
assert_eq!(version, &info.version().to_string());
}
assert!(distro_info.libc.is_some());
assert_json_snapshot!(&linehaul.distro, {
".id" => "[distro.id]",
".name" => "[distro.name]",
".version" => "[distro.version]"
// We mock the libc version already
}, @r###"
{
"name": "[distro.name]",
"version": "[distro.version]",
"id": "[distro.id]",
"libc": {
"lib": "glibc",
"version": "2.38"
}
}"###
);
// Check dynamic values
let distro_info = linehaul
.distro
.expect("got no distro, but expected one in linehaul");
// Gather distribution info from /etc/os-release.
let release_info = sys_info::linux_os_release()
.expect("got no os release info, but expected one in linux");
assert_eq!(distro_info.id, release_info.version_codename);
assert_eq!(distro_info.name, release_info.name);
assert_eq!(distro_info.version, release_info.version_id);
} else if cfg!(target_os = "macos") {
// We mock the macOS version
let distro_info = linehaul.distro.unwrap();
assert_eq!(distro_info.id, None);
assert_eq!(distro_info.name.unwrap(), "macOS");
assert_eq!(distro_info.version, Some("14.4".to_string()));
assert_eq!(distro_info.libc, None);
// We mock the macOS distro
assert_json_snapshot!(&linehaul.distro, @r###"
{
"name": "macOS",
"version": "14.4",
"id": null,
"libc": null
}"###
);
}
Ok(())

28
crates/uv-configuration/Cargo.toml Normal file
View File

@ -0,0 +1,28 @@
[package]
name = "uv-configuration"
version = "0.0.1"
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
[lints]
workspace = true
[dependencies]
pep508_rs = { workspace = true }
uv-normalize = { workspace = true }
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
itertools = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
[features]
default = []
serde = ["dep:serde", "dep:serde_json"]

38
crates/uv-types/src/build_options.rs → crates/uv-configuration/src/build_options.rs
View File

@ -1,24 +1,9 @@
use std::fmt::{Display, Formatter};
use pep508_rs::PackageName;
use uv_interpreter::PythonEnvironment;
use crate::{PackageNameSpecifier, PackageNameSpecifiers};
/// Whether to enforce build isolation when building source distributions.
#[derive(Debug, Copy, Clone)]
pub enum BuildIsolation<'a> {
Isolated,
Shared(&'a PythonEnvironment),
}
impl<'a> BuildIsolation<'a> {
/// Returns `true` if build isolation is enforced.
pub fn is_isolated(&self) -> bool {
matches!(self, Self::Isolated)
}
}
/// The strategy to use when building source distributions that lack a `pyproject.toml`.
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
pub enum SetupPyStrategy {
@ -211,6 +196,29 @@ impl NoBuild {
}
}
#[derive(Debug, Default, Clone, Hash, Eq, PartialEq)]
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
pub enum IndexStrategy {
/// Only use results from the first index that returns a match for a given package name.
///
/// While this differs from pip's behavior, it's the default index strategy as it's the most
/// secure.
#[default]
FirstMatch,
/// Search for every package name across all indexes, exhausting the versions from the first
/// index before moving on to the next.
///
/// In this strategy, we look for every package across all indexes. When resolving, we attempt
/// to use versions from the indexes in order, such that we exhaust all available versions from
/// the first index before moving on to the next. Further, if a version is found to be
/// incompatible in the first index, we do not reconsider that version in subsequent indexes,
/// even if the secondary index might contain compatible versions (e.g., variants of the same
/// versions with different ABI tags or Python version constraints).
///
/// See: https://peps.python.org/pep-0708/
UnsafeAnyMatch,
}
#[cfg(test)]
mod tests {
use std::str::FromStr;

1
crates/uv-types/src/config_settings.rs → crates/uv-configuration/src/config_settings.rs
View File

@ -40,6 +40,7 @@ enum ConfigSettingValue {
///
/// See: <https://peps.python.org/pep-0517/#config-settings>
#[derive(Debug, Default, Clone)]
#[cfg_attr(not(feature = "serde"), allow(dead_code))]
pub struct ConfigSettings(BTreeMap<String, ConfigSettingValue>);
impl FromIterator<ConfigSettingEntry> for ConfigSettings {

0
crates/uv-types/src/constraints.rs → crates/uv-configuration/src/constraints.rs
View File

13
crates/uv-configuration/src/lib.rs Normal file
View File

@ -0,0 +1,13 @@
pub use build_options::*;
pub use config_settings::*;
pub use constraints::*;
pub use name_specifiers::*;
pub use overrides::*;
pub use package_options::*;
mod build_options;
mod config_settings;
mod constraints;
mod name_specifiers;
mod overrides;
mod package_options;

0
crates/uv-types/src/name_specifiers.rs → crates/uv-configuration/src/name_specifiers.rs
View File

0
crates/uv-types/src/overrides.rs → crates/uv-configuration/src/overrides.rs
View File

0
crates/uv-types/src/package_options.rs → crates/uv-configuration/src/package_options.rs
View File

3
crates/uv-dev/Cargo.toml
View File

@ -23,11 +23,14 @@ pep508_rs = { workspace = true }
uv-build = { workspace = true }
uv-cache = { workspace = true, features = ["clap"] }
uv-client = { workspace = true }
uv-configuration = { workspace = true }
uv-dispatch = { workspace = true }
uv-fs = { workspace = true }
uv-installer = { workspace = true }
uv-interpreter = { workspace = true }
uv-normalize = { workspace = true }
uv-resolver = { workspace = true }
uv-toolchain = { workspace = true }
uv-types = { workspace = true }
# Any dependencies that are exclusively used in `uv-dev` should be listed as non-workspace

12
crates/uv-dev/src/build.rs
View File

@ -9,14 +9,12 @@ use distribution_types::IndexLocations;
use rustc_hash::FxHashMap;
use uv_build::{SourceBuild, SourceBuildContext};
use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, RegistryClientBuilder};
use uv_client::RegistryClientBuilder;
use uv_configuration::{BuildKind, ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch;
use uv_interpreter::PythonEnvironment;
use uv_resolver::InMemoryIndex;
use uv_types::NoBinary;
use uv_types::{
BuildContext, BuildIsolation, BuildKind, ConfigSettings, InFlight, NoBuild, SetupPyStrategy,
};
use uv_resolver::{FlatIndex, InMemoryIndex};
use uv_types::{BuildContext, BuildIsolation, InFlight};
#[derive(Parser)]
pub(crate) struct BuildArgs {
@ -93,5 +91,5 @@ pub(crate) async fn build(args: BuildArgs) -> Result<PathBuf> {
FxHashMap::default(),
)
.await?;
Ok(wheel_dir.join(builder.build(&wheel_dir).await?))
Ok(wheel_dir.join(builder.build_wheel(&wheel_dir).await?))
}

142
crates/uv-dev/src/fetch_python.rs Normal file
View File

@ -0,0 +1,142 @@
use anyhow::Result;
use clap::Parser;
use fs_err as fs;
#[cfg(unix)]
use fs_err::tokio::symlink;
use futures::StreamExt;
#[cfg(unix)]
use itertools::Itertools;
use std::str::FromStr;
#[cfg(unix)]
use std::{collections::HashMap, path::PathBuf};
use tokio::time::Instant;
use tracing::{info, info_span, Instrument};
use uv_fs::Simplified;
use uv_toolchain::{
DownloadResult, Error, PythonDownload, PythonDownloadRequest, TOOLCHAIN_DIRECTORY,
};
#[derive(Parser, Debug)]
pub(crate) struct FetchPythonArgs {
versions: Vec<String>,
}
pub(crate) async fn fetch_python(args: FetchPythonArgs) -> Result<()> {
let start = Instant::now();
let bootstrap_dir = &*TOOLCHAIN_DIRECTORY;
fs_err::create_dir_all(bootstrap_dir)?;
let versions = if args.versions.is_empty() {
info!("Reading versions from file...");
read_versions_file().await?
} else {
args.versions
};
let requests = versions
.iter()
.map(|version| {
PythonDownloadRequest::from_str(version).and_then(PythonDownloadRequest::fill)
})
.collect::<Result<Vec<_>, Error>>()?;
let downloads = requests
.iter()
.map(|request| match PythonDownload::from_request(request) {
Some(download) => download,
None => panic!("No download found for request {request:?}"),
})
.collect::<Vec<_>>();
let client = uv_client::BaseClientBuilder::new().build();
info!("Fetching requested versions...");
let mut tasks = futures::stream::iter(downloads.iter())
.map(|download| {
async {
let result = download.fetch(&client, bootstrap_dir).await;
(download.python_version(), result)
}
.instrument(info_span!("download", key = %download))
})
.buffered(4);
let mut results = Vec::new();
let mut downloaded = 0;
while let Some(task) = tasks.next().await {
let (version, result) = task;
let path = match result? {
DownloadResult::AlreadyAvailable(path) => {
info!("Found existing download for v{}", version);
path
}
DownloadResult::Fetched(path) => {
info!("Downloaded v{} to {}", version, path.user_display());
downloaded += 1;
path
}
};
results.push((version, path));
}
if downloaded > 0 {
let s = if downloaded == 1 { "" } else { "s" };
info!(
"Fetched {} in {}s",
format!("{} version{}", downloaded, s),
start.elapsed().as_secs()
);
} else {
info!("All versions downloaded already.");
};
// Order matters here, as we overwrite previous links
info!("Installing to `{}`...", bootstrap_dir.user_display());
// On Windows, linking the executable generally results in broken installations
// and each toolchain path will need to be added to the PATH separately in the
// desired order
#[cfg(unix)]
{
let mut links: HashMap<PathBuf, PathBuf> = HashMap::new();
for (version, path) in results {
// TODO(zanieb): This path should be a part of the download metadata
let executable = path.join("install").join("bin").join("python3");
for target in [
bootstrap_dir.join(format!("python{}", version.python_full_version())),
bootstrap_dir.join(format!("python{}.{}", version.major(), version.minor())),
bootstrap_dir.join(format!("python{}", version.major())),
bootstrap_dir.join("python"),
] {
// Attempt to remove it, we'll fail on link if we couldn't remove it for some reason
// but if it's missing we don't want to error
let _ = fs::remove_file(&target);
symlink(&executable, &target).await?;
links.insert(target, executable.clone());
}
}
for (target, executable) in links.iter().sorted() {
info!(
"Linked `{}` to `{}`",
target.user_display(),
executable.user_display()
);
}
};
info!("Installed {} versions", requests.len());
Ok(())
}
async fn read_versions_file() -> Result<Vec<String>> {
let lines: Vec<String> = fs::tokio::read_to_string(".python-versions")
.await?
.lines()
.map(ToString::to_string)
.collect();
Ok(lines)
}

5
crates/uv-dev/src/main.rs
View File

@ -21,6 +21,7 @@ use resolve_many::ResolveManyArgs;
use crate::build::{build, BuildArgs};
use crate::clear_compile::ClearCompileArgs;
use crate::compile::CompileArgs;
use crate::fetch_python::FetchPythonArgs;
use crate::render_benchmarks::RenderBenchmarksArgs;
use crate::resolve_cli::ResolveCliArgs;
use crate::wheel_metadata::WheelMetadataArgs;
@ -44,6 +45,7 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
mod build;
mod clear_compile;
mod compile;
mod fetch_python;
mod render_benchmarks;
mod resolve_cli;
mod resolve_many;
@ -72,6 +74,8 @@ enum Cli {
Compile(CompileArgs),
/// Remove all `.pyc` in the tree.
ClearCompile(ClearCompileArgs),
/// Fetch Python versions for testing
FetchPython(FetchPythonArgs),
}
#[instrument] // Anchor span to check for overhead
@ -92,6 +96,7 @@ async fn run() -> Result<()> {
Cli::RenderBenchmarks(args) => render_benchmarks::render_benchmarks(&args)?,
Cli::Compile(args) => compile::compile(args).await?,
Cli::ClearCompile(args) => clear_compile::clear_compile(&args)?,
Cli::FetchPython(args) => fetch_python::fetch_python(args).await?,
}
Ok(())
}

30
crates/uv-dev/src/resolve_cli.rs
View File

@ -12,12 +12,13 @@ use petgraph::dot::{Config as DotConfig, Dot};
use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Resolution};
use pep508_rs::Requirement;
use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
use uv_client::{FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch;
use uv_installer::SitePackages;
use uv_interpreter::PythonEnvironment;
use uv_resolver::{InMemoryIndex, Manifest, Options, Resolver};
use uv_types::{BuildIsolation, ConfigSettings, InFlight, NoBinary, NoBuild, SetupPyStrategy};
use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, Resolver};
use uv_types::{BuildIsolation, HashStrategy, InFlight};
#[derive(ValueEnum, Default, Clone)]
pub(crate) enum ResolveCliFormat {
@ -56,14 +57,6 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
let venv = PythonEnvironment::from_virtualenv(&cache)?;
let index_locations =
IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false);
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.build();
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(entries, venv.interpreter().tags()?)
};
let index = InMemoryIndex::default();
let in_flight = InFlight::default();
let no_build = if args.no_build {
@ -71,6 +64,20 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
} else {
NoBuild::None
};
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.build();
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(
entries,
venv.interpreter().tags()?,
&HashStrategy::None,
&no_build,
&NoBinary::None,
)
};
let config_settings = ConfigSettings::default();
let build_dispatch = BuildDispatch::new(
@ -101,6 +108,7 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
&client,
&flat_index,
&index,
&HashStrategy::None,
&build_dispatch,
&site_packages,
)?;

23
crates/uv-dev/src/resolve_many.rs
View File

@ -14,13 +14,13 @@ use distribution_types::IndexLocations;
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::{Requirement, VersionOrUrl};
use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, OwnedArchive, RegistryClient, RegistryClientBuilder};
use uv_client::{OwnedArchive, RegistryClient, RegistryClientBuilder};
use uv_configuration::{ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch;
use uv_interpreter::PythonEnvironment;
use uv_normalize::PackageName;
use uv_resolver::InMemoryIndex;
use uv_types::NoBinary;
use uv_types::{BuildContext, BuildIsolation, ConfigSettings, InFlight, NoBuild, SetupPyStrategy};
use uv_resolver::{FlatIndex, InMemoryIndex};
use uv_types::{BuildContext, BuildIsolation, InFlight};
#[derive(Parser)]
pub(crate) struct ResolveManyArgs {
@ -47,10 +47,17 @@ async fn find_latest_version(
client: &RegistryClient,
package_name: &PackageName,
) -> Option<Version> {
let (_, raw_simple_metadata) = client.simple(package_name).await.ok()?;
let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata);
let version = simple_metadata.into_iter().next()?.version;
Some(version)
client
.simple(package_name)
.await
.ok()
.into_iter()
.flatten()
.filter_map(|(_index, raw_simple_metadata)| {
let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata);
Some(simple_metadata.into_iter().next()?.version)
})
.max()
}
pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {

2
crates/uv-dispatch/Cargo.toml
View File

@ -21,9 +21,9 @@ uv-cache = { workspace = true }
uv-client = { workspace = true }
uv-installer = { workspace = true }
uv-interpreter = { workspace = true }
uv-requirements = { workspace = true }
uv-resolver = { workspace = true }
uv-types = { workspace = true }
uv-configuration = { workspace = true }
anyhow = { workspace = true }
futures = { workspace = true }

205
crates/uv-dispatch/src/lib.rs
View File

@ -3,8 +3,8 @@
//! implementing [`BuildContext`].
use std::ffi::OsStr;
use std::ffi::OsString;
use std::path::Path;
use std::{ffi::OsString, future::Future};
use anyhow::{bail, Context, Result};
use futures::FutureExt;
@ -16,14 +16,12 @@ use distribution_types::{IndexLocations, Name, Resolution, SourceDist};
use pep508_rs::Requirement;
use uv_build::{SourceBuild, SourceBuildContext};
use uv_cache::Cache;
use uv_client::{FlatIndex, RegistryClient};
use uv_client::RegistryClient;
use uv_configuration::{BuildKind, ConfigSettings, NoBinary, NoBuild, Reinstall, SetupPyStrategy};
use uv_installer::{Downloader, Installer, Plan, Planner, SitePackages};
use uv_interpreter::{Interpreter, PythonEnvironment};
use uv_resolver::{InMemoryIndex, Manifest, Options, Resolver};
use uv_types::{
BuildContext, BuildIsolation, BuildKind, ConfigSettings, EmptyInstalledPackages, InFlight,
NoBinary, NoBuild, Reinstall, SetupPyStrategy,
};
use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, Resolver};
use uv_types::{BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
/// The main implementation of [`BuildContext`], used by the CLI, see [`BuildContext`]
/// documentation.
@ -145,6 +143,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
self.client,
self.flat_index,
self.index,
&HashStrategy::None,
self,
&EmptyInstalledPackages,
)?;
@ -157,7 +156,6 @@ impl<'a> BuildContext for BuildDispatch<'a> {
Ok(Resolution::from(graph))
}
#[allow(clippy::manual_async_fn)] // TODO(konstin): rustc 1.75 gets into a type inference cycle with async fn
#[instrument(
skip(self, resolution, venv),
fields(
@ -165,119 +163,118 @@ impl<'a> BuildContext for BuildDispatch<'a> {
venv = ?venv.root()
)
)]
fn install<'data>(
async fn install<'data>(
&'data self,
resolution: &'data Resolution,
venv: &'data PythonEnvironment,
) -> impl Future<Output = Result<()>> + Send + 'data {
async move {
debug!(
"Installing in {} in {}",
) -> Result<()> {
debug!(
"Installing in {} in {}",
resolution
.distributions()
.map(ToString::to_string)
.join(", "),
venv.root().display(),
);
// Determine the current environment markers.
let tags = self.interpreter.tags()?;
// Determine the set of installed packages.
let site_packages = SitePackages::from_executable(venv)?;
let Plan {
cached,
remote,
installed: _,
reinstalls,
extraneous: _,
} = Planner::with_requirements(&resolution.requirements()).build(
site_packages,
&Reinstall::None,
&NoBinary::None,
&HashStrategy::None,
self.index_locations,
self.cache(),
venv,
tags,
)?;
// Nothing to do.
if remote.is_empty() && cached.is_empty() && reinstalls.is_empty() {
debug!("No build requirements to install for build");
return Ok(());
}
// Resolve any registry-based requirements.
let remote = remote
.iter()
.map(|dist| {
resolution
.distributions()
.map(ToString::to_string)
.join(", "),
venv.root().display(),
.get_remote(&dist.name)
.cloned()
.expect("Resolution should contain all packages")
})
.collect::<Vec<_>>();
// Download any missing distributions.
let wheels = if remote.is_empty() {
vec![]
} else {
// TODO(konstin): Check that there is no endless recursion.
let downloader =
Downloader::new(self.cache, tags, &HashStrategy::None, self.client, self);
debug!(
"Downloading and building requirement{} for build: {}",
if remote.len() == 1 { "" } else { "s" },
remote.iter().map(ToString::to_string).join(", ")
);
// Determine the current environment markers.
let tags = self.interpreter.tags()?;
downloader
.download(remote, self.in_flight)
.await
.context("Failed to download and build distributions")?
};
// Determine the set of installed packages.
let site_packages = SitePackages::from_executable(venv)?;
let Plan {
cached,
remote,
installed: _,
reinstalls,
extraneous: _,
} = Planner::with_requirements(&resolution.requirements()).build(
site_packages,
&Reinstall::None,
&NoBinary::None,
self.index_locations,
self.cache(),
venv,
tags,
)?;
// Nothing to do.
if remote.is_empty() && cached.is_empty() && reinstalls.is_empty() {
debug!("No build requirements to install for build");
return Ok(());
}
// Resolve any registry-based requirements.
let remote = remote
.iter()
.map(|dist| {
resolution
.get_remote(&dist.name)
.cloned()
.expect("Resolution should contain all packages")
})
.collect::<Vec<_>>();
// Download any missing distributions.
let wheels = if remote.is_empty() {
vec![]
} else {
// TODO(konstin): Check that there is no endless recursion.
let downloader = Downloader::new(self.cache, tags, self.client, self);
debug!(
"Downloading and building requirement{} for build: {}",
if remote.len() == 1 { "" } else { "s" },
remote.iter().map(ToString::to_string).join(", ")
);
downloader
.download(remote, self.in_flight)
// Remove any unnecessary packages.
if !reinstalls.is_empty() {
for dist_info in &reinstalls {
let summary = uv_installer::uninstall(dist_info)
.await
.context("Failed to download and build distributions")?
};
// Remove any unnecessary packages.
if !reinstalls.is_empty() {
for dist_info in &reinstalls {
let summary = uv_installer::uninstall(dist_info)
.await
.context("Failed to uninstall build dependencies")?;
debug!(
"Uninstalled {} ({} file{}, {} director{})",
dist_info.name(),
summary.file_count,
if summary.file_count == 1 { "" } else { "s" },
summary.dir_count,
if summary.dir_count == 1 { "y" } else { "ies" },
);
}
}
// Install the resolved distributions.
let wheels = wheels.into_iter().chain(cached).collect::<Vec<_>>();
if !wheels.is_empty() {
.context("Failed to uninstall build dependencies")?;
debug!(
"Installing build requirement{}: {}",
if wheels.len() == 1 { "" } else { "s" },
wheels.iter().map(ToString::to_string).join(", ")
"Uninstalled {} ({} file{}, {} director{})",
dist_info.name(),
summary.file_count,
if summary.file_count == 1 { "" } else { "s" },
summary.dir_count,
if summary.dir_count == 1 { "y" } else { "ies" },
);
Installer::new(venv)
.install(&wheels)
.context("Failed to install build dependencies")?;
}
Ok(())
}
// Install the resolved distributions.
let wheels = wheels.into_iter().chain(cached).collect::<Vec<_>>();
if !wheels.is_empty() {
debug!(
"Installing build requirement{}: {}",
if wheels.len() == 1 { "" } else { "s" },
wheels.iter().map(ToString::to_string).join(", ")
);
Installer::new(venv)
.install(&wheels)
.context("Failed to install build dependencies")?;
}
Ok(())
}
#[allow(clippy::manual_async_fn)] // TODO(konstin): rustc 1.75 gets into a type inference cycle with async fn
#[instrument(skip_all, fields(package_id = package_id, subdirectory = ?subdirectory))]
#[instrument(skip_all, fields(version_id = version_id, subdirectory = ?subdirectory))]
async fn setup_build<'data>(
&'data self,
source: &'data Path,
subdirectory: Option<&'data Path>,
package_id: &'data str,
version_id: &'data str,
dist: Option<&'data SourceDist>,
build_kind: BuildKind,
) -> Result<SourceBuild> {
@ -307,7 +304,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
self.interpreter,
self,
self.source_build_context.clone(),
package_id.to_string(),
version_id.to_string(),
self.setup_py,
self.config_settings.clone(),
self.build_isolation,

4
crates/uv-distribution/Cargo.toml
View File

@ -28,10 +28,12 @@ uv-fs = { workspace = true, features = ["tokio"] }
uv-git = { workspace = true, features = ["vendored-openssl"] }
uv-normalize = { workspace = true }
uv-types = { workspace = true }
uv-configuration = { workspace = true }
anyhow = { workspace = true }
fs-err = { workspace = true }
futures = { workspace = true }
md-5 = { workspace = true }
nanoid = { workspace = true }
once_cell = { workspace = true }
reqwest = { workspace = true }
@ -39,6 +41,7 @@ reqwest-middleware = { workspace = true }
rmp-serde = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive"] }
sha2 = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
@ -46,4 +49,3 @@ tokio-util = { workspace = true, features = ["compat"] }
tracing = { workspace = true }
url = { workspace = true }
zip = { workspace = true }

25
crates/uv-distribution/src/archive.rs Normal file
View File

@ -0,0 +1,25 @@
use distribution_types::Hashed;
use pypi_types::HashDigest;
use uv_cache::ArchiveId;
/// An archive (unzipped wheel) that exists in the local cache.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Archive {
/// The unique ID of the entry in the wheel's archive bucket.
pub id: ArchiveId,
/// The computed hashes of the archive.
pub hashes: Vec<HashDigest>,
}
impl Archive {
/// Create a new [`Archive`] with the given ID and hashes.
pub(crate) fn new(id: ArchiveId, hashes: Vec<HashDigest>) -> Self {
Self { id, hashes }
}
}
impl Hashed for Archive {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}

582
crates/uv-distribution/src/distribution_database.rs
View File

@ -1,8 +1,9 @@
use std::io;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::sync::Arc;
use futures::{FutureExt, TryStreamExt};
use tempfile::TempDir;
use tokio::io::AsyncSeekExt;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info_span, instrument, warn, Instrument};
@ -10,17 +11,23 @@ use url::Url;
use distribution_filename::WheelFilename;
use distribution_types::{
BuildableSource, BuiltDist, Dist, FileLocation, IndexLocations, LocalEditable, Name, SourceDist,
BuildableSource, BuiltDist, Dist, FileLocation, HashPolicy, Hashed, IndexLocations,
LocalEditable, Name, SourceDist,
};
use platform_tags::Tags;
use pypi_types::Metadata23;
use uv_cache::{ArchiveTarget, ArchiveTimestamp, CacheBucket, CacheEntry, WheelCache};
use uv_client::{CacheControl, CachedClientError, Connectivity, RegistryClient};
use uv_types::{BuildContext, NoBinary, NoBuild};
use pypi_types::{HashDigest, Metadata23};
use uv_cache::{ArchiveId, ArchiveTimestamp, CacheBucket, CacheEntry, Timestamp, WheelCache};
use uv_client::{
CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient,
};
use uv_configuration::{NoBinary, NoBuild};
use uv_extract::hash::Hasher;
use uv_fs::write_atomic;
use uv_types::BuildContext;
use crate::download::{BuiltWheel, UnzippedWheel};
use crate::archive::Archive;
use crate::locks::Locks;
use crate::{DiskWheel, Error, LocalWheel, Reporter, SourceDistributionBuilder};
use crate::{ArchiveMetadata, Error, LocalWheel, Reporter, SourceDistributionBuilder};
/// A cached high-level interface to convert distributions (a requirement resolved to a location)
/// to a wheel or wheel metadata.
@ -77,28 +84,38 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
/// Either fetch the wheel or fetch and build the source distribution
///
/// If `no_remote_wheel` is set, the wheel will be built from a source distribution
/// even if compatible pre-built wheels are available.
/// Returns a wheel that's compliant with the given platform tags.
///
/// While hashes will be generated in some cases, hash-checking is only enforced for source
/// distributions, and should be enforced by the caller for wheels.
#[instrument(skip_all, fields(%dist))]
pub async fn get_or_build_wheel(&self, dist: &Dist, tags: &Tags) -> Result<LocalWheel, Error> {
pub async fn get_or_build_wheel(
&self,
dist: &Dist,
tags: &Tags,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
match dist {
Dist::Built(built) => self.get_wheel(built).await,
Dist::Source(source) => self.build_wheel(source, tags).await,
Dist::Built(built) => self.get_wheel(built, hashes).await,
Dist::Source(source) => self.build_wheel(source, tags, hashes).await,
}
}
/// Either fetch the only wheel metadata (directly from the index or with range requests) or
/// fetch and build the source distribution.
///
/// Returns the [`Metadata23`], along with a "precise" URL for the source distribution, if
/// possible. For example, given a Git dependency with a reference to a branch or tag, return a
/// URL with a precise reference to the current commit of that branch or tag.
/// While hashes will be generated in some cases, hash-checking is only enforced for source
/// distributions, and should be enforced by the caller for wheels.
#[instrument(skip_all, fields(%dist))]
pub async fn get_or_build_wheel_metadata(&self, dist: &Dist) -> Result<Metadata23, Error> {
pub async fn get_or_build_wheel_metadata(
&self,
dist: &Dist,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
match dist {
Dist::Built(built) => self.get_wheel_metadata(built).await,
Dist::Built(built) => self.get_wheel_metadata(built, hashes).await,
Dist::Source(source) => {
self.build_wheel_metadata(&BuildableSource::Dist(source))
self.build_wheel_metadata(&BuildableSource::Dist(source), hashes)
.await
}
}
@ -110,22 +127,35 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
editable: &LocalEditable,
editable_wheel_dir: &Path,
) -> Result<(LocalWheel, Metadata23), Error> {
// Build the wheel.
let (dist, disk_filename, filename, metadata) = self
.builder
.build_editable(editable, editable_wheel_dir)
.await?;
let built_wheel = BuiltWheel {
// Unzip into the editable wheel directory.
let path = editable_wheel_dir.join(&disk_filename);
let target = editable_wheel_dir.join(cache_key::digest(&editable.path));
let id = self.unzip_wheel(&path, &target).await?;
let wheel = LocalWheel {
dist,
filename,
path: editable_wheel_dir.join(disk_filename),
target: editable_wheel_dir.join(cache_key::digest(&editable.path)),
archive: self.build_context.cache().archive(&id),
hashes: vec![],
};
Ok((LocalWheel::Built(built_wheel), metadata))
Ok((wheel, metadata))
}
/// Fetch a wheel from the cache or download it from the index.
async fn get_wheel(&self, dist: &BuiltDist) -> Result<LocalWheel, Error> {
///
/// While hashes will be generated in all cases, hash-checking is _not_ enforced and should
/// instead be enforced by the caller.
async fn get_wheel(
&self,
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
let no_binary = match self.build_context.no_binary() {
NoBinary::None => false,
NoBinary::All => true,
@ -145,41 +175,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
Url::parse(url).map_err(|err| Error::Url(url.clone(), err))?
}
FileLocation::Path(path) => {
let url = Url::from_file_path(path).expect("path is absolute");
let cache_entry = self.build_context.cache().entry(
CacheBucket::Wheels,
WheelCache::Url(&url).wheel_dir(wheel.name().as_ref()),
WheelCache::Index(&wheel.index).wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(),
);
// If the file is already unzipped, and the unzipped directory is fresh,
// return it.
match cache_entry.path().canonicalize() {
Ok(archive) => {
if ArchiveTimestamp::up_to_date_with(
path,
ArchiveTarget::Cache(&archive),
)
.map_err(Error::CacheRead)?
{
return Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: Dist::Built(dist.clone()),
archive,
filename: wheel.filename.clone(),
}));
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
// Otherwise, unzip the file.
return Ok(LocalWheel::Disk(DiskWheel {
dist: Dist::Built(dist.clone()),
path: path.clone(),
target: cache_entry.into_path_buf(),
filename: wheel.filename.clone(),
}));
return self
.load_wheel(path, &wheel.filename, cache_entry, dist, hashes)
.await;
}
};
@ -192,14 +195,15 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
// Download and unzip.
match self
.stream_wheel(url.clone(), &wheel.filename, &wheel_entry, dist)
.stream_wheel(url.clone(), &wheel.filename, &wheel_entry, dist, hashes)
.await
{
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel {
Ok(archive) => Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive,
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(),
})),
}),
Err(Error::Extract(err)) if err.is_http_streaming_unsupported() => {
warn!(
"Streaming unsupported for {dist}; downloading wheel to disk ({err})"
@ -208,13 +212,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
// If the request failed because streaming is unsupported, download the
// wheel directly.
let archive = self
.download_wheel(url, &wheel.filename, &wheel_entry, dist)
.download_wheel(url, &wheel.filename, &wheel_entry, dist, hashes)
.await?;
Ok(LocalWheel::Unzipped(UnzippedWheel {
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive,
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(),
}))
})
}
Err(err) => Err(err),
}
@ -230,14 +235,21 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
// Download and unzip.
match self
.stream_wheel(wheel.url.raw().clone(), &wheel.filename, &wheel_entry, dist)
.stream_wheel(
wheel.url.raw().clone(),
&wheel.filename,
&wheel_entry,
dist,
hashes,
)
.await
{
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel {
Ok(archive) => Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive,
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(),
})),
}),
Err(Error::Client(err)) if err.is_http_streaming_unsupported() => {
warn!(
"Streaming unsupported for {dist}; downloading wheel to disk ({err})"
@ -251,13 +263,15 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
&wheel.filename,
&wheel_entry,
dist,
hashes,
)
.await?;
Ok(LocalWheel::Unzipped(UnzippedWheel {
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive,
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(),
}))
})
}
Err(err) => Err(err),
}
@ -270,90 +284,107 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
wheel.filename.stem(),
);
// If the file is already unzipped, and the unzipped directory is fresh,
// return it.
match cache_entry.path().canonicalize() {
Ok(archive) => {
if ArchiveTimestamp::up_to_date_with(
&wheel.path,
ArchiveTarget::Cache(&archive),
)
.map_err(Error::CacheRead)?
{
return Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: Dist::Built(dist.clone()),
archive,
filename: wheel.filename.clone(),
}));
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
Ok(LocalWheel::Disk(DiskWheel {
dist: Dist::Built(dist.clone()),
path: wheel.path.clone(),
target: cache_entry.into_path_buf(),
filename: wheel.filename.clone(),
}))
self.load_wheel(&wheel.path, &wheel.filename, cache_entry, dist, hashes)
.await
}
}
}
/// Convert a source distribution into a wheel, fetching it from the cache or building it if
/// necessary.
async fn build_wheel(&self, dist: &SourceDist, tags: &Tags) -> Result<LocalWheel, Error> {
///
/// The returned wheel is guaranteed to come from a distribution with a matching hash, and
/// no build processes will be executed for distributions with mismatched hashes.
async fn build_wheel(
&self,
dist: &SourceDist,
tags: &Tags,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
let lock = self.locks.acquire(&Dist::Source(dist.clone())).await;
let _guard = lock.lock().await;
let built_wheel = self
.builder
.download_and_build(&BuildableSource::Dist(dist), tags)
.download_and_build(&BuildableSource::Dist(dist), tags, hashes)
.boxed()
.await?;
// If the wheel was unzipped previously, respect it. Source distributions are
// cached under a unique build ID, so unzipped directories are never stale.
// cached under a unique revision ID, so unzipped directories are never stale.
match built_wheel.target.canonicalize() {
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: Dist::Source(dist.clone()),
archive,
filename: built_wheel.filename,
})),
Err(err) if err.kind() == io::ErrorKind::NotFound => {
Ok(LocalWheel::Built(BuiltWheel {
Ok(archive) => {
return Ok(LocalWheel {
dist: Dist::Source(dist.clone()),
path: built_wheel.path,
target: built_wheel.target,
archive,
filename: built_wheel.filename,
}))
hashes: built_wheel.hashes,
});
}
Err(err) => Err(Error::CacheRead(err)),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
// Otherwise, unzip the wheel.
let id = self
.unzip_wheel(&built_wheel.path, &built_wheel.target)
.await?;
Ok(LocalWheel {
dist: Dist::Source(dist.clone()),
archive: self.build_context.cache().archive(&id),
hashes: built_wheel.hashes,
filename: built_wheel.filename,
})
}
/// Fetch the wheel metadata from the index, or from the cache if possible.
pub async fn get_wheel_metadata(&self, dist: &BuiltDist) -> Result<Metadata23, Error> {
///
/// While hashes will be generated in some cases, hash-checking is _not_ enforced and should
/// instead be enforced by the caller.
pub async fn get_wheel_metadata(
&self,
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
// If hash generation is enabled, and the distribution isn't hosted on an index, get the
// entire wheel to ensure that the hashes are included in the response. If the distribution
// is hosted on an index, the hashes will be included in the simple metadata response.
// For hash _validation_, callers are expected to enforce the policy when retrieving the
// wheel.
// TODO(charlie): Request the hashes via a separate method, to reduce the coupling in this API.
if hashes.is_generate() && matches!(dist, BuiltDist::DirectUrl(_) | BuiltDist::Path(_)) {
let wheel = self.get_wheel(dist, hashes).await?;
let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
return Ok(ArchiveMetadata { metadata, hashes });
}
match self.client.wheel_metadata(dist).boxed().await {
Ok(metadata) => Ok(metadata),
Ok(metadata) => Ok(ArchiveMetadata::from(metadata)),
Err(err) if err.is_http_streaming_unsupported() => {
warn!("Streaming unsupported when fetching metadata for {dist}; downloading wheel directly ({err})");
// If the request failed due to an error that could be resolved by
// downloading the wheel directly, try that.
let wheel = self.get_wheel(dist).await?;
Ok(wheel.metadata()?)
let wheel = self.get_wheel(dist, hashes).await?;
let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
Ok(ArchiveMetadata { metadata, hashes })
}
Err(err) => Err(err.into()),
}
}
/// Build the wheel metadata for a source distribution, or fetch it from the cache if possible.
///
/// The returned metadata is guaranteed to come from a distribution with a matching hash, and
/// no build processes will be executed for distributions with mismatched hashes.
pub async fn build_wheel_metadata(
&self,
source: &BuildableSource<'_>,
) -> Result<Metadata23, Error> {
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
let no_build = match self.build_context.no_build() {
NoBuild::All => true,
NoBuild::None => false,
@ -372,7 +403,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
let metadata = self
.builder
.download_and_build_metadata(source)
.download_and_build_metadata(source, hashes)
.boxed()
.await?;
Ok(metadata)
@ -385,7 +416,8 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
filename: &WheelFilename,
wheel_entry: &CacheEntry,
dist: &BuiltDist,
) -> Result<PathBuf, Error> {
hashes: HashPolicy<'_>,
) -> Result<Archive, Error> {
// Create an entry for the HTTP cache.
let http_entry = wheel_entry.with_file(format!("{}.http", filename.stem()));
@ -396,35 +428,39 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
.map_err(|err| self.handle_response_errors(err))
.into_async_read();
// Create a hasher for each hash algorithm.
let algorithms = hashes.algorithms();
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);
// Download and unzip the wheel to a temporary directory.
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
.map_err(Error::CacheWrite)?;
uv_extract::stream::unzip(reader.compat(), temp_dir.path()).await?;
uv_extract::stream::unzip(&mut hasher, temp_dir.path()).await?;
// If necessary, exhaust the reader to compute the hash.
if !hashes.is_none() {
hasher.finish().await.map_err(Error::HashExhaustion)?;
}
// Persist the temporary directory to the directory store.
let archive = self
let id = self
.build_context
.cache()
.persist(temp_dir.into_path(), wheel_entry.path())
.await
.map_err(Error::CacheRead)?;
Ok(archive)
Ok(Archive::new(
id,
hashers.into_iter().map(HashDigest::from).collect(),
))
}
.instrument(info_span!("wheel", wheel = %dist))
};
let req = self
.client
.uncached_client()
.get(url)
.header(
// `reqwest` defaults to accepting compressed responses.
// Specify identity encoding to get consistent .whl downloading
// behavior from servers. ref: https://github.com/pypa/pip/pull/1688
"accept-encoding",
reqwest::header::HeaderValue::from_static("identity"),
)
.build()?;
// Fetch the archive from the cache, or download it if necessary.
let req = self.request(url.clone())?;
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.build_context
@ -434,7 +470,6 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
),
Connectivity::Offline => CacheControl::AllowStale,
};
let archive = self
.client
.cached_client()
@ -445,6 +480,20 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
CachedClientError::Client(err) => Error::Client(err),
})?;
// If the archive is missing the required hashes, force a refresh.
let archive = if archive.has_digests(hashes) {
archive
} else {
self.client
.cached_client()
.skip_cache(self.request(url)?, &http_entry, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
CachedClientError::Client(err) => Error::Client(err),
})?
};
Ok(archive)
}
@ -455,7 +504,8 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
filename: &WheelFilename,
wheel_entry: &CacheEntry,
dist: &BuiltDist,
) -> Result<PathBuf, Error> {
hashes: HashPolicy<'_>,
) -> Result<Archive, Error> {
// Create an entry for the HTTP cache.
let http_entry = wheel_entry.with_file(format!("{}.http", filename.stem()));
@ -481,33 +531,48 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
file.seek(io::SeekFrom::Start(0))
.await
.map_err(Error::CacheWrite)?;
let reader = tokio::io::BufReader::new(file);
uv_extract::seek::unzip(reader, temp_dir.path()).await?;
// If no hashes are required, parallelize the unzip operation.
let hashes = if hashes.is_none() {
let file = file.into_std().await;
tokio::task::spawn_blocking({
let target = temp_dir.path().to_owned();
move || -> Result<(), uv_extract::Error> {
// Unzip the wheel into a temporary directory.
uv_extract::unzip(file, &target)?;
Ok(())
}
})
.await??;
vec![]
} else {
// Create a hasher for each hash algorithm.
let algorithms = hashes.algorithms();
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers);
uv_extract::stream::unzip(&mut hasher, temp_dir.path()).await?;
// If necessary, exhaust the reader to compute the hash.
hasher.finish().await.map_err(Error::HashExhaustion)?;
hashers.into_iter().map(HashDigest::from).collect()
};
// Persist the temporary directory to the directory store.
let archive = self
let id = self
.build_context
.cache()
.persist(temp_dir.into_path(), wheel_entry.path())
.await
.map_err(Error::CacheRead)?;
Ok(archive)
Ok(Archive::new(id, hashes))
}
.instrument(info_span!("wheel", wheel = %dist))
};
let req = self
.client
.uncached_client()
.get(url)
.header(
// `reqwest` defaults to accepting compressed responses.
// Specify identity encoding to get consistent .whl downloading
// behavior from servers. ref: https://github.com/pypa/pip/pull/1688
"accept-encoding",
reqwest::header::HeaderValue::from_static("identity"),
)
.build()?;
let req = self.request(url.clone())?;
let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from(
self.build_context
@ -517,7 +582,6 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
),
Connectivity::Offline => CacheControl::AllowStale,
};
let archive = self
.client
.cached_client()
@ -528,11 +592,225 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
CachedClientError::Client(err) => Error::Client(err),
})?;
// If the archive is missing the required hashes, force a refresh.
let archive = if archive.has_digests(hashes) {
archive
} else {
self.client
.cached_client()
.skip_cache(self.request(url)?, &http_entry, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
CachedClientError::Client(err) => Error::Client(err),
})?
};
Ok(archive)
}
/// Load a wheel from a local path.
async fn load_wheel(
&self,
path: &Path,
filename: &WheelFilename,
wheel_entry: CacheEntry,
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
// Determine the last-modified time of the wheel.
let modified = ArchiveTimestamp::from_file(path).map_err(Error::CacheRead)?;
// Attempt to read the archive pointer from the cache.
let pointer_entry = wheel_entry.with_file(format!("{}.rev", filename.stem()));
let pointer = LocalArchivePointer::read_from(&pointer_entry)?;
// Extract the archive from the pointer.
let archive = pointer
.filter(|pointer| pointer.is_up_to_date(modified))
.map(LocalArchivePointer::into_archive)
.filter(|archive| archive.has_digests(hashes));
// If the file is already unzipped, and the cache is up-to-date, return it.
if let Some(archive) = archive {
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: filename.clone(),
})
} else if hashes.is_none() {
// Otherwise, unzip the wheel.
let archive = Archive::new(self.unzip_wheel(path, wheel_entry.path()).await?, vec![]);
// Write the archive pointer to the cache.
let pointer = LocalArchivePointer {
timestamp: modified.timestamp(),
archive: archive.clone(),
};
pointer.write_to(&pointer_entry).await?;
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: filename.clone(),
})
} else {
// If necessary, compute the hashes of the wheel.
let file = fs_err::tokio::File::open(path)
.await
.map_err(Error::CacheRead)?;
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
.map_err(Error::CacheWrite)?;
// Create a hasher for each hash algorithm.
let algorithms = hashes.algorithms();
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers);
// Unzip the wheel to a temporary directory.
uv_extract::stream::unzip(&mut hasher, temp_dir.path()).await?;
// Exhaust the reader to compute the hash.
hasher.finish().await.map_err(Error::HashExhaustion)?;
let hashes = hashers.into_iter().map(HashDigest::from).collect();
// Persist the temporary directory to the directory store.
let id = self
.build_context
.cache()
.persist(temp_dir.into_path(), wheel_entry.path())
.await
.map_err(Error::CacheWrite)?;
// Create an archive.
let archive = Archive::new(id, hashes);
// Write the archive pointer to the cache.
let pointer = LocalArchivePointer {
timestamp: modified.timestamp(),
archive: archive.clone(),
};
pointer.write_to(&pointer_entry).await?;
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: filename.clone(),
})
}
}
/// Unzip a wheel into the cache, returning the path to the unzipped directory.
async fn unzip_wheel(&self, path: &Path, target: &Path) -> Result<ArchiveId, Error> {
let temp_dir = tokio::task::spawn_blocking({
let path = path.to_owned();
let root = self.build_context.cache().root().to_path_buf();
move || -> Result<TempDir, uv_extract::Error> {
// Unzip the wheel into a temporary directory.
let temp_dir = tempfile::tempdir_in(root)?;
uv_extract::unzip(fs_err::File::open(path)?, temp_dir.path())?;
Ok(temp_dir)
}
})
.await??;
// Persist the temporary directory to the directory store.
let id = self
.build_context
.cache()
.persist(temp_dir.into_path(), target)
.await
.map_err(Error::CacheWrite)?;
Ok(id)
}
/// Returns a GET [`reqwest::Request`] for the given URL.
fn request(&self, url: Url) -> Result<reqwest::Request, reqwest::Error> {
self.client
.uncached_client()
.get(url)
.header(
// `reqwest` defaults to accepting compressed responses.
// Specify identity encoding to get consistent .whl downloading
// behavior from servers. ref: https://github.com/pypa/pip/pull/1688
"accept-encoding",
reqwest::header::HeaderValue::from_static("identity"),
)
.build()
}
/// Return the [`IndexLocations`] used by this resolver.
pub fn index_locations(&self) -> &IndexLocations {
self.build_context.index_locations()
}
}
/// A pointer to an archive in the cache, fetched from an HTTP archive.
///
/// Encoded with `MsgPack`, and represented on disk by a `.http` file.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct HttpArchivePointer {
archive: Archive,
}
impl HttpArchivePointer {
/// Read an [`HttpArchivePointer`] from the cache.
pub fn read_from(path: impl AsRef<Path>) -> Result<Option<Self>, Error> {
match fs_err::File::open(path.as_ref()) {
Ok(file) => {
let data = DataWithCachePolicy::from_reader(file)?.data;
let archive = rmp_serde::from_slice::<Archive>(&data)?;
Ok(Some(Self { archive }))
}
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(Error::CacheRead(err)),
}
}
/// Return the [`Archive`] from the pointer.
pub fn into_archive(self) -> Archive {
self.archive
}
}
/// A pointer to an archive in the cache, fetched from a local path.
///
/// Encoded with `MsgPack`, and represented on disk by a `.rev` file.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct LocalArchivePointer {
timestamp: Timestamp,
archive: Archive,
}
impl LocalArchivePointer {
/// Read an [`LocalArchivePointer`] from the cache.
pub fn read_from(path: impl AsRef<Path>) -> Result<Option<Self>, Error> {
match fs_err::read(path) {
Ok(cached) => Ok(Some(rmp_serde::from_slice::<LocalArchivePointer>(&cached)?)),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(Error::CacheRead(err)),
}
}
/// Write an [`LocalArchivePointer`] to the cache.
pub async fn write_to(&self, entry: &CacheEntry) -> Result<(), Error> {
write_atomic(entry.path(), rmp_serde::to_vec(&self)?)
.await
.map_err(Error::CacheWrite)
}
/// Returns `true` if the archive is up-to-date with the given modified timestamp.
pub fn is_up_to_date(&self, modified: ArchiveTimestamp) -> bool {
self.timestamp == modified.timestamp()
}
/// Return the [`Archive`] from the pointer.
pub fn into_archive(self) -> Archive {
self.archive
}
}

119
crates/uv-distribution/src/download.rs
View File

@ -1,14 +1,14 @@
use std::path::{Path, PathBuf};
use distribution_filename::WheelFilename;
use distribution_types::{CachedDist, Dist};
use pypi_types::Metadata23;
use distribution_types::{CachedDist, Dist, Hashed};
use pypi_types::{HashDigest, Metadata23};
use crate::Error;
/// A wheel that's been unzipped while downloading
/// A locally available wheel.
#[derive(Debug, Clone)]
pub struct UnzippedWheel {
pub struct LocalWheel {
/// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist,
/// The parsed filename.
@ -16,115 +16,42 @@ pub struct UnzippedWheel {
/// The canonicalized path in the cache directory to which the wheel was downloaded.
/// Typically, a directory within the archive bucket.
pub(crate) archive: PathBuf,
}
/// A downloaded wheel that's stored on-disk.
#[derive(Debug, Clone)]
pub struct DiskWheel {
/// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The path to the downloaded wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
/// Typically, a symlink within the wheels or built wheels bucket.
pub(crate) target: PathBuf,
}
/// A wheel built from a source distribution that's stored on-disk.
#[derive(Debug, Clone)]
pub struct BuiltWheel {
/// The remote source distribution from which this wheel was built.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The path to the built wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
/// Typically, a symlink within the wheels or built wheels bucket.
pub(crate) target: PathBuf,
}
/// A downloaded or built wheel.
#[derive(Debug, Clone)]
pub enum LocalWheel {
Unzipped(UnzippedWheel),
Disk(DiskWheel),
Built(BuiltWheel),
/// The computed hashes of the wheel.
pub(crate) hashes: Vec<HashDigest>,
}
impl LocalWheel {
/// Return the path to the downloaded wheel's entry in the cache.
pub fn target(&self) -> &Path {
match self {
Self::Unzipped(wheel) => &wheel.archive,
Self::Disk(wheel) => &wheel.target,
Self::Built(wheel) => &wheel.target,
}
&self.archive
}
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
match self {
Self::Unzipped(wheel) => wheel.remote(),
Self::Disk(wheel) => wheel.remote(),
Self::Built(wheel) => wheel.remote(),
}
&self.dist
}
/// Return the [`WheelFilename`] of this wheel.
pub fn filename(&self) -> &WheelFilename {
match self {
Self::Unzipped(wheel) => &wheel.filename,
Self::Disk(wheel) => &wheel.filename,
Self::Built(wheel) => &wheel.filename,
}
}
/// Convert a [`LocalWheel`] into a [`CachedDist`].
pub fn into_cached_dist(self, archive: PathBuf) -> CachedDist {
match self {
Self::Unzipped(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
Self::Disk(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
Self::Built(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
}
&self.filename
}
/// Read the [`Metadata23`] from a wheel.
pub fn metadata(&self) -> Result<Metadata23, Error> {
match self {
Self::Unzipped(wheel) => read_flat_wheel_metadata(&wheel.filename, &wheel.archive),
Self::Disk(wheel) => read_built_wheel_metadata(&wheel.filename, &wheel.path),
Self::Built(wheel) => read_built_wheel_metadata(&wheel.filename, &wheel.path),
}
read_flat_wheel_metadata(&self.filename, &self.archive)
}
}
impl UnzippedWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
}
/// Convert an [`UnzippedWheel`] into a [`CachedDist`].
pub fn into_cached_dist(self) -> CachedDist {
CachedDist::from_remote(self.dist, self.filename, self.archive)
impl Hashed for LocalWheel {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}
impl DiskWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
}
}
impl BuiltWheel {
/// Return the [`Dist`] from which this source distribution that this wheel was built from was
/// downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
/// Convert a [`LocalWheel`] into a [`CachedDist`].
impl From<LocalWheel> for CachedDist {
fn from(wheel: LocalWheel) -> CachedDist {
CachedDist::from_remote(wheel.dist, wheel.filename, wheel.hashes, wheel.archive)
}
}
@ -134,18 +61,6 @@ impl std::fmt::Display for LocalWheel {
}
}
/// Read the [`Metadata23`] from a built wheel.
fn read_built_wheel_metadata(
filename: &WheelFilename,
wheel: impl AsRef<Path>,
) -> Result<Metadata23, Error> {
let file = fs_err::File::open(wheel.as_ref()).map_err(Error::CacheRead)?;
let reader = std::io::BufReader::new(file);
let mut archive = zip::ZipArchive::new(reader)?;
let metadata = install_wheel_rs::metadata::read_archive_metadata(filename, &mut archive)?;
Ok(Metadata23::parse_metadata(&metadata)?)
}
/// Read the [`Metadata23`] from an unzipped wheel.
fn read_flat_wheel_metadata(
filename: &WheelFilename,

100
crates/uv-distribution/src/error.rs
View File

@ -3,6 +3,8 @@ use tokio::task::JoinError;
use zip::result::ZipError;
use distribution_filename::WheelFilenameError;
use pep440_rs::Version;
use pypi_types::HashDigest;
use uv_client::BetterReqwestError;
use uv_normalize::PackageName;
@ -47,8 +49,10 @@ pub enum Error {
given: PackageName,
metadata: PackageName,
},
#[error("Package metadata version `{metadata}` does not match given version `{given}`")]
VersionMismatch { given: Version, metadata: Version },
#[error("Failed to parse metadata from built wheel")]
Metadata(#[from] pypi_types::Error),
Metadata(#[from] pypi_types::MetadataError),
#[error("Failed to read `dist-info` metadata from built wheel")]
DistInfo(#[from] install_wheel_rs::Error),
#[error("Failed to read zip archive from built wheel")]
@ -62,11 +66,11 @@ pub enum Error {
#[error("The source distribution is missing a `PKG-INFO` file")]
MissingPkgInfo,
#[error("The source distribution does not support static metadata in `PKG-INFO`")]
DynamicPkgInfo(#[source] pypi_types::Error),
DynamicPkgInfo(#[source] pypi_types::MetadataError),
#[error("The source distribution is missing a `pyproject.toml` file")]
MissingPyprojectToml,
#[error("The source distribution does not support static metadata in `pyproject.toml`")]
DynamicPyprojectToml(#[source] pypi_types::Error),
DynamicPyprojectToml(#[source] pypi_types::MetadataError),
#[error("Unsupported scheme in URL: {0}")]
UnsupportedScheme(String),
@ -78,6 +82,40 @@ pub enum Error {
/// Should not occur; only seen when another task panicked.
#[error("The task executor is broken, did some other task panic?")]
Join(#[from] JoinError),
/// An I/O error that occurs while exhausting a reader to compute a hash.
#[error("Failed to hash distribution")]
HashExhaustion(#[source] std::io::Error),
#[error("Hash mismatch for {distribution}\n\nExpected:\n{expected}\n\nComputed:\n{actual}")]
MismatchedHashes {
distribution: String,
expected: String,
actual: String,
},
#[error(
"Hash-checking is enabled, but no hashes were provided or computed for: {distribution}"
)]
MissingHashes { distribution: String },
#[error("Hash-checking is enabled, but no hashes were computed for: {distribution}\n\nExpected:\n{expected}")]
MissingActualHashes {
distribution: String,
expected: String,
},
#[error("Hash-checking is enabled, but no hashes were provided for: {distribution}\n\nComputed:\n{actual}")]
MissingExpectedHashes {
distribution: String,
actual: String,
},
#[error("Hash-checking is not supported for local directories: {0}")]
HashesNotSupportedSourceTree(String),
#[error("Hash-checking is not supported for Git repositories: {0}")]
HashesNotSupportedGit(String),
}
impl From<reqwest::Error> for Error {
@ -96,3 +134,59 @@ impl From<reqwest_middleware::Error> for Error {
}
}
}
impl Error {
/// Construct a hash mismatch error.
pub fn hash_mismatch(
distribution: String,
expected: &[HashDigest],
actual: &[HashDigest],
) -> Error {
match (expected.is_empty(), actual.is_empty()) {
(true, true) => Self::MissingHashes { distribution },
(true, false) => {
let actual = actual
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
Self::MissingExpectedHashes {
distribution,
actual,
}
}
(false, true) => {
let expected = expected
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
Self::MissingActualHashes {
distribution,
expected,
}
}
(false, false) => {
let expected = expected
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
let actual = actual
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
Self::MismatchedHashes {
distribution,
expected,
actual,
}
}
}
}
}

97
crates/uv-distribution/src/index/built_wheel_index.rs
View File

@ -1,52 +1,72 @@
use distribution_types::{git_reference, DirectUrlSourceDist, GitSourceDist, PathSourceDist};
use distribution_types::{
git_reference, DirectUrlSourceDist, GitSourceDist, Hashed, PathSourceDist,
};
use platform_tags::Tags;
use uv_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheShard, WheelCache};
use uv_fs::symlinks;
use uv_types::HashStrategy;
use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, read_timestamp_manifest, MANIFEST};
use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
use crate::Error;
/// A local index of built distributions for a specific source distribution.
pub struct BuiltWheelIndex;
#[derive(Debug)]
pub struct BuiltWheelIndex<'a> {
cache: &'a Cache,
tags: &'a Tags,
hasher: &'a HashStrategy,
}
impl<'a> BuiltWheelIndex<'a> {
/// Initialize an index of built distributions.
pub fn new(cache: &'a Cache, tags: &'a Tags, hasher: &'a HashStrategy) -> Self {
Self {
cache,
tags,
hasher,
}
}
impl BuiltWheelIndex {
/// Return the most compatible [`CachedWheel`] for a given source distribution at a direct URL.
///
/// This method does not perform any freshness checks and assumes that the source distribution
/// is already up-to-date.
pub fn url(
source_dist: &DirectUrlSourceDist,
cache: &Cache,
tags: &Tags,
) -> Result<Option<CachedWheel>, Error> {
pub fn url(&self, source_dist: &DirectUrlSourceDist) -> Result<Option<CachedWheel>, Error> {
// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = cache.shard(
let cache_shard = self.cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Url(source_dist.url.raw()).root(),
);
// Read the manifest from the cache. There's no need to enforce freshness, since we
// enforce freshness on the entries.
let manifest_entry = cache_shard.entry(MANIFEST);
let Some(manifest) = read_http_manifest(&manifest_entry)? else {
// Read the revision from the cache.
let Some(pointer) = HttpRevisionPointer::read_from(cache_shard.entry(HTTP_REVISION))?
else {
return Ok(None);
};
Ok(Self::find(&cache_shard.shard(manifest.id()), tags))
// Enforce hash-checking by omitting any wheels that don't satisfy the required hashes.
let revision = pointer.into_revision();
if !revision.satisfies(self.hasher.get(source_dist)) {
return Ok(None);
}
Ok(self.find(&cache_shard.shard(revision.id())))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a local path.
pub fn path(
source_dist: &PathSourceDist,
cache: &Cache,
tags: &Tags,
) -> Result<Option<CachedWheel>, Error> {
let cache_shard = cache.shard(
pub fn path(&self, source_dist: &PathSourceDist) -> Result<Option<CachedWheel>, Error> {
let cache_shard = self.cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Path(&source_dist.url).root(),
);
// Read the revision from the cache.
let Some(pointer) = LocalRevisionPointer::read_from(cache_shard.entry(LOCAL_REVISION))?
else {
return Ok(None);
};
// Determine the last-modified time of the source distribution.
let Some(modified) =
ArchiveTimestamp::from_path(&source_dist.path).map_err(Error::CacheRead)?
@ -54,28 +74,37 @@ impl BuiltWheelIndex {
return Err(Error::DirWithoutEntrypoint);
};
// Read the manifest from the cache. There's no need to enforce freshness, since we
// enforce freshness on the entries.
let manifest_entry = cache_shard.entry(MANIFEST);
let Some(manifest) = read_timestamp_manifest(&manifest_entry, modified)? else {
// If the distribution is stale, omit it from the index.
if !pointer.is_up_to_date(modified) {
return Ok(None);
};
}
Ok(Self::find(&cache_shard.shard(manifest.id()), tags))
// Enforce hash-checking by omitting any wheels that don't satisfy the required hashes.
let revision = pointer.into_revision();
if !revision.satisfies(self.hasher.get(source_dist)) {
return Ok(None);
}
Ok(self.find(&cache_shard.shard(revision.id())))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a git URL.
pub fn git(source_dist: &GitSourceDist, cache: &Cache, tags: &Tags) -> Option<CachedWheel> {
pub fn git(&self, source_dist: &GitSourceDist) -> Option<CachedWheel> {
// Enforce hash-checking, which isn't supported for Git distributions.
if self.hasher.get(source_dist).is_validate() {
return None;
}
let Ok(Some(git_sha)) = git_reference(&source_dist.url) else {
return None;
};
let cache_shard = cache.shard(
let cache_shard = self.cache.shard(
CacheBucket::BuiltWheels,
WheelCache::Git(&source_dist.url, &git_sha.to_short_string()).root(),
);
Self::find(&cache_shard, tags)
self.find(&cache_shard)
}
/// Find the "best" distribution in the index for a given source distribution.
@ -94,16 +123,16 @@ impl BuiltWheelIndex {
/// ```
///
/// The `shard` should be `built-wheels-v0/pypi/django-allauth-0.51.0.tar.gz`.
fn find(shard: &CacheShard, tags: &Tags) -> Option<CachedWheel> {
fn find(&self, shard: &CacheShard) -> Option<CachedWheel> {
let mut candidate: Option<CachedWheel> = None;
// Unzipped wheels are stored as symlinks into the archive directory.
for subdir in symlinks(shard) {
match CachedWheel::from_path(&subdir) {
match CachedWheel::from_built_source(&subdir) {
None => {}
Some(dist_info) => {
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);
let compatibility = dist_info.filename.compatibility(self.tags);
// Only consider wheels that are compatible with our tags.
if !compatibility.is_compatible() {
@ -113,7 +142,7 @@ impl BuiltWheelIndex {
if let Some(existing) = candidate.as_ref() {
// Override if the wheel is newer, or "more" compatible.
if dist_info.filename.version > existing.filename.version
|| compatibility > existing.filename.compatibility(tags)
|| compatibility > existing.filename.compatibility(self.tags)
{
candidate = Some(dist_info);
}

73
crates/uv-distribution/src/index/cached_wheel.rs
View File

@ -1,9 +1,12 @@
use std::path::Path;
use distribution_filename::WheelFilename;
use distribution_types::{CachedDirectUrlDist, CachedRegistryDist};
use distribution_types::{CachedDirectUrlDist, CachedRegistryDist, Hashed};
use pep508_rs::VerbatimUrl;
use uv_cache::CacheEntry;
use pypi_types::HashDigest;
use uv_cache::{Cache, CacheBucket, CacheEntry};
use crate::{Archive, HttpArchivePointer, LocalArchivePointer};
#[derive(Debug, Clone)]
pub struct CachedWheel {
@ -11,16 +14,28 @@ pub struct CachedWheel {
pub filename: WheelFilename,
/// The [`CacheEntry`] for the wheel.
pub entry: CacheEntry,
/// The [`HashDigest`]s for the wheel.
pub hashes: Vec<HashDigest>,
}
impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_path(path: &Path) -> Option<Self> {
pub fn from_built_source(path: impl AsRef<Path>) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
// Convert to a cached wheel.
let archive = path.canonicalize().ok()?;
let entry = CacheEntry::from_path(archive);
Some(Self { filename, entry })
let hashes = Vec::new();
Some(Self {
filename,
entry,
hashes,
})
}
/// Convert a [`CachedWheel`] into a [`CachedRegistryDist`].
@ -28,6 +43,7 @@ impl CachedWheel {
CachedRegistryDist {
filename: self.filename,
path: self.entry.into_path_buf(),
hashes: self.hashes,
}
}
@ -38,6 +54,55 @@ impl CachedWheel {
url,
path: self.entry.into_path_buf(),
editable: false,
hashes: self.hashes,
}
}
/// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`).
pub fn from_http_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
// Read the pointer.
let pointer = HttpArchivePointer::read_from(path).ok()??;
let Archive { id, hashes } = pointer.into_archive();
// Convert to a cached wheel.
let entry = cache.entry(CacheBucket::Archive, "", id);
Some(Self {
filename,
entry,
hashes,
})
}
/// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`).
pub fn from_local_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
// Read the pointer.
let pointer = LocalArchivePointer::read_from(path).ok()??;
let Archive { id, hashes } = pointer.into_archive();
// Convert to a cached wheel.
let entry = cache.entry(CacheBucket::Archive, "", id);
Some(Self {
filename,
entry,
hashes,
})
}
}
impl Hashed for CachedWheel {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}

136
crates/uv-distribution/src/index/registry_wheel_index.rs
View File

@ -1,19 +1,19 @@
use std::collections::hash_map::Entry;
use std::collections::BTreeMap;
use std::path::Path;
use rustc_hash::FxHashMap;
use distribution_types::{CachedRegistryDist, FlatIndexLocation, IndexLocations, IndexUrl};
use distribution_types::{CachedRegistryDist, FlatIndexLocation, Hashed, IndexLocations, IndexUrl};
use pep440_rs::Version;
use pep508_rs::VerbatimUrl;
use platform_tags::Tags;
use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_fs::{directories, symlinks};
use uv_fs::{directories, files, symlinks};
use uv_normalize::PackageName;
use uv_types::HashStrategy;
use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, MANIFEST};
use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
/// A local index of distributions that originate from a registry, like `PyPI`.
#[derive(Debug)]
@ -21,16 +21,23 @@ pub struct RegistryWheelIndex<'a> {
cache: &'a Cache,
tags: &'a Tags,
index_locations: &'a IndexLocations,
hasher: &'a HashStrategy,
index: FxHashMap<&'a PackageName, BTreeMap<Version, CachedRegistryDist>>,
}
impl<'a> RegistryWheelIndex<'a> {
/// Initialize an index of cached distributions from a directory.
pub fn new(cache: &'a Cache, tags: &'a Tags, index_locations: &'a IndexLocations) -> Self {
/// Initialize an index of registry distributions.
pub fn new(
cache: &'a Cache,
tags: &'a Tags,
index_locations: &'a IndexLocations,
hasher: &'a HashStrategy,
) -> Self {
Self {
cache,
tags,
index_locations,
hasher,
index: FxHashMap::default(),
}
}
@ -65,6 +72,7 @@ impl<'a> RegistryWheelIndex<'a> {
self.cache,
self.tags,
self.index_locations,
self.hasher,
)),
};
versions
@ -76,14 +84,18 @@ impl<'a> RegistryWheelIndex<'a> {
cache: &Cache,
tags: &Tags,
index_locations: &IndexLocations,
hasher: &HashStrategy,
) -> BTreeMap<Version, CachedRegistryDist> {
let mut versions = BTreeMap::new();
// Collect into owned `IndexUrl`
// Collect into owned `IndexUrl`.
let flat_index_urls: Vec<IndexUrl> = index_locations
.flat_index()
.filter_map(|flat_index| match flat_index {
FlatIndexLocation::Path(_) => None,
FlatIndexLocation::Path(path) => {
let path = fs_err::canonicalize(path).ok()?;
Some(IndexUrl::Path(VerbatimUrl::from_path(path)))
}
FlatIndexLocation::Url(url) => {
Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone())))
}
@ -97,7 +109,44 @@ impl<'a> RegistryWheelIndex<'a> {
WheelCache::Index(index_url).wheel_dir(package.to_string()),
);
Self::add_directory(&wheel_dir, tags, &mut versions);
// For registry wheels, the cache structure is: `<index>/<package-name>/<wheel>.http`
// or `<index>/<package-name>/<version>/<wheel>.rev`.
for file in files(&wheel_dir) {
match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("http"))
{
if let Some(wheel) =
CachedWheel::from_http_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get_package(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("rev"))
{
if let Some(wheel) =
CachedWheel::from_local_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get_package(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
}
}
// Index all the built wheels, created by downloading and building source distributions
// from the registry.
@ -110,43 +159,62 @@ impl<'a> RegistryWheelIndex<'a> {
for shard in directories(&cache_shard) {
// Read the existing metadata from the cache, if it exists.
let cache_shard = cache_shard.shard(shard);
let manifest_entry = cache_shard.entry(MANIFEST);
if let Ok(Some(manifest)) = read_http_manifest(&manifest_entry) {
Self::add_directory(cache_shard.join(manifest.id()), tags, &mut versions);
// Read the revision from the cache.
let revision = match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
let revision_entry = cache_shard.entry(LOCAL_REVISION);
if let Ok(Some(pointer)) = LocalRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
};
if let Some(revision) = revision {
// Enforce hash-checking based on the source distribution.
if revision.satisfies(hasher.get_package(package)) {
for wheel_dir in symlinks(cache_shard.join(revision.id())) {
if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
}
}
versions
}
/// Add the wheels in a given directory to the index.
///
/// Each subdirectory in the given path is expected to be that of an unzipped wheel.
fn add_directory(
path: impl AsRef<Path>,
/// Add the [`CachedWheel`] to the index.
fn add_wheel(
wheel: CachedWheel,
tags: &Tags,
versions: &mut BTreeMap<Version, CachedRegistryDist>,
) {
// Unzipped wheels are stored as symlinks into the archive directory.
for wheel_dir in symlinks(path.as_ref()) {
match CachedWheel::from_path(&wheel_dir) {
None => {}
Some(dist_info) => {
let dist_info = dist_info.into_registry_dist();
let dist_info = wheel.into_registry_dist();
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);
if let Some(existing) = versions.get_mut(&dist_info.filename.version) {
// Override if we have better compatibility
if compatibility > existing.filename.compatibility(tags) {
*existing = dist_info;
}
} else if compatibility.is_compatible() {
versions.insert(dist_info.filename.version.clone(), dist_info);
}
}
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);
if let Some(existing) = versions.get_mut(&dist_info.filename.version) {
// Override if we have better compatibility
if compatibility > existing.filename.compatibility(tags) {
*existing = dist_info;
}
} else if compatibility.is_compatible() {
versions.insert(dist_info.filename.version.clone(), dist_info);
}
}
}

27
crates/uv-distribution/src/lib.rs
View File

@ -1,12 +1,14 @@
pub use distribution_database::DistributionDatabase;
pub use download::{BuiltWheel, DiskWheel, LocalWheel};
pub use archive::Archive;
pub use distribution_database::{DistributionDatabase, HttpArchivePointer, LocalArchivePointer};
pub use download::LocalWheel;
pub use error::Error;
pub use git::{is_same_reference, to_precise};
pub use index::{BuiltWheelIndex, RegistryWheelIndex};
use pypi_types::{HashDigest, Metadata23};
pub use reporter::Reporter;
pub use source::SourceDistributionBuilder;
pub use unzip::Unzip;
mod archive;
mod distribution_database;
mod download;
mod error;
@ -15,4 +17,21 @@ mod index;
mod locks;
mod reporter;
mod source;
mod unzip;
/// The metadata associated with an archive.
#[derive(Debug, Clone)]
pub struct ArchiveMetadata {
/// The [`Metadata23`] for the underlying distribution.
pub metadata: Metadata23,
/// The hashes of the source or built archive.
pub hashes: Vec<HashDigest>,
}
impl From<Metadata23> for ArchiveMetadata {
fn from(metadata: Metadata23) -> Self {
Self {
metadata,
hashes: vec![],
}
}
}

22
crates/uv-distribution/src/source/built_wheel_metadata.rs
View File

@ -2,23 +2,27 @@ use std::path::PathBuf;
use std::str::FromStr;
use distribution_filename::WheelFilename;
use distribution_types::Hashed;
use platform_tags::Tags;
use pypi_types::HashDigest;
use uv_cache::CacheShard;
use uv_fs::files;
/// The information about the wheel we either just built or got from the cache.
#[derive(Debug, Clone)]
pub struct BuiltWheelMetadata {
pub(crate) struct BuiltWheelMetadata {
/// The path to the built wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
pub(crate) target: PathBuf,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The computed hashes of the source distribution from which the wheel was built.
pub(crate) hashes: Vec<HashDigest>,
}
impl BuiltWheelMetadata {
/// Find a compatible wheel in the cache based on the given manifest.
/// Find a compatible wheel in the cache.
pub(crate) fn find_in_cache(tags: &Tags, cache_shard: &CacheShard) -> Option<Self> {
for directory in files(cache_shard) {
if let Some(metadata) = Self::from_path(directory, cache_shard) {
@ -39,6 +43,20 @@ impl BuiltWheelMetadata {
target: cache_shard.join(filename.stem()),
path,
filename,
hashes: vec![],
})
}
/// Set the computed hashes of the wheel.
#[must_use]
pub(crate) fn with_hashes(mut self, hashes: Vec<HashDigest>) -> Self {
self.hashes = hashes;
self
}
}
impl Hashed for BuiltWheelMetadata {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}

17
crates/uv-distribution/src/source/manifest.rs
View File

@ -1,17 +0,0 @@
use serde::{Deserialize, Serialize};
/// The [`Manifest`] is a thin wrapper around a unique identifier for the source distribution.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct Manifest(String);
impl Manifest {
/// Initialize a new [`Manifest`] with a random UUID.
pub(crate) fn new() -> Self {
Self(nanoid::nanoid!())
}
/// Return the unique ID of the manifest.
pub(crate) fn id(&self) -> &str {
&self.0
}
}

1324
crates/uv-distribution/src/source/mod.rs
View File

File diff suppressed because it is too large Load Diff

72
crates/uv-distribution/src/source/revision.rs Normal file
View File

@ -0,0 +1,72 @@
use distribution_types::Hashed;
use serde::{Deserialize, Serialize};
use std::path::Path;
use pypi_types::HashDigest;
/// The [`Revision`] is a thin wrapper around a unique identifier for the source distribution.
///
/// A revision represents a unique version of a source distribution, at a level more granular than
/// (e.g.) the version number of the distribution itself. For example, a source distribution hosted
/// at a URL or a local file path may have multiple revisions, each representing a unique state of
/// the distribution, despite the reported version number remaining the same.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct Revision {
id: RevisionId,
hashes: Vec<HashDigest>,
}
impl Revision {
/// Initialize a new [`Revision`] with a random UUID.
pub(crate) fn new() -> Self {
Self {
id: RevisionId::new(),
hashes: vec![],
}
}
/// Return the unique ID of the manifest.
pub(crate) fn id(&self) -> &RevisionId {
&self.id
}
/// Return the computed hashes of the archive.
pub(crate) fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
/// Return the computed hashes of the archive.
pub(crate) fn into_hashes(self) -> Vec<HashDigest> {
self.hashes
}
/// Set the computed hashes of the archive.
#[must_use]
pub(crate) fn with_hashes(mut self, hashes: Vec<HashDigest>) -> Self {
self.hashes = hashes;
self
}
}
impl Hashed for Revision {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}
/// A unique identifier for a revision of a source distribution.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct RevisionId(String);
impl RevisionId {
/// Generate a new unique identifier for an archive.
fn new() -> Self {
Self(nanoid::nanoid!())
}
}
impl AsRef<Path> for RevisionId {
fn as_ref(&self) -> &Path {
self.0.as_ref()
}
}

36
crates/uv-distribution/src/unzip.rs
View File

@ -1,36 +0,0 @@
use std::path::Path;
use tracing::instrument;
use uv_extract::Error;
use crate::download::BuiltWheel;
use crate::{DiskWheel, LocalWheel};
pub trait Unzip {
/// Unzip a wheel into the target directory.
fn unzip(&self, target: &Path) -> Result<(), Error>;
}
impl Unzip for DiskWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
uv_extract::unzip(fs_err::File::open(&self.path)?, target)
}
}
impl Unzip for BuiltWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
uv_extract::unzip(fs_err::File::open(&self.path)?, target)
}
}
impl Unzip for LocalWheel {
#[instrument(skip_all, fields(filename=self.filename().to_string()))]
fn unzip(&self, target: &Path) -> Result<(), Error> {
match self {
Self::Unzipped(_) => Ok(()),
Self::Disk(wheel) => wheel.unzip(target),
Self::Built(wheel) => wheel.unzip(target),
}
}
}

6
crates/uv-extract/Cargo.toml
View File

@ -13,12 +13,16 @@ license = { workspace = true }
workspace = true
[dependencies]
async-compression = { workspace = true, features = ["gzip"] }
pypi-types = { workspace = true }
async-compression = { workspace = true, features = ["gzip", "zstd"] }
async_zip = { workspace = true, features = ["tokio"] }
fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true }
md-5.workspace = true
rayon = { workspace = true }
rustc-hash = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["io-util"] }
tokio-tar = { workspace = true }

146
crates/uv-extract/src/hash.rs Normal file
View File

@ -0,0 +1,146 @@
use std::pin::Pin;
use std::task::{Context, Poll};
use sha2::Digest;
use tokio::io::{AsyncReadExt, ReadBuf};
use pypi_types::{HashAlgorithm, HashDigest};
pub struct Sha256Reader<'a, R> {
reader: R,
hasher: &'a mut sha2::Sha256,
}
impl<'a, R> Sha256Reader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
pub fn new(reader: R, hasher: &'a mut sha2::Sha256) -> Self {
Sha256Reader { reader, hasher }
}
}
impl<'a, R> tokio::io::AsyncRead for Sha256Reader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let reader = Pin::new(&mut self.reader);
match reader.poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
self.hasher.update(buf.filled());
Poll::Ready(Ok(()))
}
other => other,
}
}
}
#[derive(Debug)]
pub enum Hasher {
Md5(md5::Md5),
Sha256(sha2::Sha256),
Sha384(sha2::Sha384),
Sha512(sha2::Sha512),
}
impl Hasher {
pub fn update(&mut self, data: &[u8]) {
match self {
Hasher::Md5(hasher) => hasher.update(data),
Hasher::Sha256(hasher) => hasher.update(data),
Hasher::Sha384(hasher) => hasher.update(data),
Hasher::Sha512(hasher) => hasher.update(data),
}
}
pub fn finalize(self) -> Vec<u8> {
match self {
Hasher::Md5(hasher) => hasher.finalize().to_vec(),
Hasher::Sha256(hasher) => hasher.finalize().to_vec(),
Hasher::Sha384(hasher) => hasher.finalize().to_vec(),
Hasher::Sha512(hasher) => hasher.finalize().to_vec(),
}
}
}
impl From<HashAlgorithm> for Hasher {
fn from(algorithm: HashAlgorithm) -> Self {
match algorithm {
HashAlgorithm::Md5 => Hasher::Md5(md5::Md5::new()),
HashAlgorithm::Sha256 => Hasher::Sha256(sha2::Sha256::new()),
HashAlgorithm::Sha384 => Hasher::Sha384(sha2::Sha384::new()),
HashAlgorithm::Sha512 => Hasher::Sha512(sha2::Sha512::new()),
}
}
}
impl From<Hasher> for HashDigest {
fn from(hasher: Hasher) -> Self {
match hasher {
Hasher::Md5(hasher) => HashDigest {
algorithm: HashAlgorithm::Md5,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
Hasher::Sha256(hasher) => HashDigest {
algorithm: HashAlgorithm::Sha256,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
Hasher::Sha384(hasher) => HashDigest {
algorithm: HashAlgorithm::Sha384,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
Hasher::Sha512(hasher) => HashDigest {
algorithm: HashAlgorithm::Sha512,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
}
}
}
pub struct HashReader<'a, R> {
reader: R,
hashers: &'a mut [Hasher],
}
impl<'a, R> HashReader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
pub fn new(reader: R, hashers: &'a mut [Hasher]) -> Self {
HashReader { reader, hashers }
}
/// Exhaust the underlying reader.
pub async fn finish(&mut self) -> Result<(), std::io::Error> {
while self.read(&mut vec![0; 8192]).await? > 0 {}
Ok(())
}
}
impl<'a, R> tokio::io::AsyncRead for HashReader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let reader = Pin::new(&mut self.reader);
match reader.poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
for hasher in self.hashers.iter_mut() {
hasher.update(buf.filled());
}
Poll::Ready(Ok(()))
}
other => other,
}
}
}

Some files were not shown because too many files have changed in this diff Show More