Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into zb/site-packages-link

This commit is contained in:
Charlie Marsh 2024-04-11 17:26:04 -04:00
parent 192b214dc6 8507ba872f
commit 375f76e701
194 changed files with 17549 additions and 7273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/renovate.json5 vendored
View File

@ -29,6 +29,18 @@
matchManagers: ["pre-commit"], matchManagers: ["pre-commit"],
description: "Weekly update of pre-commit dependencies", description: "Weekly update of pre-commit dependencies",
}, },
{
groupName: "Rust dev-dependencies",
matchManagers: ["cargo"],
matchDepTypes: ["devDependencies"],
description: "Weekly update of Rust development dependencies",
},
{
groupName: "pyo3",
matchManagers: ["cargo"],
matchPackagePatterns: ["pyo3"],
description: "Weekly update of pyo3 dependencies",
},
], ],
vulnerabilityAlerts: { vulnerabilityAlerts: {
commitMessageSuffix: "", commitMessageSuffix: "",

76
.github/workflows/ci.yml vendored
View File

@ -72,33 +72,23 @@ jobs:
name: "cargo test | ${{ matrix.os }}" name: "cargo test | ${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- if: ${{ matrix.os == 'macos' }}
name: "Install bootstrap dependencies"
run: brew install coreutils
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: "Install required Python versions"
run: |
python -m pip install "zstandard==0.22.0"
python scripts/bootstrap/install.py
- name: "Install Rust toolchain" - name: "Install Rust toolchain"
run: rustup show run: rustup show
- name: "Install cargo nextest"
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- if: ${{ matrix.os != 'windows' }} - if: ${{ matrix.os != 'windows' }}
uses: rui314/setup-mold@v1 uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2 - uses: Swatinem/rust-cache@v2
- name: "Install required Python versions"
run: |
cargo run -p uv-dev -- fetch-python
- name: "Install cargo nextest"
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- name: "Cargo test" - name: "Cargo test"
run: | run: |
cargo nextest run --workspace --status-level skip --failure-output immediate-final --no-fail-fast -j 12 --final-status-level slow cargo nextest run --workspace --status-level skip --failure-output immediate-final --no-fail-fast -j 12 --final-status-level slow
@ -224,6 +214,42 @@ jobs:
path: ./target/debug/uv.exe path: ./target/debug/uv.exe
retention-days: 1 retention-days: 1
ecosystem-test:
needs: build-binary-linux
name: "ecosystem test | ${{ matrix.repo }}"
runs-on: ubuntu-latest
strategy:
matrix:
include:
- repo: "prefecthq/prefect"
command: "uv pip install -e '.[dev]'"
python: "3.9"
- repo: "pallets/flask"
command: "uv pip install -r requirements/dev.txt"
python: "3.12"
fail-fast: false
steps:
- uses: actions/checkout@v4
with:
repository: ${{ matrix.repo }}
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: "Download binary"
uses: actions/download-artifact@v4
with:
name: uv-linux-${{ github.sha }}
- name: "Prepare binary"
run: chmod +x ./uv
- name: "Test"
run: |
./uv venv
./${{ matrix.command }}
cache-test-ubuntu: cache-test-ubuntu:
needs: build-binary-linux needs: build-binary-linux
name: "check cache | ubuntu" name: "check cache | ubuntu"
@ -274,12 +300,12 @@ jobs:
needs: build-binary-linux needs: build-binary-linux
name: "check system | python on debian" name: "check system | python on debian"
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: debian:bullseye container: debian:bookworm
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: "Install Python" - name: "Install Python"
run: apt-get update && apt-get install -y python3.9 python3-pip python3.9-venv run: apt-get update && apt-get install -y python3.11 python3-pip python3.11-venv
- name: "Download binary" - name: "Download binary"
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
@ -290,16 +316,16 @@ jobs:
run: chmod +x ./uv run: chmod +x ./uv
- name: "Print Python path" - name: "Print Python path"
run: echo $(which python3.9) run: echo $(which python3.11)
- name: "Validate global Python install" - name: "Validate global Python install"
run: python3.9 scripts/check_system_python.py --uv ./uv run: python3.11 scripts/check_system_python.py --uv ./uv --externally-managed
system-test-fedora: system-test-fedora:
needs: build-binary-linux needs: build-binary-linux
name: "check system | python on fedora" name: "check system | python on fedora"
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: fedora:39 container: fedora:41
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
@ -346,6 +372,8 @@ jobs:
run: python scripts/check_system_python.py --uv ./uv run: python scripts/check_system_python.py --uv ./uv
system-test-centos: system-test-centos:
# https://github.com/astral-sh/uv/issues/2915
if: false
needs: build-binary-linux needs: build-binary-linux
name: "check system | python on centos" name: "check system | python on centos"
runs-on: ubuntu-latest runs-on: ubuntu-latest

6
.github/workflows/release.yml vendored
View File

@ -65,7 +65,7 @@ jobs:
# we specify bash to get pipefail; it guards against the `curl` command # we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0 # failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0-prerelease.3/cargo-dist-installer.sh | sh" run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0/cargo-dist-installer.sh | sh"
# sure would be cool if github gave us proper conditionals... # sure would be cool if github gave us proper conditionals...
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
# functionality based on whether this is a pull_request, and whether it's from a fork. # functionality based on whether this is a pull_request, and whether it's from a fork.
@ -107,7 +107,7 @@ jobs:
submodules: recursive submodules: recursive
- name: Install cargo-dist - name: Install cargo-dist
shell: bash shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0-prerelease.3/cargo-dist-installer.sh | sh" run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0/cargo-dist-installer.sh | sh"
# Get all the local artifacts for the global tasks to use (for e.g. checksums) # Get all the local artifacts for the global tasks to use (for e.g. checksums)
- name: Fetch local artifacts - name: Fetch local artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
@ -151,7 +151,7 @@ jobs:
with: with:
submodules: recursive submodules: recursive
- name: Install cargo-dist - name: Install cargo-dist
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0-prerelease.3/cargo-dist-installer.sh | sh" run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.0/cargo-dist-installer.sh | sh"
# Fetch artifacts from scratch-storage # Fetch artifacts from scratch-storage
- name: Fetch artifacts - name: Fetch artifacts
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4

2
.gitignore vendored
View File

@ -8,7 +8,7 @@ target/
target-alpine/ target-alpine/
# Bootstrapped Python versions # Bootstrapped Python versions
bin/ /bin/
# These are backup files generated by rustfmt # These are backup files generated by rustfmt
**/*.rs.bk **/*.rs.bk

4
.pre-commit-config.yaml
View File

@ -12,7 +12,7 @@ repos:
- id: validate-pyproject - id: validate-pyproject
- repo: https://github.com/crate-ci/typos - repo: https://github.com/crate-ci/typos
rev: v1.18.2 rev: v1.20.4
hooks: hooks:
- id: typos - id: typos
@ -32,7 +32,7 @@ repos:
types_or: [yaml, json5] types_or: [yaml, json5]
- repo: https://github.com/astral-sh/ruff-pre-commit - repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.2.1 rev: v0.3.5
hooks: hooks:
- id: ruff-format - id: ruff-format
- id: ruff - id: ruff

49
CHANGELOG.md
View File

@ -1,5 +1,54 @@
# Changelog # Changelog
## 0.1.31
### Bug fixes
- Ignore direct URL distributions in prefetcher ([#2943](https://github.com/astral-sh/uv/pull/2943))
## 0.1.30
### Enhancements
- Show resolution diagnostics after `pip install` ([#2829](https://github.com/astral-sh/uv/pull/2829))
### Performance
- Speed up cold-cache `urllib3`-`boto3`-`botocore` performance with batched prefetching ([#2452](https://github.com/astral-sh/uv/pull/2452))
### Bug fixes
- Backtrack on distributions with invalid metadata ([#2834](https://github.com/astral-sh/uv/pull/2834))
- Include LICENSE files in source distribution ([#2855](https://github.com/astral-sh/uv/pull/2855))
- Respect `--no-build` and `--no-binary` in `--find-links` ([#2826](https://github.com/astral-sh/uv/pull/2826))
- Respect cached local `--find-links` in install plan ([#2907](https://github.com/astral-sh/uv/pull/2907))
- Avoid panic with multiple confirmation handlers ([#2903](https://github.com/astral-sh/uv/pull/2903))
- Use scheme parsing to determine absolute vs. relative URLs ([#2904](https://github.com/astral-sh/uv/pull/2904))
- Remove additional 'because' in resolution failure messages ([#2849](https://github.com/astral-sh/uv/pull/2849))
- Use `miette` when printing `pip sync` resolution failures ([#2848](https://github.com/astral-sh/uv/pull/2848))
## 0.1.29
### Enhancements
- Allow conflicting Git URLs that refer to the same commit SHA ([#2769](https://github.com/astral-sh/uv/pull/2769))
- Allow package lookups across multiple indexes via explicit opt-in (`--index-strategy unsafe-any-match`) ([#2815](https://github.com/astral-sh/uv/pull/2815))
- Allow no-op `--no-compile` flag on CLI ([#2816](https://github.com/astral-sh/uv/pull/2816))
- Upgrade `rs-async-zip` to support data descriptors ([#2809](https://github.com/astral-sh/uv/pull/2809))
### Bug fixes
- Avoid unused extras check in `pip install` for source trees ([#2811](https://github.com/astral-sh/uv/pull/2811))
- Deduplicate editables during install commands ([#2820](https://github.com/astral-sh/uv/pull/2820))
- Fix windows lock race: lock exclusive after all try lock errors ([#2800](https://github.com/astral-sh/uv/pull/2800))
- Preserve `.git` suffixes and casing in Git dependencies ([#2789](https://github.com/astral-sh/uv/pull/2789))
- Respect Git tags and branches that look like short commits ([#2795](https://github.com/astral-sh/uv/pull/2795))
- Enable virtualenv creation on Windows with cpython-x86 ([#2707](https://github.com/astral-sh/uv/pull/2707))
### Documentation
- Document that uv is safe to run concurrently ([#2818](https://github.com/astral-sh/uv/pull/2818))
## 0.1.28 ## 0.1.28
### Enhancements ### Enhancements

12
CONTRIBUTING.md
View File

@ -22,12 +22,6 @@ CMake may be installed with Homebrew:
brew install cmake brew install cmake
``` ```
The Python bootstrapping script requires `coreutils` and `zstd`; we recommend installing them with Homebrew:
```shell
brew install coreutils zstd
```
See the [Python](#python) section for instructions on installing the Python versions. See the [Python](#python) section for instructions on installing the Python versions.
### Windows ### Windows
@ -45,13 +39,13 @@ Testing uv requires multiple specific Python versions. You can install them into
`<project root>/bin` via our bootstrapping script: `<project root>/bin` via our bootstrapping script:
```shell ```shell
pipx run scripts/bootstrap/install.py cargo run -p uv-dev -- fetch-python
``` ```
Alternatively, you can install `zstandard` from PyPI, then run: You may need to add the versions to your `PATH`:
```shell ```shell
python3.12 scripts/bootstrap/install.py source .env
``` ```
You can configure the bootstrapping directory with `UV_BOOTSTRAP_DIR`. You can configure the bootstrapping directory with `UV_BOOTSTRAP_DIR`.

787
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

44
Cargo.toml
View File

@ -44,21 +44,23 @@ uv-normalize = { path = "crates/uv-normalize" }
uv-requirements = { path = "crates/uv-requirements" } uv-requirements = { path = "crates/uv-requirements" }
uv-resolver = { path = "crates/uv-resolver" } uv-resolver = { path = "crates/uv-resolver" }
uv-types = { path = "crates/uv-types" } uv-types = { path = "crates/uv-types" }
uv-configuration = { path = "crates/uv-configuration" }
uv-trampoline = { path = "crates/uv-trampoline" } uv-trampoline = { path = "crates/uv-trampoline" }
uv-version = { path = "crates/uv-version" } uv-version = { path = "crates/uv-version" }
uv-virtualenv = { path = "crates/uv-virtualenv" } uv-virtualenv = { path = "crates/uv-virtualenv" }
uv-warnings = { path = "crates/uv-warnings" } uv-warnings = { path = "crates/uv-warnings" }
uv-toolchain = { path = "crates/uv-toolchain" }
anstream = { version = "0.6.13" } anstream = { version = "0.6.13" }
anyhow = { version = "1.0.80" } anyhow = { version = "1.0.80" }
async-channel = { version = "2.2.0" } async-channel = { version = "2.2.0" }
async-compression = { version = "0.4.6" } async-compression = { version = "0.4.6" }
async-trait = { version = "0.1.78" } async-trait = { version = "0.1.78" }
async_http_range_reader = { version = "0.7.0" } async_http_range_reader = { version = "0.7.1" }
async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "d76801da0943de985254fc6255c0e476b57c5836", features = ["deflate"] } async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "1dcb40cfe1bf5325a6fd4bfcf9894db40241f585", features = ["deflate"] }
axoupdater = { version = "0.3.1", default-features = false } axoupdater = { version = "0.4.0", default-features = false }
backoff = { version = "0.4.0" } backoff = { version = "0.4.0" }
base64 = { version = "0.21.7" } base64 = { version = "0.22.0" }
cachedir = { version = "0.3.1" } cachedir = { version = "0.3.1" }
cargo-util = { version = "0.2.8" } cargo-util = { version = "0.2.8" }
chrono = { version = "0.4.31" } chrono = { version = "0.4.31" }
@ -85,13 +87,14 @@ hex = { version = "0.4.3" }
hmac = { version = "0.12.1" } hmac = { version = "0.12.1" }
home = { version = "0.5.9" } home = { version = "0.5.9" }
html-escape = { version = "0.2.13" } html-escape = { version = "0.2.13" }
http = { version = "0.2.12" } http = { version = "1.1.0" }
indexmap = { version = "2.2.5" } indexmap = { version = "2.2.5" }
indicatif = { version = "0.17.7" } indicatif = { version = "0.17.7" }
indoc = { version = "2.0.4" } indoc = { version = "2.0.4" }
itertools = { version = "0.12.1" } itertools = { version = "0.12.1" }
junction = { version = "1.0.0" } junction = { version = "1.0.0" }
mailparse = { version = "0.14.0" } mailparse = { version = "0.14.0" }
md-5 = { version = "0.10.6" }
miette = { version = "7.2.0" } miette = { version = "7.2.0" }
nanoid = { version = "0.4.0" } nanoid = { version = "0.4.0" }
once_cell = { version = "1.19.0" } once_cell = { version = "1.19.0" }
@ -106,9 +109,9 @@ rand = { version = "0.8.5" }
rayon = { version = "1.8.0" } rayon = { version = "1.8.0" }
reflink-copy = { version = "0.1.15" } reflink-copy = { version = "0.1.15" }
regex = { version = "1.10.2" } regex = { version = "1.10.2" }
reqwest = { version = "0.11.23", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls", "rustls-tls-native-roots"] } reqwest = { version = "0.12.3", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls", "rustls-tls-native-roots"] }
reqwest-middleware = { version = "0.2.4" } reqwest-middleware = { version = "0.3.0" }
reqwest-retry = { version = "0.3.0" } reqwest-retry = { version = "0.5.0" }
rkyv = { version = "0.7.43", features = ["strict", "validation"] } rkyv = { version = "0.7.43", features = ["strict", "validation"] }
rmp-serde = { version = "1.1.2" } rmp-serde = { version = "1.1.2" }
rust-netrc = { version = "0.1.1" } rust-netrc = { version = "0.1.1" }
@ -120,7 +123,6 @@ serde_json = { version = "1.0.114" }
sha1 = { version = "0.10.6" } sha1 = { version = "0.10.6" }
sha2 = { version = "0.10.8" } sha2 = { version = "0.10.8" }
sys-info = { version = "0.9.1" } sys-info = { version = "0.9.1" }
task-local-extensions = { version = "0.1.4" }
tempfile = { version = "3.9.0" } tempfile = { version = "3.9.0" }
textwrap = { version = "0.16.1" } textwrap = { version = "0.16.1" }
thiserror = { version = "1.0.56" } thiserror = { version = "1.0.56" }
@ -133,7 +135,7 @@ toml = { version = "0.8.12" }
tracing = { version = "0.1.40" } tracing = { version = "0.1.40" }
tracing-durations-export = { version = "0.2.0", features = ["plot"] } tracing-durations-export = { version = "0.2.0", features = ["plot"] }
tracing-indicatif = { version = "0.3.6" } tracing-indicatif = { version = "0.3.6" }
tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json", "registry"] }
tracing-tree = { version = "0.3.0" } tracing-tree = { version = "0.3.0" }
unicode-width = { version = "0.1.11" } unicode-width = { version = "0.1.11" }
unscanny = { version = "0.1.0" } unscanny = { version = "0.1.0" }
@ -197,7 +199,7 @@ lto = "thin"
# Config for 'cargo dist' # Config for 'cargo dist'
[workspace.metadata.dist] [workspace.metadata.dist]
# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) # The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
cargo-dist-version = "0.13.0-prerelease.3" cargo-dist-version = "0.13.0"
# CI backends to support # CI backends to support
ci = ["github"] ci = ["github"]
# The installers to generate for each app # The installers to generate for each app
@ -207,8 +209,24 @@ windows-archive = ".zip"
# The archive format to use for non-windows builds (defaults .tar.xz) # The archive format to use for non-windows builds (defaults .tar.xz)
unix-archive = ".tar.gz" unix-archive = ".tar.gz"
# Target platforms to build apps for (Rust target-triple syntax) # Target platforms to build apps for (Rust target-triple syntax)
targets = ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", "i686-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin", "aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl", "i686-unknown-linux-musl", "x86_64-pc-windows-msvc", "i686-pc-windows-msvc", "armv7-unknown-linux-gnueabihf", "powerpc64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "s390x-unknown-linux-gnu", "armv7-unknown-linux-musleabihf", "arm-unknown-linux-musleabihf"] targets = [
# Whether to auto-include files like READMEs and CHANGELOGs (default true) "aarch64-apple-darwin",
"aarch64-unknown-linux-gnu",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-musleabihf",
"armv7-unknown-linux-gnueabihf",
"armv7-unknown-linux-musleabihf",
"i686-pc-windows-msvc",
"i686-unknown-linux-gnu",
"i686-unknown-linux-musl",
"powerpc64-unknown-linux-gnu",
"powerpc64le-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-pc-windows-msvc",
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
]# Whether to auto-include files like READMEs and CHANGELOGs (default true)
auto-includes = false auto-includes = false
# Whether cargo-dist should create a Github Release or use an existing draft # Whether cargo-dist should create a Github Release or use an existing draft
create-release = true create-release = true

20
PIP_COMPATIBILITY.md
View File

@ -30,7 +30,7 @@ drawbacks:
target tool the user is expecting to use. target tool the user is expecting to use.
4. It prevents uv from introducing any settings or configuration that don't exist in the target 4. It prevents uv from introducing any settings or configuration that don't exist in the target
tool, since otherwise `pip.conf` (or similar) would no longer be usable with `pip`. tool, since otherwise `pip.conf` (or similar) would no longer be usable with `pip`.
5. It can lead user confusion, since uv would be reading settings that don't actually affect its 5. It can lead to user confusion, since uv would be reading settings that don't actually affect its
behavior, and many users may _not_ expect uv to read configuration files intended for other behavior, and many users may _not_ expect uv to read configuration files intended for other
tools. tools.
@ -107,7 +107,7 @@ the available versions of a given package. However, uv and `pip` differ in how t
packages that exist on multiple indexes. packages that exist on multiple indexes.
For example, imagine that a company publishes an internal version of `requests` on a private index For example, imagine that a company publishes an internal version of `requests` on a private index
(`--extra-index-url`), but also allow installing packages from PyPI by default. In this case, the (`--extra-index-url`), but also allows installing packages from PyPI by default. In this case, the
private `requests` would conflict with the public [`requests`](https://pypi.org/project/requests/) private `requests` would conflict with the public [`requests`](https://pypi.org/project/requests/)
on PyPI. on PyPI.
@ -117,7 +117,7 @@ finds a match. This means that if a package exists on multiple indexes, uv will
candidate versions to those present in the first index that contains the package. candidate versions to those present in the first index that contains the package.
`pip`, meanwhile, will combine the candidate versions from all indexes, and select the best `pip`, meanwhile, will combine the candidate versions from all indexes, and select the best
version from the combined set., though it makes [no guarantees around the order](https://github.com/pypa/pip/issues/5045#issuecomment-369521345) version from the combined set, though it makes [no guarantees around the order](https://github.com/pypa/pip/issues/5045#issuecomment-369521345)
in which it searches indexes, and expects that packages are unique up to name and version, even in which it searches indexes, and expects that packages are unique up to name and version, even
across indexes. across indexes.
@ -128,6 +128,12 @@ internal package, thus causing the malicious package to be installed instead of
package. See, for example, [the `torchtriton` attack](https://pytorch.org/blog/compromised-nightly-dependency/) package. See, for example, [the `torchtriton` attack](https://pytorch.org/blog/compromised-nightly-dependency/)
from December 2022. from December 2022.
As of v0.1.29, users can opt in to `pip`-style behavior for multiple indexes via the
`--index-strategy unsafe-any-match` command-line option, or the `UV_INDEX_STRATEGY` environment
variable. When enabled, uv will search for each package across all indexes, and consider all
available versions when resolving dependencies, prioritizing the `--extra-index-url` indexes over
the default index URL. (Versions that are duplicated _across_ indexes will be ignored.)
In the future, uv will support pinning packages to dedicated indexes (see: [#171](https://github.com/astral-sh/uv/issues/171)). In the future, uv will support pinning packages to dedicated indexes (see: [#171](https://github.com/astral-sh/uv/issues/171)).
Additionally, [PEP 708](https://peps.python.org/pep-0708/) is a provisional standard that aims to Additionally, [PEP 708](https://peps.python.org/pep-0708/) is a provisional standard that aims to
address the "dependency confusion" issue across package registries and installers. address the "dependency confusion" issue across package registries and installers.
@ -253,14 +259,6 @@ When uv resolutions differ from `pip` in undesirable ways, it's often a sign tha
are too loose, and that the user should consider tightening them. For example, in the case of are too loose, and that the user should consider tightening them. For example, in the case of
`starlette` and `fastapi`, the user could require `fastapi>=0.110.0`. `starlette` and `fastapi`, the user could require `fastapi>=0.110.0`.
## Hash-checking mode
While uv will include hashes via `uv pip compile --generate-hashes`, it does not support
hash-checking mode, which is a feature of `pip` that allows users to verify the integrity of
downloaded packages by checking their hashes against those provided in the `requirements.txt` file.
In the future, uv will support hash-checking mode. For more, see [#474](https://github.com/astral-sh/uv/issues/474).
## `pip check` ## `pip check`
At present, `uv pip check` will surface the following diagnostics: At present, `uv pip check` will surface the following diagnostics:

28
README.md
View File

@ -261,8 +261,18 @@ The specifics of uv's caching semantics vary based on the nature of the dependen
- **For Git dependencies**, uv caches based on the fully-resolved Git commit hash. As such, - **For Git dependencies**, uv caches based on the fully-resolved Git commit hash. As such,
`uv pip compile` will pin Git dependencies to a specific commit hash when writing the resolved `uv pip compile` will pin Git dependencies to a specific commit hash when writing the resolved
dependency set. dependency set.
- **For local dependencies**, uv caches based on the last-modified time of the `setup.py` or - **For local dependencies**, uv caches based on the last-modified time of the source archive (i.e.,
`pyproject.toml` file. the local `.whl` or `.tar.gz` file). For directories, uv caches based on the last-modified time of
the `pyproject.toml`, `setup.py`, or `setup.cfg` file.
It's safe to run multiple `uv` commands concurrently, even against the same virtual environment.
uv's cache is designed to be thread-safe and append-only, and thus robust to multiple concurrent
readers and writers. uv applies a file-based lock to the target virtual environment when installing,
to avoid concurrent modifications across processes.
Note that it's _not_ safe to modify the uv cache directly (e.g., `uv cache clean`) while other `uv`
commands are running, and _never_ safe to modify the cache directly (e.g., by removing a file or
directory).
If you're running into caching issues, uv includes a few escape hatches: If you're running into caching issues, uv includes a few escape hatches:
@ -445,10 +455,20 @@ uv accepts the following command-line arguments as environment variables:
`allow`, uv will allow pre-release versions for all dependencies. `allow`, uv will allow pre-release versions for all dependencies.
- `UV_SYSTEM_PYTHON`: Equivalent to the `--system` command-line argument. If set to `true`, uv - `UV_SYSTEM_PYTHON`: Equivalent to the `--system` command-line argument. If set to `true`, uv
will use the first Python interpreter found in the system `PATH`. will use the first Python interpreter found in the system `PATH`.
WARNING: `UV_SYSTEM_PYTHON=true` is intended for use in continuous integration (CI) environments and WARNING: `UV_SYSTEM_PYTHON=true` is intended for use in continuous integration (CI) or
should be used with caution, as it can modify the system Python installation. containerized environments and should be used with caution, as modifying the system Python
can lead to unexpected behavior.
- `UV_BREAK_SYSTEM_PACKAGES`: Equivalent to the `--break-system-packages` command-line argument. If
set to `true`, uv will allow the installation of packages that conflict with system-installed
packages.
WARNING: `UV_BREAK_SYSTEM_PACKAGES=true` is intended for use in continuous integration (CI) or
containerized environments and should be used with caution, as modifying the system Python
can lead to unexpected behavior.
- `UV_NATIVE_TLS`: Equivalent to the `--native-tls` command-line argument. If set to `true`, uv - `UV_NATIVE_TLS`: Equivalent to the `--native-tls` command-line argument. If set to `true`, uv
will use the system's trust store instead of the bundled `webpki-roots` crate. will use the system's trust store instead of the bundled `webpki-roots` crate.
- `UV_INDEX_STRATEGY`: Equivalent to the `--index-strategy` command-line argument. For example, if
set to `unsafe-any-match`, uv will consider versions of a given package available across all
index URLs, rather than limiting its search to the first index URL that contains the package.
In each case, the corresponding command-line argument takes precedence over an environment variable. In each case, the corresponding command-line argument takes precedence over an environment variable.

10
crates/distribution-types/src/buildable.rs
View File

@ -1,6 +1,7 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::path::Path; use std::path::Path;
use pep440_rs::Version;
use url::Url; use url::Url;
use uv_normalize::PackageName; use uv_normalize::PackageName;
@ -28,6 +29,15 @@ impl BuildableSource<'_> {
} }
} }
/// Return the [`Version`] of the source, if available.
pub fn version(&self) -> Option<&Version> {
match self {
Self::Dist(SourceDist::Registry(dist)) => Some(&dist.filename.version),
Self::Dist(_) => None,
Self::Url(_) => None,
}
}
/// Return the [`BuildableSource`] as a [`SourceDist`], if it is a distribution. /// Return the [`BuildableSource`] as a [`SourceDist`], if it is a distribution.
pub fn as_dist(&self) -> Option<&SourceDist> { pub fn as_dist(&self) -> Option<&SourceDist> {
match self { match self {

48
crates/distribution-types/src/cached.rs
View File

@ -4,9 +4,11 @@ use anyhow::Result;
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
use pep508_rs::VerbatimUrl; use pep508_rs::VerbatimUrl;
use pypi_types::HashDigest;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use crate::direct_url::{DirectUrl, LocalFileUrl}; use crate::direct_url::{DirectUrl, LocalFileUrl};
use crate::hash::Hashed;
use crate::{ use crate::{
BuiltDist, Dist, DistributionMetadata, InstalledMetadata, InstalledVersion, Name, SourceDist, BuiltDist, Dist, DistributionMetadata, InstalledMetadata, InstalledVersion, Name, SourceDist,
VersionOrUrl, VersionOrUrl,
@ -25,6 +27,7 @@ pub enum CachedDist {
pub struct CachedRegistryDist { pub struct CachedRegistryDist {
pub filename: WheelFilename, pub filename: WheelFilename,
pub path: PathBuf, pub path: PathBuf,
pub hashes: Vec<HashDigest>,
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@ -33,45 +36,60 @@ pub struct CachedDirectUrlDist {
pub url: VerbatimUrl, pub url: VerbatimUrl,
pub path: PathBuf, pub path: PathBuf,
pub editable: bool, pub editable: bool,
pub hashes: Vec<HashDigest>,
} }
impl CachedDist { impl CachedDist {
/// Initialize a [`CachedDist`] from a [`Dist`]. /// Initialize a [`CachedDist`] from a [`Dist`].
pub fn from_remote(remote: Dist, filename: WheelFilename, path: PathBuf) -> Self { pub fn from_remote(
remote: Dist,
filename: WheelFilename,
hashes: Vec<HashDigest>,
path: PathBuf,
) -> Self {
match remote { match remote {
Dist::Built(BuiltDist::Registry(_dist)) => { Dist::Built(BuiltDist::Registry(_dist)) => Self::Registry(CachedRegistryDist {
Self::Registry(CachedRegistryDist { filename, path }) filename,
} path,
hashes,
}),
Dist::Built(BuiltDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist { Dist::Built(BuiltDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist {
filename, filename,
url: dist.url, url: dist.url,
hashes,
path, path,
editable: false, editable: false,
}), }),
Dist::Built(BuiltDist::Path(dist)) => Self::Url(CachedDirectUrlDist { Dist::Built(BuiltDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
filename, filename,
url: dist.url, url: dist.url,
hashes,
path, path,
editable: false, editable: false,
}), }),
Dist::Source(SourceDist::Registry(_dist)) => { Dist::Source(SourceDist::Registry(_dist)) => Self::Registry(CachedRegistryDist {
Self::Registry(CachedRegistryDist { filename, path }) filename,
} path,
hashes,
}),
Dist::Source(SourceDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist { Dist::Source(SourceDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist {
filename, filename,
url: dist.url, url: dist.url,
hashes,
path, path,
editable: false, editable: false,
}), }),
Dist::Source(SourceDist::Git(dist)) => Self::Url(CachedDirectUrlDist { Dist::Source(SourceDist::Git(dist)) => Self::Url(CachedDirectUrlDist {
filename, filename,
url: dist.url, url: dist.url,
hashes,
path, path,
editable: false, editable: false,
}), }),
Dist::Source(SourceDist::Path(dist)) => Self::Url(CachedDirectUrlDist { Dist::Source(SourceDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
filename, filename,
url: dist.url, url: dist.url,
hashes,
path, path,
editable: dist.editable, editable: dist.editable,
}), }),
@ -104,6 +122,7 @@ impl CachedDist {
} }
} }
/// Returns `true` if the distribution is editable.
pub fn editable(&self) -> bool { pub fn editable(&self) -> bool {
match self { match self {
Self::Registry(_) => false, Self::Registry(_) => false,
@ -111,6 +130,7 @@ impl CachedDist {
} }
} }
/// Returns the [`WheelFilename`] of the distribution.
pub fn filename(&self) -> &WheelFilename { pub fn filename(&self) -> &WheelFilename {
match self { match self {
Self::Registry(dist) => &dist.filename, Self::Registry(dist) => &dist.filename,
@ -119,12 +139,24 @@ impl CachedDist {
} }
} }
impl Hashed for CachedRegistryDist {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}
impl CachedDirectUrlDist { impl CachedDirectUrlDist {
/// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`url::Url`], and [`Path`]. /// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`url::Url`], and [`Path`].
pub fn from_url(filename: WheelFilename, url: VerbatimUrl, path: PathBuf) -> Self { pub fn from_url(
filename: WheelFilename,
url: VerbatimUrl,
hashes: Vec<HashDigest>,
path: PathBuf,
) -> Self {
Self { Self {
filename, filename,
url, url,
hashes,
path, path,
editable: false, editable: false,
} }

52
crates/distribution-types/src/editable.rs
View File

@ -1,4 +1,6 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;
use std::path::PathBuf; use std::path::PathBuf;
use url::Url; use url::Url;
@ -41,3 +43,53 @@ impl std::fmt::Display for LocalEditable {
std::fmt::Display::fmt(&self.url, f) std::fmt::Display::fmt(&self.url, f)
} }
} }
/// A collection of [`LocalEditable`]s.
#[derive(Debug, Clone)]
pub struct LocalEditables(Vec<LocalEditable>);
impl LocalEditables {
/// Merge and dedupe a list of [`LocalEditable`]s.
///
/// This function will deduplicate any editables that point to identical paths, merging their
/// extras.
pub fn from_editables(editables: impl Iterator<Item = LocalEditable>) -> Self {
let mut map = BTreeMap::new();
for editable in editables {
match map.entry(editable.path.clone()) {
Entry::Vacant(entry) => {
entry.insert(editable);
}
Entry::Occupied(mut entry) => {
let existing = entry.get_mut();
existing.extras.extend(editable.extras);
}
}
}
Self(map.into_values().collect())
}
/// Return the number of editables.
pub fn len(&self) -> usize {
self.0.len()
}
/// Return whether the editables are empty.
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Return the editables as a vector.
pub fn into_vec(self) -> Vec<LocalEditable> {
self.0
}
}
impl IntoIterator for LocalEditables {
type Item = LocalEditable;
type IntoIter = std::vec::IntoIter<LocalEditable>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}

18
crates/distribution-types/src/file.rs
View File

@ -6,7 +6,8 @@ use thiserror::Error;
use url::Url; use url::Url;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError}; use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
use pypi_types::{DistInfoMetadata, Hashes, Yanked}; use pep508_rs::split_scheme;
use pypi_types::{DistInfoMetadata, HashDigest, Yanked};
/// Error converting [`pypi_types::File`] to [`distribution_type::File`]. /// Error converting [`pypi_types::File`] to [`distribution_type::File`].
#[derive(Debug, Error)] #[derive(Debug, Error)]
@ -24,9 +25,9 @@ pub enum FileConversionError {
#[archive(check_bytes)] #[archive(check_bytes)]
#[archive_attr(derive(Debug))] #[archive_attr(derive(Debug))]
pub struct File { pub struct File {
pub dist_info_metadata: Option<DistInfoMetadata>, pub dist_info_metadata: bool,
pub filename: String, pub filename: String,
pub hashes: Hashes, pub hashes: Vec<HashDigest>,
pub requires_python: Option<VersionSpecifiers>, pub requires_python: Option<VersionSpecifiers>,
pub size: Option<u64>, pub size: Option<u64>,
// N.B. We don't use a chrono DateTime<Utc> here because it's a little // N.B. We don't use a chrono DateTime<Utc> here because it's a little
@ -42,19 +43,24 @@ impl File {
/// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers /// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers
pub fn try_from(file: pypi_types::File, base: &Url) -> Result<Self, FileConversionError> { pub fn try_from(file: pypi_types::File, base: &Url) -> Result<Self, FileConversionError> {
Ok(Self { Ok(Self {
dist_info_metadata: file.dist_info_metadata, dist_info_metadata: file
.dist_info_metadata
.as_ref()
.is_some_and(DistInfoMetadata::is_available),
filename: file.filename, filename: file.filename,
hashes: file.hashes, hashes: file.hashes.into_digests(),
requires_python: file requires_python: file
.requires_python .requires_python
.transpose() .transpose()
.map_err(|err| FileConversionError::RequiresPython(err.line().clone(), err))?, .map_err(|err| FileConversionError::RequiresPython(err.line().clone(), err))?,
size: file.size, size: file.size,
upload_time_utc_ms: file.upload_time.map(|dt| dt.timestamp_millis()), upload_time_utc_ms: file.upload_time.map(|dt| dt.timestamp_millis()),
url: if file.url.contains("://") { url: {
if split_scheme(&file.url).is_some() {
FileLocation::AbsoluteUrl(file.url) FileLocation::AbsoluteUrl(file.url)
} else { } else {
FileLocation::RelativeUrl(base.to_string(), file.url) FileLocation::RelativeUrl(base.to_string(), file.url)
}
}, },
yanked: file.yanked, yanked: file.yanked,
}) })

84
crates/distribution-types/src/hash.rs Normal file
View File

@ -0,0 +1,84 @@
use pypi_types::{HashAlgorithm, HashDigest};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HashPolicy<'a> {
/// No hash policy is specified.
None,
/// Hashes should be generated (specifically, a SHA-256 hash), but not validated.
Generate,
/// Hashes should be validated against a pre-defined list of hashes. If necessary, hashes should
/// be generated so as to ensure that the archive is valid.
Validate(&'a [HashDigest]),
}
impl<'a> HashPolicy<'a> {
/// Returns `true` if the hash policy is `None`.
pub fn is_none(&self) -> bool {
matches!(self, Self::None)
}
/// Returns `true` if the hash policy is `Generate`.
pub fn is_generate(&self) -> bool {
matches!(self, Self::Generate)
}
/// Returns `true` if the hash policy is `Validate`.
pub fn is_validate(&self) -> bool {
matches!(self, Self::Validate(_))
}
/// Return the algorithms used in the hash policy.
pub fn algorithms(&self) -> Vec<HashAlgorithm> {
match self {
Self::None => vec![],
Self::Generate => vec![HashAlgorithm::Sha256],
Self::Validate(hashes) => {
let mut algorithms = hashes.iter().map(HashDigest::algorithm).collect::<Vec<_>>();
algorithms.sort();
algorithms.dedup();
algorithms
}
}
}
/// Return the digests used in the hash policy.
pub fn digests(&self) -> &[HashDigest] {
match self {
Self::None => &[],
Self::Generate => &[],
Self::Validate(hashes) => hashes,
}
}
}
pub trait Hashed {
/// Return the [`HashDigest`]s for the archive.
fn hashes(&self) -> &[HashDigest];
/// Returns `true` if the archive satisfies the given hash policy.
fn satisfies(&self, hashes: HashPolicy) -> bool {
match hashes {
HashPolicy::None => true,
HashPolicy::Generate => self
.hashes()
.iter()
.any(|hash| hash.algorithm == HashAlgorithm::Sha256),
HashPolicy::Validate(hashes) => self.hashes().iter().any(|hash| hashes.contains(hash)),
}
}
/// Returns `true` if the archive includes a hash for at least one of the given algorithms.
fn has_digests(&self, hashes: HashPolicy) -> bool {
match hashes {
HashPolicy::None => true,
HashPolicy::Generate => self
.hashes()
.iter()
.any(|hash| hash.algorithm == HashAlgorithm::Sha256),
HashPolicy::Validate(hashes) => hashes
.iter()
.map(HashDigest::algorithm)
.any(|algorithm| self.hashes().iter().any(|hash| hash.algorithm == algorithm)),
}
}
}

94
crates/distribution-types/src/id.rs
View File

@ -1,30 +1,66 @@
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use std::path::PathBuf;
use cache_key::{CanonicalUrl, RepositoryUrl};
use url::Url; use url::Url;
use pep440_rs::Version; use pep440_rs::Version;
use pypi_types::HashDigest;
use uv_normalize::PackageName; use uv_normalize::PackageName;
/// A unique identifier for a package (e.g., `black==23.10.0`). /// A unique identifier for a package. A package can either be identified by a name (e.g., `black`)
/// or a URL (e.g., `git+https://github.com/psf/black`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum PackageId { pub enum PackageId {
NameVersion(PackageName, Version), /// The identifier consists of a package name.
Url(String), Name(PackageName),
/// The identifier consists of a URL.
Url(CanonicalUrl),
} }
impl PackageId { impl PackageId {
/// Create a new [`PackageId`] from a package name and version. /// Create a new [`PackageId`] from a package name and version.
pub fn from_registry(name: PackageName, version: Version) -> Self { pub fn from_registry(name: PackageName) -> Self {
Self::NameVersion(name, version) Self::Name(name)
} }
/// Create a new [`PackageId`] from a URL. /// Create a new [`PackageId`] from a URL.
pub fn from_url(url: &Url) -> Self { pub fn from_url(url: &Url) -> Self {
Self::Url(cache_key::digest(&cache_key::CanonicalUrl::new(url))) Self::Url(CanonicalUrl::new(url))
} }
} }
impl Display for PackageId { impl Display for PackageId {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Name(name) => write!(f, "{name}"),
Self::Url(url) => write!(f, "{url}"),
}
}
}
/// A unique identifier for a package at a specific version (e.g., `black==23.10.0`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum VersionId {
/// The identifier consists of a package name and version.
NameVersion(PackageName, Version),
/// The identifier consists of a URL.
Url(CanonicalUrl),
}
impl VersionId {
/// Create a new [`VersionId`] from a package name and version.
pub fn from_registry(name: PackageName, version: Version) -> Self {
Self::NameVersion(name, version)
}
/// Create a new [`VersionId`] from a URL.
pub fn from_url(url: &Url) -> Self {
Self::Url(CanonicalUrl::new(url))
}
}
impl Display for VersionId {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self { match self {
Self::NameVersion(name, version) => write!(f, "{name}-{version}"), Self::NameVersion(name, version) => write!(f, "{name}-{version}"),
@ -33,33 +69,39 @@ impl Display for PackageId {
} }
} }
/// A unique identifier for a distribution (e.g., `black-23.10.0-py3-none-any.whl`). /// A unique resource identifier for the distribution, like a SHA-256 hash of the distribution's
/// contents.
///
/// A distribution is a specific archive of a package at a specific version. For a given package
/// version, there may be multiple distributions, e.g., source distribution, along with
/// multiple binary distributions (wheels) for different platforms. As a concrete example,
/// `black-23.10.0-py3-none-any.whl` would represent a (binary) distribution of the `black` package
/// at version `23.10.0`.
///
/// The distribution ID is used to uniquely identify a distribution. Ideally, the distribution
/// ID should be a hash of the distribution's contents, though in practice, it's only required
/// that the ID is unique within a single invocation of the resolver (and so, e.g., a hash of
/// the URL would also be sufficient).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DistributionId(String); pub enum DistributionId {
Url(CanonicalUrl),
impl DistributionId { PathBuf(PathBuf),
pub fn new(id: impl Into<String>) -> Self { Digest(HashDigest),
Self(id.into()) AbsoluteUrl(String),
} RelativeUrl(String, String),
}
impl DistributionId {
pub fn as_str(&self) -> &str {
&self.0
}
} }
/// A unique identifier for a resource, like a URL or a Git repository. /// A unique identifier for a resource, like a URL or a Git repository.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ResourceId(String); pub enum ResourceId {
Url(RepositoryUrl),
impl ResourceId { PathBuf(PathBuf),
pub fn new(id: impl Into<String>) -> Self { Digest(HashDigest),
Self(id.into()) AbsoluteUrl(String),
} RelativeUrl(String, String),
} }
impl From<&Self> for PackageId { impl From<&Self> for VersionId {
/// Required for `WaitMap::wait`. /// Required for `WaitMap::wait`.
fn from(value: &Self) -> Self { fn from(value: &Self) -> Self {
value.clone() value.clone()

20
crates/distribution-types/src/index_url.rs
View File

@ -24,6 +24,7 @@ static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
pub enum IndexUrl { pub enum IndexUrl {
Pypi(VerbatimUrl), Pypi(VerbatimUrl),
Url(VerbatimUrl), Url(VerbatimUrl),
Path(VerbatimUrl),
} }
impl IndexUrl { impl IndexUrl {
@ -32,6 +33,7 @@ impl IndexUrl {
match self { match self {
Self::Pypi(url) => url.raw(), Self::Pypi(url) => url.raw(),
Self::Url(url) => url.raw(), Self::Url(url) => url.raw(),
Self::Path(url) => url.raw(),
} }
} }
} }
@ -41,6 +43,7 @@ impl Display for IndexUrl {
match self { match self {
Self::Pypi(url) => Display::fmt(url, f), Self::Pypi(url) => Display::fmt(url, f),
Self::Url(url) => Display::fmt(url, f), Self::Url(url) => Display::fmt(url, f),
Self::Path(url) => Display::fmt(url, f),
} }
} }
} }
@ -50,6 +53,7 @@ impl Verbatim for IndexUrl {
match self { match self {
Self::Pypi(url) => url.verbatim(), Self::Pypi(url) => url.verbatim(),
Self::Url(url) => url.verbatim(), Self::Url(url) => url.verbatim(),
Self::Path(url) => url.verbatim(),
} }
} }
} }
@ -83,6 +87,7 @@ impl From<IndexUrl> for Url {
match index { match index {
IndexUrl::Pypi(url) => url.to_url(), IndexUrl::Pypi(url) => url.to_url(),
IndexUrl::Url(url) => url.to_url(), IndexUrl::Url(url) => url.to_url(),
IndexUrl::Path(url) => url.to_url(),
} }
} }
} }
@ -94,6 +99,7 @@ impl Deref for IndexUrl {
match &self { match &self {
Self::Pypi(url) => url, Self::Pypi(url) => url,
Self::Url(url) => url, Self::Url(url) => url,
Self::Path(url) => url,
} }
} }
} }
@ -165,15 +171,7 @@ impl Display for FlatIndexLocation {
} }
} }
/// The index locations to use for fetching packages. /// The index locations to use for fetching packages. By default, uses the PyPI index.
///
/// By default, uses the PyPI index.
///
/// "pip treats all package sources equally" (<https://github.com/pypa/pip/issues/8606#issuecomment-788754817>),
/// and so do we, i.e., you can't rely that on any particular order of querying indices.
///
/// If the fields are none and empty, ignore the package index, instead rely on local archives and
/// caches.
/// ///
/// From a pip perspective, this type merges `--index-url`, `--extra-index-url`, and `--find-links`. /// From a pip perspective, this type merges `--index-url`, `--extra-index-url`, and `--find-links`.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@ -338,7 +336,9 @@ impl<'a> IndexUrls {
} }
} }
/// Return an iterator over all [`IndexUrl`] entries. /// Return an iterator over all [`IndexUrl`] entries in order.
///
/// Prioritizes the extra indexes over the main index.
/// ///
/// If `no_index` was enabled, then this always returns an empty /// If `no_index` was enabled, then this always returns an empty
/// iterator. /// iterator.

97
crates/distribution-types/src/lib.rs
View File

@ -51,6 +51,7 @@ pub use crate::direct_url::*;
pub use crate::editable::*; pub use crate::editable::*;
pub use crate::error::*; pub use crate::error::*;
pub use crate::file::*; pub use crate::file::*;
pub use crate::hash::*;
pub use crate::id::*; pub use crate::id::*;
pub use crate::index_url::*; pub use crate::index_url::*;
pub use crate::installed::*; pub use crate::installed::*;
@ -66,6 +67,7 @@ mod direct_url;
mod editable; mod editable;
mod error; mod error;
mod file; mod file;
mod hash;
mod id; mod id;
mod index_url; mod index_url;
mod installed; mod installed;
@ -371,6 +373,14 @@ impl Dist {
} }
} }
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Built(dist) => dist.index(),
Self::Source(dist) => dist.index(),
}
}
/// Returns the [`File`] instance, if this dist is from a registry with simple json api support /// Returns the [`File`] instance, if this dist is from a registry with simple json api support
pub fn file(&self) -> Option<&File> { pub fn file(&self) -> Option<&File> {
match self { match self {
@ -388,7 +398,16 @@ impl Dist {
} }
impl BuiltDist { impl BuiltDist {
/// Returns the [`File`] instance, if this dist is from a registry with simple json api support /// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Registry(registry) => Some(&registry.index),
Self::DirectUrl(_) => None,
Self::Path(_) => None,
}
}
/// Returns the [`File`] instance, if this distribution is from a registry.
pub fn file(&self) -> Option<&File> { pub fn file(&self) -> Option<&File> {
match self { match self {
Self::Registry(registry) => Some(&registry.file), Self::Registry(registry) => Some(&registry.file),
@ -406,6 +425,14 @@ impl BuiltDist {
} }
impl SourceDist { impl SourceDist {
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Registry(registry) => Some(&registry.index),
Self::DirectUrl(_) | Self::Git(_) | Self::Path(_) => None,
}
}
/// Returns the [`File`] instance, if this dist is from a registry with simple json api support /// Returns the [`File`] instance, if this dist is from a registry with simple json api support
pub fn file(&self) -> Option<&File> { pub fn file(&self) -> Option<&File> {
match self { match self {
@ -764,26 +791,26 @@ impl RemoteSource for Dist {
impl Identifier for Url { impl Identifier for Url {
fn distribution_id(&self) -> DistributionId { fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&cache_key::CanonicalUrl::new(self))) DistributionId::Url(cache_key::CanonicalUrl::new(self))
} }
fn resource_id(&self) -> ResourceId { fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&cache_key::RepositoryUrl::new(self))) ResourceId::Url(cache_key::RepositoryUrl::new(self))
} }
} }
impl Identifier for File { impl Identifier for File {
fn distribution_id(&self) -> DistributionId { fn distribution_id(&self) -> DistributionId {
if let Some(hash) = self.hashes.as_str() { if let Some(hash) = self.hashes.first() {
DistributionId::new(hash) DistributionId::Digest(hash.clone())
} else { } else {
self.url.distribution_id() self.url.distribution_id()
} }
} }
fn resource_id(&self) -> ResourceId { fn resource_id(&self) -> ResourceId {
if let Some(hash) = self.hashes.as_str() { if let Some(hash) = self.hashes.first() {
ResourceId::new(hash) ResourceId::Digest(hash.clone())
} else { } else {
self.url.resource_id() self.url.resource_id()
} }
@ -792,67 +819,31 @@ impl Identifier for File {
impl Identifier for Path { impl Identifier for Path {
fn distribution_id(&self) -> DistributionId { fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self)) DistributionId::PathBuf(self.to_path_buf())
} }
fn resource_id(&self) -> ResourceId { fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self)) ResourceId::PathBuf(self.to_path_buf())
}
}
impl Identifier for String {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for &str {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for (&str, &str) {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
}
}
impl Identifier for (&Url, &str) {
fn distribution_id(&self) -> DistributionId {
DistributionId::new(cache_key::digest(&self))
}
fn resource_id(&self) -> ResourceId {
ResourceId::new(cache_key::digest(&self))
} }
} }
impl Identifier for FileLocation { impl Identifier for FileLocation {
fn distribution_id(&self) -> DistributionId { fn distribution_id(&self) -> DistributionId {
match self { match self {
Self::RelativeUrl(base, url) => (base.as_str(), url.as_str()).distribution_id(), Self::RelativeUrl(base, url) => {
Self::AbsoluteUrl(url) => url.distribution_id(), DistributionId::RelativeUrl(base.to_string(), url.to_string())
}
Self::AbsoluteUrl(url) => DistributionId::AbsoluteUrl(url.to_string()),
Self::Path(path) => path.distribution_id(), Self::Path(path) => path.distribution_id(),
} }
} }
fn resource_id(&self) -> ResourceId { fn resource_id(&self) -> ResourceId {
match self { match self {
Self::RelativeUrl(base, url) => (base.as_str(), url.as_str()).resource_id(), Self::RelativeUrl(base, url) => {
Self::AbsoluteUrl(url) => url.resource_id(), ResourceId::RelativeUrl(base.to_string(), url.to_string())
}
Self::AbsoluteUrl(url) => ResourceId::AbsoluteUrl(url.to_string()),
Self::Path(path) => path.resource_id(), Self::Path(path) => path.resource_id(),
} }
} }

131
crates/distribution-types/src/prioritized_distribution.rs
View File

@ -1,8 +1,8 @@
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use pep440_rs::VersionSpecifiers; use pep440_rs::VersionSpecifiers;
use platform_tags::{IncompatibleTag, TagCompatibility, TagPriority}; use platform_tags::{IncompatibleTag, TagPriority};
use pypi_types::{Hashes, Yanked}; use pypi_types::{HashDigest, Yanked};
use crate::{Dist, InstalledDist, ResolvedDistRef}; use crate::{Dist, InstalledDist, ResolvedDistRef};
@ -18,7 +18,7 @@ struct PrioritizedDistInner {
/// The highest-priority wheel. /// The highest-priority wheel.
wheel: Option<(Dist, WheelCompatibility)>, wheel: Option<(Dist, WheelCompatibility)>,
/// The hashes for each distribution. /// The hashes for each distribution.
hashes: Vec<Hashes>, hashes: Vec<HashDigest>,
} }
/// A distribution that can be used for both resolution and installation. /// A distribution that can be used for both resolution and installation.
@ -113,7 +113,7 @@ impl Display for IncompatibleDist {
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum WheelCompatibility { pub enum WheelCompatibility {
Incompatible(IncompatibleWheel), Incompatible(IncompatibleWheel),
Compatible(TagPriority), Compatible(Hash, TagPriority),
} }
#[derive(Debug, PartialEq, Eq, Clone)] #[derive(Debug, PartialEq, Eq, Clone)]
@ -128,7 +128,7 @@ pub enum IncompatibleWheel {
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum SourceDistCompatibility { pub enum SourceDistCompatibility {
Incompatible(IncompatibleSource), Incompatible(IncompatibleSource),
Compatible, Compatible(Hash),
} }
#[derive(Debug, PartialEq, Eq, Clone)] #[derive(Debug, PartialEq, Eq, Clone)]
@ -139,26 +139,40 @@ pub enum IncompatibleSource {
NoBuild, NoBuild,
} }
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum Hash {
/// The hash is present, but does not match the expected value.
Mismatched,
/// The hash is missing.
Missing,
/// The hash matches the expected value.
Matched,
}
impl PrioritizedDist { impl PrioritizedDist {
/// Create a new [`PrioritizedDist`] from the given wheel distribution. /// Create a new [`PrioritizedDist`] from the given wheel distribution.
pub fn from_built(dist: Dist, hash: Option<Hashes>, compatibility: WheelCompatibility) -> Self { pub fn from_built(
dist: Dist,
hashes: Vec<HashDigest>,
compatibility: WheelCompatibility,
) -> Self {
Self(Box::new(PrioritizedDistInner { Self(Box::new(PrioritizedDistInner {
wheel: Some((dist, compatibility)), wheel: Some((dist, compatibility)),
source: None, source: None,
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(), hashes,
})) }))
} }
/// Create a new [`PrioritizedDist`] from the given source distribution. /// Create a new [`PrioritizedDist`] from the given source distribution.
pub fn from_source( pub fn from_source(
dist: Dist, dist: Dist,
hash: Option<Hashes>, hashes: Vec<HashDigest>,
compatibility: SourceDistCompatibility, compatibility: SourceDistCompatibility,
) -> Self { ) -> Self {
Self(Box::new(PrioritizedDistInner { Self(Box::new(PrioritizedDistInner {
wheel: None, wheel: None,
source: Some((dist, compatibility)), source: Some((dist, compatibility)),
hashes: hash.map(|hash| vec![hash]).unwrap_or_default(), hashes,
})) }))
} }
@ -166,7 +180,7 @@ impl PrioritizedDist {
pub fn insert_built( pub fn insert_built(
&mut self, &mut self,
dist: Dist, dist: Dist,
hash: Option<Hashes>, hashes: Vec<HashDigest>,
compatibility: WheelCompatibility, compatibility: WheelCompatibility,
) { ) {
// Track the highest-priority wheel. // Track the highest-priority wheel.
@ -178,16 +192,14 @@ impl PrioritizedDist {
self.0.wheel = Some((dist, compatibility)); self.0.wheel = Some((dist, compatibility));
} }
if let Some(hash) = hash { self.0.hashes.extend(hashes);
self.0.hashes.push(hash);
}
} }
/// Insert the given source distribution into the [`PrioritizedDist`]. /// Insert the given source distribution into the [`PrioritizedDist`].
pub fn insert_source( pub fn insert_source(
&mut self, &mut self,
dist: Dist, dist: Dist,
hash: Option<Hashes>, hashes: Vec<HashDigest>,
compatibility: SourceDistCompatibility, compatibility: SourceDistCompatibility,
) { ) {
// Track the highest-priority source. // Track the highest-priority source.
@ -199,16 +211,25 @@ impl PrioritizedDist {
self.0.source = Some((dist, compatibility)); self.0.source = Some((dist, compatibility));
} }
if let Some(hash) = hash { self.0.hashes.extend(hashes);
self.0.hashes.push(hash);
}
} }
/// Return the highest-priority distribution for the package version, if any. /// Return the highest-priority distribution for the package version, if any.
pub fn get(&self) -> Option<CompatibleDist> { pub fn get(&self) -> Option<CompatibleDist> {
match (&self.0.wheel, &self.0.source) { match (&self.0.wheel, &self.0.source) {
// If both are compatible, break ties based on the hash.
(
Some((wheel, WheelCompatibility::Compatible(wheel_hash, tag_priority))),
Some((source_dist, SourceDistCompatibility::Compatible(source_hash))),
) => {
if source_hash > wheel_hash {
Some(CompatibleDist::SourceDist(source_dist))
} else {
Some(CompatibleDist::CompatibleWheel(wheel, *tag_priority))
}
}
// Prefer the highest-priority, platform-compatible wheel. // Prefer the highest-priority, platform-compatible wheel.
(Some((wheel, WheelCompatibility::Compatible(tag_priority))), _) => { (Some((wheel, WheelCompatibility::Compatible(_, tag_priority))), _) => {
Some(CompatibleDist::CompatibleWheel(wheel, *tag_priority)) Some(CompatibleDist::CompatibleWheel(wheel, *tag_priority))
} }
// If we have a compatible source distribution and an incompatible wheel, return the // If we have a compatible source distribution and an incompatible wheel, return the
@ -217,64 +238,42 @@ impl PrioritizedDist {
// using the wheel is faster. // using the wheel is faster.
( (
Some((wheel, WheelCompatibility::Incompatible(_))), Some((wheel, WheelCompatibility::Incompatible(_))),
Some((source_dist, SourceDistCompatibility::Compatible)), Some((source_dist, SourceDistCompatibility::Compatible(_))),
) => Some(CompatibleDist::IncompatibleWheel { source_dist, wheel }), ) => Some(CompatibleDist::IncompatibleWheel { source_dist, wheel }),
// Otherwise, if we have a source distribution, return it. // Otherwise, if we have a source distribution, return it.
(None, Some((source_dist, SourceDistCompatibility::Compatible))) => { (None, Some((source_dist, SourceDistCompatibility::Compatible(_)))) => {
Some(CompatibleDist::SourceDist(source_dist)) Some(CompatibleDist::SourceDist(source_dist))
} }
_ => None, _ => None,
} }
} }
/// Return the compatible source distribution, if any.
pub fn compatible_source(&self) -> Option<&Dist> {
self.0
.source
.as_ref()
.and_then(|(dist, compatibility)| match compatibility {
SourceDistCompatibility::Compatible => Some(dist),
SourceDistCompatibility::Incompatible(_) => None,
})
}
/// Return the incompatible source distribution, if any. /// Return the incompatible source distribution, if any.
pub fn incompatible_source(&self) -> Option<(&Dist, &IncompatibleSource)> { pub fn incompatible_source(&self) -> Option<(&Dist, &IncompatibleSource)> {
self.0 self.0
.source .source
.as_ref() .as_ref()
.and_then(|(dist, compatibility)| match compatibility { .and_then(|(dist, compatibility)| match compatibility {
SourceDistCompatibility::Compatible => None, SourceDistCompatibility::Compatible(_) => None,
SourceDistCompatibility::Incompatible(incompatibility) => { SourceDistCompatibility::Incompatible(incompatibility) => {
Some((dist, incompatibility)) Some((dist, incompatibility))
} }
}) })
} }
/// Return the compatible built distribution, if any.
pub fn compatible_wheel(&self) -> Option<(&Dist, TagPriority)> {
self.0
.wheel
.as_ref()
.and_then(|(dist, compatibility)| match compatibility {
WheelCompatibility::Compatible(priority) => Some((dist, *priority)),
WheelCompatibility::Incompatible(_) => None,
})
}
/// Return the incompatible built distribution, if any. /// Return the incompatible built distribution, if any.
pub fn incompatible_wheel(&self) -> Option<(&Dist, &IncompatibleWheel)> { pub fn incompatible_wheel(&self) -> Option<(&Dist, &IncompatibleWheel)> {
self.0 self.0
.wheel .wheel
.as_ref() .as_ref()
.and_then(|(dist, compatibility)| match compatibility { .and_then(|(dist, compatibility)| match compatibility {
WheelCompatibility::Compatible(_) => None, WheelCompatibility::Compatible(_, _) => None,
WheelCompatibility::Incompatible(incompatibility) => Some((dist, incompatibility)), WheelCompatibility::Incompatible(incompatibility) => Some((dist, incompatibility)),
}) })
} }
/// Return the hashes for each distribution. /// Return the hashes for each distribution.
pub fn hashes(&self) -> &[Hashes] { pub fn hashes(&self) -> &[HashDigest] {
&self.0.hashes &self.0.hashes
} }
@ -311,11 +310,23 @@ impl<'a> CompatibleDist<'a> {
} => ResolvedDistRef::Installable(source_dist), } => ResolvedDistRef::Installable(source_dist),
} }
} }
/// Returns whether the distribution is a source distribution.
///
/// Avoid building source distributions we don't need.
pub fn prefetchable(&self) -> bool {
match *self {
CompatibleDist::SourceDist(_) => false,
CompatibleDist::InstalledDist(_)
| CompatibleDist::CompatibleWheel(_, _)
| CompatibleDist::IncompatibleWheel { .. } => true,
}
}
} }
impl WheelCompatibility { impl WheelCompatibility {
pub fn is_compatible(&self) -> bool { pub fn is_compatible(&self) -> bool {
matches!(self, Self::Compatible(_)) matches!(self, Self::Compatible(_, _))
} }
/// Return `true` if the current compatibility is more compatible than another. /// Return `true` if the current compatibility is more compatible than another.
@ -324,11 +335,12 @@ impl WheelCompatibility {
/// Compatible wheel ordering is determined by tag priority. /// Compatible wheel ordering is determined by tag priority.
pub fn is_more_compatible(&self, other: &Self) -> bool { pub fn is_more_compatible(&self, other: &Self) -> bool {
match (self, other) { match (self, other) {
(Self::Compatible(_), Self::Incompatible(_)) => true, (Self::Compatible(_, _), Self::Incompatible(_)) => true,
(Self::Compatible(tag_priority), Self::Compatible(other_tag_priority)) => { (
tag_priority > other_tag_priority Self::Compatible(hash, tag_priority),
} Self::Compatible(other_hash, other_tag_priority),
(Self::Incompatible(_), Self::Compatible(_)) => false, ) => (hash, tag_priority) > (other_hash, other_tag_priority),
(Self::Incompatible(_), Self::Compatible(_, _)) => false,
(Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => { (Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => {
incompatibility.is_more_compatible(other_incompatibility) incompatibility.is_more_compatible(other_incompatibility)
} }
@ -344,9 +356,11 @@ impl SourceDistCompatibility {
/// Incompatible source distribution priority selects a source distribution that was "closest" to being usable. /// Incompatible source distribution priority selects a source distribution that was "closest" to being usable.
pub fn is_more_compatible(&self, other: &Self) -> bool { pub fn is_more_compatible(&self, other: &Self) -> bool {
match (self, other) { match (self, other) {
(Self::Compatible, Self::Incompatible(_)) => true, (Self::Compatible(_), Self::Incompatible(_)) => true,
(Self::Compatible, Self::Compatible) => false, // Arbitrary (Self::Compatible(compatibility), Self::Compatible(other_compatibility)) => {
(Self::Incompatible(_), Self::Compatible) => false, compatibility > other_compatibility
}
(Self::Incompatible(_), Self::Compatible(_)) => false,
(Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => { (Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => {
incompatibility.is_more_compatible(other_incompatibility) incompatibility.is_more_compatible(other_incompatibility)
} }
@ -354,15 +368,6 @@ impl SourceDistCompatibility {
} }
} }
impl From<TagCompatibility> for WheelCompatibility {
fn from(value: TagCompatibility) -> Self {
match value {
TagCompatibility::Compatible(priority) => Self::Compatible(priority),
TagCompatibility::Incompatible(tag) => Self::Incompatible(IncompatibleWheel::Tag(tag)),
}
}
}
impl IncompatibleSource { impl IncompatibleSource {
fn is_more_compatible(&self, other: &Self) -> bool { fn is_more_compatible(&self, other: &Self) -> bool {
match self { match self {

23
crates/distribution-types/src/resolved.rs
View File

@ -1,10 +1,10 @@
use std::fmt::Display; use std::fmt::{Display, Formatter};
use pep508_rs::PackageName; use pep508_rs::PackageName;
use crate::{ use crate::{
Dist, DistributionId, DistributionMetadata, Identifier, InstalledDist, Name, ResourceId, Dist, DistributionId, DistributionMetadata, Identifier, IndexUrl, InstalledDist, Name,
VersionOrUrl, ResourceId, VersionOrUrl,
}; };
/// A distribution that can be used for resolution and installation. /// A distribution that can be used for resolution and installation.
@ -31,6 +31,14 @@ impl ResolvedDist {
Self::Installed(dist) => dist.is_editable(), Self::Installed(dist) => dist.is_editable(),
} }
} }
/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Self::Installable(dist) => dist.index(),
Self::Installed(_) => None,
}
}
} }
impl ResolvedDistRef<'_> { impl ResolvedDistRef<'_> {
@ -42,6 +50,15 @@ impl ResolvedDistRef<'_> {
} }
} }
impl Display for ResolvedDistRef<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Installable(dist) => Display::fmt(dist, f),
Self::Installed(dist) => Display::fmt(dist, f),
}
}
}
impl Name for ResolvedDistRef<'_> { impl Name for ResolvedDistRef<'_> {
fn name(&self) -> &PackageName { fn name(&self) -> &PackageName {
match self { match self {

35
crates/distribution-types/src/traits.rs
View File

@ -10,7 +10,8 @@ use crate::{
BuiltDist, CachedDirectUrlDist, CachedDist, CachedRegistryDist, DirectUrlBuiltDist, BuiltDist, CachedDirectUrlDist, CachedDist, CachedRegistryDist, DirectUrlBuiltDist,
DirectUrlSourceDist, Dist, DistributionId, GitSourceDist, InstalledDirectUrlDist, DirectUrlSourceDist, Dist, DistributionId, GitSourceDist, InstalledDirectUrlDist,
InstalledDist, InstalledRegistryDist, InstalledVersion, LocalDist, PackageId, PathBuiltDist, InstalledDist, InstalledRegistryDist, InstalledVersion, LocalDist, PackageId, PathBuiltDist,
PathSourceDist, RegistryBuiltDist, RegistrySourceDist, ResourceId, SourceDist, VersionOrUrl, PathSourceDist, RegistryBuiltDist, RegistrySourceDist, ResourceId, SourceDist, VersionId,
VersionOrUrl,
}; };
pub trait Name { pub trait Name {
@ -25,16 +26,29 @@ pub trait DistributionMetadata: Name {
/// for URL-based distributions. /// for URL-based distributions.
fn version_or_url(&self) -> VersionOrUrl; fn version_or_url(&self) -> VersionOrUrl;
/// Returns a unique identifier for the package. /// Returns a unique identifier for the package at the given version (e.g., `black==23.10.0`).
/// ///
/// Note that this is not equivalent to a unique identifier for the _distribution_, as multiple /// Note that this is not equivalent to a unique identifier for the _distribution_, as multiple
/// registry-based distributions (e.g., different wheels for the same package and version) /// registry-based distributions (e.g., different wheels for the same package and version)
/// will return the same package ID, but different distribution IDs. /// will return the same version ID, but different distribution IDs.
fn package_id(&self) -> PackageId { fn version_id(&self) -> VersionId {
match self.version_or_url() { match self.version_or_url() {
VersionOrUrl::Version(version) => { VersionOrUrl::Version(version) => {
PackageId::from_registry(self.name().clone(), version.clone()) VersionId::from_registry(self.name().clone(), version.clone())
} }
VersionOrUrl::Url(url) => VersionId::from_url(url),
}
}
/// Returns a unique identifier for a package. A package can either be identified by a name
/// (e.g., `black`) or a URL (e.g., `git+https://github.com/psf/black`).
///
/// Note that this is not equivalent to a unique identifier for the _distribution_, as multiple
/// registry-based distributions (e.g., different wheels for the same package and version)
/// will return the same version ID, but different distribution IDs.
fn package_id(&self) -> PackageId {
match self.version_or_url() {
VersionOrUrl::Version(_) => PackageId::from_registry(self.name().clone()),
VersionOrUrl::Url(url) => PackageId::from_url(url), VersionOrUrl::Url(url) => PackageId::from_url(url),
} }
} }
@ -57,6 +71,17 @@ pub trait RemoteSource {
pub trait Identifier { pub trait Identifier {
/// Return a unique resource identifier for the distribution, like a SHA-256 hash of the /// Return a unique resource identifier for the distribution, like a SHA-256 hash of the
/// distribution's contents. /// distribution's contents.
///
/// A distribution is a specific archive of a package at a specific version. For a given package
/// version, there may be multiple distributions, e.g., source distribution, along with
/// multiple binary distributions (wheels) for different platforms. As a concrete example,
/// `black-23.10.0-py3-none-any.whl` would represent a (binary) distribution of the `black` package
/// at version `23.10.0`.
///
/// The distribution ID is used to uniquely identify a distribution. Ideally, the distribution
/// ID should be a hash of the distribution's contents, though in practice, it's only required
/// that the ID is unique within a single invocation of the resolver (and so, e.g., a hash of
/// the URL would also be sufficient).
fn distribution_id(&self) -> DistributionId; fn distribution_id(&self) -> DistributionId;
/// Return a unique resource identifier for the underlying resource backing the distribution. /// Return a unique resource identifier for the underlying resource backing the distribution.

2
crates/pep508-rs/src/lib.rs
View File

@ -72,7 +72,7 @@ pub enum Pep508ErrorSource {
String(String), String(String),
/// A URL parsing error. /// A URL parsing error.
#[error(transparent)] #[error(transparent)]
UrlError(#[from] verbatim_url::VerbatimUrlError), UrlError(#[from] VerbatimUrlError),
/// The version requirement is not supported. /// The version requirement is not supported.
#[error("{0}")] #[error("{0}")]
UnsupportedRequirement(String), UnsupportedRequirement(String),

28
crates/pypi-types/src/base_url.rs
View File

@ -1,37 +1,19 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use url::Url; use url::Url;
/// Join a possibly relative URL to a base URL. /// Join a relative URL to a base URL.
/// pub fn base_url_join_relative(base: &str, relative: &str) -> Result<Url, JoinRelativeError> {
/// When `maybe_relative` is not relative, then it is parsed and returned with
/// `base` being ignored.
///
/// This is useful for parsing URLs that may be absolute or relative, with a
/// known base URL, and that doesn't require having already parsed a `BaseUrl`.
pub fn base_url_join_relative(base: &str, maybe_relative: &str) -> Result<Url, JoinRelativeError> {
match Url::parse(maybe_relative) {
Ok(absolute) => Ok(absolute),
Err(err) => {
if err == url::ParseError::RelativeUrlWithoutBase {
let base_url = Url::parse(base).map_err(|err| JoinRelativeError::ParseError { let base_url = Url::parse(base).map_err(|err| JoinRelativeError::ParseError {
original: base.to_string(), original: base.to_string(),
source: err, source: err,
})?; })?;
base_url base_url
.join(maybe_relative) .join(relative)
.map_err(|_| JoinRelativeError::ParseError { .map_err(|err| JoinRelativeError::ParseError {
original: format!("{base}/{maybe_relative}"), original: format!("{base}/{relative}"),
source: err, source: err,
}) })
} else {
Err(JoinRelativeError::ParseError {
original: maybe_relative.to_string(),
source: err,
})
}
}
}
} }
/// An error that occurs when `base_url_join_relative` fails. /// An error that occurs when `base_url_join_relative` fails.

108
crates/pypi-types/src/metadata.rs
View File

@ -1,9 +1,8 @@
//! Derived from `pypi_types_crate`. //! Derived from `pypi_types_crate`.
use indexmap::IndexMap;
use std::io;
use std::str::FromStr; use std::str::FromStr;
use indexmap::IndexMap;
use mailparse::{MailHeaderMap, MailParseError}; use mailparse::{MailHeaderMap, MailParseError};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
@ -39,38 +38,17 @@ pub struct Metadata23 {
/// ///
/// The error type /// The error type
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum MetadataError {
/// I/O error
#[error(transparent)]
Io(#[from] io::Error),
/// mail parse error
#[error(transparent)] #[error(transparent)]
MailParse(#[from] MailParseError), MailParse(#[from] MailParseError),
/// TOML parse error
#[error(transparent)] #[error(transparent)]
Toml(#[from] toml::de::Error), Toml(#[from] toml::de::Error),
/// Metadata field not found
#[error("metadata field {0} not found")] #[error("metadata field {0} not found")]
FieldNotFound(&'static str), FieldNotFound(&'static str),
/// Unknown distribution type
#[error("unknown distribution type")]
UnknownDistributionType,
/// Metadata file not found
#[error("metadata file not found")]
MetadataNotFound,
/// Invalid project URL (no comma)
#[error("Invalid Project-URL field (missing comma): '{0}'")]
InvalidProjectUrl(String),
/// Multiple metadata files found
#[error("found multiple metadata files: {0:?}")]
MultipleMetadataFiles(Vec<String>),
/// Invalid Version
#[error("invalid version: {0}")] #[error("invalid version: {0}")]
Pep440VersionError(VersionParseError), Pep440VersionError(VersionParseError),
/// Invalid VersionSpecifier
#[error(transparent)] #[error(transparent)]
Pep440Error(#[from] VersionSpecifiersParseError), Pep440Error(#[from] VersionSpecifiersParseError),
/// Invalid Requirement
#[error(transparent)] #[error(transparent)]
Pep508Error(#[from] Pep508Error), Pep508Error(#[from] Pep508Error),
#[error(transparent)] #[error(transparent)]
@ -86,20 +64,20 @@ pub enum Error {
/// From <https://github.com/PyO3/python-pkginfo-rs/blob/d719988323a0cfea86d4737116d7917f30e819e2/src/metadata.rs#LL78C2-L91C26> /// From <https://github.com/PyO3/python-pkginfo-rs/blob/d719988323a0cfea86d4737116d7917f30e819e2/src/metadata.rs#LL78C2-L91C26>
impl Metadata23 { impl Metadata23 {
/// Parse the [`Metadata23`] from a `METADATA` file, as included in a built distribution (wheel). /// Parse the [`Metadata23`] from a `METADATA` file, as included in a built distribution (wheel).
pub fn parse_metadata(content: &[u8]) -> Result<Self, Error> { pub fn parse_metadata(content: &[u8]) -> Result<Self, MetadataError> {
let headers = Headers::parse(content)?; let headers = Headers::parse(content)?;
let name = PackageName::new( let name = PackageName::new(
headers headers
.get_first_value("Name") .get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?, .ok_or(MetadataError::FieldNotFound("Name"))?,
)?; )?;
let version = Version::from_str( let version = Version::from_str(
&headers &headers
.get_first_value("Version") .get_first_value("Version")
.ok_or(Error::FieldNotFound("Version"))?, .ok_or(MetadataError::FieldNotFound("Version"))?,
) )
.map_err(Error::Pep440VersionError)?; .map_err(MetadataError::Pep440VersionError)?;
let requires_dist = headers let requires_dist = headers
.get_all_values("Requires-Dist") .get_all_values("Requires-Dist")
.map(|requires_dist| { .map(|requires_dist| {
@ -135,28 +113,28 @@ impl Metadata23 {
/// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2 /// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2
/// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and /// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and
/// `Provides-Extra`) are marked as dynamic. /// `Provides-Extra`) are marked as dynamic.
pub fn parse_pkg_info(content: &[u8]) -> Result<Self, Error> { pub fn parse_pkg_info(content: &[u8]) -> Result<Self, MetadataError> {
let headers = Headers::parse(content)?; let headers = Headers::parse(content)?;
// To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be // To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be
// present and set to a value of at least `2.2`. // present and set to a value of at least `2.2`.
let metadata_version = headers let metadata_version = headers
.get_first_value("Metadata-Version") .get_first_value("Metadata-Version")
.ok_or(Error::FieldNotFound("Metadata-Version"))?; .ok_or(MetadataError::FieldNotFound("Metadata-Version"))?;
// Parse the version into (major, minor). // Parse the version into (major, minor).
let (major, minor) = parse_version(&metadata_version)?; let (major, minor) = parse_version(&metadata_version)?;
if (major, minor) < (2, 2) || (major, minor) >= (3, 0) { if (major, minor) < (2, 2) || (major, minor) >= (3, 0) {
return Err(Error::UnsupportedMetadataVersion(metadata_version)); return Err(MetadataError::UnsupportedMetadataVersion(metadata_version));
} }
// If any of the fields we need are marked as dynamic, we can't use the `PKG-INFO` file. // If any of the fields we need are marked as dynamic, we can't use the `PKG-INFO` file.
let dynamic = headers.get_all_values("Dynamic").collect::<Vec<_>>(); let dynamic = headers.get_all_values("Dynamic").collect::<Vec<_>>();
for field in dynamic { for field in dynamic {
match field.as_str() { match field.as_str() {
"Requires-Python" => return Err(Error::DynamicField("Requires-Python")), "Requires-Python" => return Err(MetadataError::DynamicField("Requires-Python")),
"Requires-Dist" => return Err(Error::DynamicField("Requires-Dist")), "Requires-Dist" => return Err(MetadataError::DynamicField("Requires-Dist")),
"Provides-Extra" => return Err(Error::DynamicField("Provides-Extra")), "Provides-Extra" => return Err(MetadataError::DynamicField("Provides-Extra")),
_ => (), _ => (),
} }
} }
@ -165,14 +143,14 @@ impl Metadata23 {
let name = PackageName::new( let name = PackageName::new(
headers headers
.get_first_value("Name") .get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?, .ok_or(MetadataError::FieldNotFound("Name"))?,
)?; )?;
let version = Version::from_str( let version = Version::from_str(
&headers &headers
.get_first_value("Version") .get_first_value("Version")
.ok_or(Error::FieldNotFound("Version"))?, .ok_or(MetadataError::FieldNotFound("Version"))?,
) )
.map_err(Error::Pep440VersionError)?; .map_err(MetadataError::Pep440VersionError)?;
// The remaining fields are required to be present. // The remaining fields are required to be present.
let requires_dist = headers let requires_dist = headers
@ -208,29 +186,31 @@ impl Metadata23 {
} }
/// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621. /// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621.
pub fn parse_pyproject_toml(contents: &str) -> Result<Self, Error> { pub fn parse_pyproject_toml(contents: &str) -> Result<Self, MetadataError> {
let pyproject_toml: PyProjectToml = toml::from_str(contents)?; let pyproject_toml: PyProjectToml = toml::from_str(contents)?;
let project = pyproject_toml let project = pyproject_toml
.project .project
.ok_or(Error::FieldNotFound("project"))?; .ok_or(MetadataError::FieldNotFound("project"))?;
// If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file. // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file.
let dynamic = project.dynamic.unwrap_or_default(); let dynamic = project.dynamic.unwrap_or_default();
for field in dynamic { for field in dynamic {
match field.as_str() { match field.as_str() {
"dependencies" => return Err(Error::DynamicField("dependencies")), "dependencies" => return Err(MetadataError::DynamicField("dependencies")),
"optional-dependencies" => { "optional-dependencies" => {
return Err(Error::DynamicField("optional-dependencies")) return Err(MetadataError::DynamicField("optional-dependencies"))
} }
"requires-python" => return Err(Error::DynamicField("requires-python")), "requires-python" => return Err(MetadataError::DynamicField("requires-python")),
"version" => return Err(Error::DynamicField("version")), "version" => return Err(MetadataError::DynamicField("version")),
_ => (), _ => (),
} }
} }
let name = project.name; let name = project.name;
let version = project.version.ok_or(Error::FieldNotFound("version"))?; let version = project
.version
.ok_or(MetadataError::FieldNotFound("version"))?;
let requires_python = project.requires_python.map(VersionSpecifiers::from); let requires_python = project.requires_python.map(VersionSpecifiers::from);
// Extract the requirements. // Extract the requirements.
@ -309,28 +289,31 @@ pub struct Metadata10 {
impl Metadata10 { impl Metadata10 {
/// Parse the [`Metadata10`] from a `PKG-INFO` file, as included in a source distribution. /// Parse the [`Metadata10`] from a `PKG-INFO` file, as included in a source distribution.
pub fn parse_pkg_info(content: &[u8]) -> Result<Self, Error> { pub fn parse_pkg_info(content: &[u8]) -> Result<Self, MetadataError> {
let headers = Headers::parse(content)?; let headers = Headers::parse(content)?;
let name = PackageName::new( let name = PackageName::new(
headers headers
.get_first_value("Name") .get_first_value("Name")
.ok_or(Error::FieldNotFound("Name"))?, .ok_or(MetadataError::FieldNotFound("Name"))?,
)?; )?;
Ok(Self { name }) Ok(Self { name })
} }
} }
/// Parse a `Metadata-Version` field into a (major, minor) tuple. /// Parse a `Metadata-Version` field into a (major, minor) tuple.
fn parse_version(metadata_version: &str) -> Result<(u8, u8), Error> { fn parse_version(metadata_version: &str) -> Result<(u8, u8), MetadataError> {
let (major, minor) = metadata_version let (major, minor) =
metadata_version
.split_once('.') .split_once('.')
.ok_or(Error::InvalidMetadataVersion(metadata_version.to_string()))?; .ok_or(MetadataError::InvalidMetadataVersion(
metadata_version.to_string(),
))?;
let major = major let major = major
.parse::<u8>() .parse::<u8>()
.map_err(|_| Error::InvalidMetadataVersion(metadata_version.to_string()))?; .map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?;
let minor = minor let minor = minor
.parse::<u8>() .parse::<u8>()
.map_err(|_| Error::InvalidMetadataVersion(metadata_version.to_string()))?; .map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?;
Ok((major, minor)) Ok((major, minor))
} }
@ -373,7 +356,7 @@ mod tests {
use pep440_rs::Version; use pep440_rs::Version;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use crate::Error; use crate::MetadataError;
use super::Metadata23; use super::Metadata23;
@ -381,11 +364,11 @@ mod tests {
fn test_parse_metadata() { fn test_parse_metadata() {
let s = "Metadata-Version: 1.0"; let s = "Metadata-Version: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes()); let meta = Metadata23::parse_metadata(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Name")))); assert!(matches!(meta, Err(MetadataError::FieldNotFound("Name"))));
let s = "Metadata-Version: 1.0\nName: asdf"; let s = "Metadata-Version: 1.0\nName: asdf";
let meta = Metadata23::parse_metadata(s.as_bytes()); let meta = Metadata23::parse_metadata(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Version")))); assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version"))));
let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0"; let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap();
@ -404,22 +387,25 @@ mod tests {
let s = "Metadata-Version: 1.0\nName: =?utf-8?q?=C3=A4_space?= <x@y.org>\nVersion: 1.0"; let s = "Metadata-Version: 1.0\nName: =?utf-8?q?=C3=A4_space?= <x@y.org>\nVersion: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes()); let meta = Metadata23::parse_metadata(s.as_bytes());
assert!(matches!(meta, Err(Error::InvalidName(_)))); assert!(matches!(meta, Err(MetadataError::InvalidName(_))));
} }
#[test] #[test]
fn test_parse_pkg_info() { fn test_parse_pkg_info() {
let s = "Metadata-Version: 2.1"; let s = "Metadata-Version: 2.1";
let meta = Metadata23::parse_pkg_info(s.as_bytes()); let meta = Metadata23::parse_pkg_info(s.as_bytes());
assert!(matches!(meta, Err(Error::UnsupportedMetadataVersion(_)))); assert!(matches!(
meta,
Err(MetadataError::UnsupportedMetadataVersion(_))
));
let s = "Metadata-Version: 2.2\nName: asdf"; let s = "Metadata-Version: 2.2\nName: asdf";
let meta = Metadata23::parse_pkg_info(s.as_bytes()); let meta = Metadata23::parse_pkg_info(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Version")))); assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version"))));
let s = "Metadata-Version: 2.3\nName: asdf"; let s = "Metadata-Version: 2.3\nName: asdf";
let meta = Metadata23::parse_pkg_info(s.as_bytes()); let meta = Metadata23::parse_pkg_info(s.as_bytes());
assert!(matches!(meta, Err(Error::FieldNotFound("Version")))); assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version"))));
let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0"; let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap();
@ -428,7 +414,7 @@ mod tests {
let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nDynamic: Requires-Dist"; let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nDynamic: Requires-Dist";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap_err(); let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap_err();
assert!(matches!(meta, Error::DynamicField("Requires-Dist"))); assert!(matches!(meta, MetadataError::DynamicField("Requires-Dist")));
let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo"; let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap();
@ -444,7 +430,7 @@ mod tests {
name = "asdf" name = "asdf"
"#; "#;
let meta = Metadata23::parse_pyproject_toml(s); let meta = Metadata23::parse_pyproject_toml(s);
assert!(matches!(meta, Err(Error::FieldNotFound("version")))); assert!(matches!(meta, Err(MetadataError::FieldNotFound("version"))));
let s = r#" let s = r#"
[project] [project]
@ -452,7 +438,7 @@ mod tests {
dynamic = ["version"] dynamic = ["version"]
"#; "#;
let meta = Metadata23::parse_pyproject_toml(s); let meta = Metadata23::parse_pyproject_toml(s);
assert!(matches!(meta, Err(Error::DynamicField("version")))); assert!(matches!(meta, Err(MetadataError::DynamicField("version"))));
let s = r#" let s = r#"
[project] [project]

187
crates/pypi-types/src/simple_json.rs
View File

@ -68,11 +68,7 @@ where
)) ))
} }
#[derive( #[derive(Debug, Clone, Deserialize)]
Debug, Clone, Serialize, Deserialize, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
#[serde(untagged)] #[serde(untagged)]
pub enum DistInfoMetadata { pub enum DistInfoMetadata {
Bool(bool), Bool(bool),
@ -125,23 +121,7 @@ impl Default for Yanked {
/// A dictionary mapping a hash name to a hex encoded digest of the file. /// A dictionary mapping a hash name to a hex encoded digest of the file.
/// ///
/// PEP 691 says multiple hashes can be included and the interpretation is left to the client. /// PEP 691 says multiple hashes can be included and the interpretation is left to the client.
#[derive( #[derive(Debug, Clone, Eq, PartialEq, Default, Deserialize)]
Debug,
Clone,
Ord,
PartialOrd,
Eq,
PartialEq,
Hash,
Default,
Serialize,
Deserialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
pub struct Hashes { pub struct Hashes {
pub md5: Option<Box<str>>, pub md5: Option<Box<str>>,
pub sha256: Option<Box<str>>, pub sha256: Option<Box<str>>,
@ -150,31 +130,34 @@ pub struct Hashes {
} }
impl Hashes { impl Hashes {
/// Format as `<algorithm>:<hash>`. /// Convert a set of [`Hashes`] into a list of [`HashDigest`]s.
pub fn to_string(&self) -> Option<String> { pub fn into_digests(self) -> Vec<HashDigest> {
self.sha512 let mut digests = Vec::new();
.as_ref() if let Some(sha512) = self.sha512 {
.map(|sha512| format!("sha512:{sha512}")) digests.push(HashDigest {
.or_else(|| { algorithm: HashAlgorithm::Sha512,
self.sha384 digest: sha512,
.as_ref() });
.map(|sha384| format!("sha384:{sha384}"))
})
.or_else(|| {
self.sha256
.as_ref()
.map(|sha256| format!("sha256:{sha256}"))
})
.or_else(|| self.md5.as_ref().map(|md5| format!("md5:{md5}")))
} }
if let Some(sha384) = self.sha384 {
/// Return the hash digest. digests.push(HashDigest {
pub fn as_str(&self) -> Option<&str> { algorithm: HashAlgorithm::Sha384,
self.sha512 digest: sha384,
.as_deref() });
.or(self.sha384.as_deref()) }
.or(self.sha256.as_deref()) if let Some(sha256) = self.sha256 {
.or(self.md5.as_deref()) digests.push(HashDigest {
algorithm: HashAlgorithm::Sha256,
digest: sha256,
});
}
if let Some(md5) = self.md5 {
digests.push(HashDigest {
algorithm: HashAlgorithm::Md5,
digest: md5,
});
}
digests
} }
} }
@ -239,6 +222,118 @@ impl FromStr for Hashes {
} }
} }
#[derive(
Debug,
Clone,
Copy,
Ord,
PartialOrd,
Eq,
PartialEq,
Hash,
Serialize,
Deserialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
pub enum HashAlgorithm {
Md5,
Sha256,
Sha384,
Sha512,
}
impl FromStr for HashAlgorithm {
type Err = HashError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"md5" => Ok(Self::Md5),
"sha256" => Ok(Self::Sha256),
"sha384" => Ok(Self::Sha384),
"sha512" => Ok(Self::Sha512),
_ => Err(HashError::UnsupportedHashAlgorithm(s.to_string())),
}
}
}
impl std::fmt::Display for HashAlgorithm {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Md5 => write!(f, "md5"),
Self::Sha256 => write!(f, "sha256"),
Self::Sha384 => write!(f, "sha384"),
Self::Sha512 => write!(f, "sha512"),
}
}
}
/// A hash name and hex encoded digest of the file.
#[derive(
Debug,
Clone,
Ord,
PartialOrd,
Eq,
PartialEq,
Hash,
Serialize,
Deserialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[archive(check_bytes)]
#[archive_attr(derive(Debug))]
pub struct HashDigest {
pub algorithm: HashAlgorithm,
pub digest: Box<str>,
}
impl HashDigest {
/// Return the [`HashAlgorithm`] of the digest.
pub fn algorithm(&self) -> HashAlgorithm {
self.algorithm
}
}
impl std::fmt::Display for HashDigest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}:{}", self.algorithm, self.digest)
}
}
impl FromStr for HashDigest {
type Err = HashError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut parts = s.split(':');
// Extract the key and value.
let name = parts
.next()
.ok_or_else(|| HashError::InvalidStructure(s.to_string()))?;
let value = parts
.next()
.ok_or_else(|| HashError::InvalidStructure(s.to_string()))?;
// Ensure there are no more parts.
if parts.next().is_some() {
return Err(HashError::InvalidStructure(s.to_string()));
}
let algorithm = HashAlgorithm::from_str(name)?;
Ok(HashDigest {
algorithm,
digest: value.to_owned().into_boxed_str(),
})
}
}
#[derive(thiserror::Error, Debug)] #[derive(thiserror::Error, Debug)]
pub enum HashError { pub enum HashError {
#[error("Unexpected hash (expected `<algorithm>:<hash>`): {0}")] #[error("Unexpected hash (expected `<algorithm>:<hash>`): {0}")]

2
crates/requirements-txt/Cargo.toml
View File

@ -17,7 +17,7 @@ pep508_rs = { workspace = true, features = ["rkyv", "serde", "non-pep508-extensi
uv-client = { workspace = true } uv-client = { workspace = true }
uv-fs = { workspace = true } uv-fs = { workspace = true }
uv-normalize = { workspace = true } uv-normalize = { workspace = true }
uv-types = { workspace = true } uv-configuration = { workspace = true }
uv-warnings = { workspace = true } uv-warnings = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }

12
crates/requirements-txt/src/lib.rs
View File

@ -52,9 +52,9 @@ use pep508_rs::{
#[cfg(feature = "http")] #[cfg(feature = "http")]
use uv_client::BaseClient; use uv_client::BaseClient;
use uv_client::BaseClientBuilder; use uv_client::BaseClientBuilder;
use uv_configuration::{NoBinary, NoBuild, PackageNameSpecifier};
use uv_fs::{normalize_url_path, Simplified}; use uv_fs::{normalize_url_path, Simplified};
use uv_normalize::ExtraName; use uv_normalize::ExtraName;
use uv_types::{NoBinary, NoBuild, PackageNameSpecifier};
use uv_warnings::warn_user; use uv_warnings::warn_user;
/// We emit one of those for each requirements.txt entry /// We emit one of those for each requirements.txt entry
@ -293,21 +293,16 @@ impl Display for EditableRequirement {
/// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in /// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in
/// the future also editable an similar information /// the future also editable an similar information
#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Serialize)] #[derive(Debug, Deserialize, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct RequirementEntry { pub struct RequirementEntry {
/// The actual PEP 508 requirement /// The actual PEP 508 requirement
pub requirement: RequirementsTxtRequirement, pub requirement: RequirementsTxtRequirement,
/// Hashes of the downloadable packages /// Hashes of the downloadable packages
pub hashes: Vec<String>, pub hashes: Vec<String>,
/// Editable installation, see e.g. <https://stackoverflow.com/q/35064426/3549270>
pub editable: bool,
} }
impl Display for RequirementEntry { impl Display for RequirementEntry {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if self.editable {
write!(f, "-e ")?;
}
write!(f, "{}", self.requirement)?; write!(f, "{}", self.requirement)?;
for hash in &self.hashes { for hash in &self.hashes {
write!(f, " --hash {hash}")?; write!(f, " --hash {hash}")?;
@ -670,7 +665,6 @@ fn parse_entry(
RequirementsTxtStatement::RequirementEntry(RequirementEntry { RequirementsTxtStatement::RequirementEntry(RequirementEntry {
requirement, requirement,
hashes, hashes,
editable: false,
}) })
} else if let Some(char) = s.peek() { } else if let Some(char) = s.peek() {
let (line, column) = calculate_row_column(content, s.cursor()); let (line, column) = calculate_row_column(content, s.cursor());
@ -1743,7 +1737,6 @@ mod test {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],
@ -1798,7 +1791,6 @@ mod test {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

6
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -77,7 +75,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -102,7 +99,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -127,7 +123,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -152,7 +147,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [ constraints: [

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -57,7 +56,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

4
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -66,7 +64,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -99,7 +96,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -41,7 +40,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

5
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap
View File

@ -56,7 +56,6 @@ RequirementsTxt {
hashes: [ hashes: [
"sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe", "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -110,7 +109,6 @@ RequirementsTxt {
hashes: [ hashes: [
"sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305", "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -175,7 +173,6 @@ RequirementsTxt {
hashes: [ hashes: [
"sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1", "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -230,7 +227,6 @@ RequirementsTxt {
"sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5",
"sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a", "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -287,7 +283,6 @@ RequirementsTxt {
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e", "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e",
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a", "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a",
], ],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -57,7 +56,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

6
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -77,7 +75,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -102,7 +99,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -127,7 +123,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -152,7 +147,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [ constraints: [

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

4
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -66,7 +64,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -99,7 +96,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -41,7 +40,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

1
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

5
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap
View File

@ -56,7 +56,6 @@ RequirementsTxt {
hashes: [ hashes: [
"sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe", "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -110,7 +109,6 @@ RequirementsTxt {
hashes: [ hashes: [
"sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305", "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -175,7 +173,6 @@ RequirementsTxt {
hashes: [ hashes: [
"sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1", "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -230,7 +227,6 @@ RequirementsTxt {
"sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5",
"sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a", "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a",
], ],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -287,7 +283,6 @@ RequirementsTxt {
"sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e", "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e",
"sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a", "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a",
], ],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap
View File

@ -27,7 +27,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -52,7 +51,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

3
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-bare-url.txt.snap
View File

@ -28,7 +28,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Unnamed( requirement: Unnamed(
@ -58,7 +57,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Unnamed( requirement: Unnamed(
@ -84,7 +82,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

2
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap
View File

@ -16,7 +16,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Pep508( requirement: Pep508(
@ -57,7 +56,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

3
crates/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-bare-url.txt.snap
View File

@ -28,7 +28,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Unnamed( requirement: Unnamed(
@ -58,7 +57,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
RequirementEntry { RequirementEntry {
requirement: Unnamed( requirement: Unnamed(
@ -84,7 +82,6 @@ RequirementsTxt {
}, },
), ),
hashes: [], hashes: [],
editable: false,
}, },
], ],
constraints: [], constraints: [],

4
crates/uv-auth/Cargo.toml
View File

@ -7,15 +7,15 @@ edition = "2021"
async-trait = { workspace = true } async-trait = { workspace = true }
base64 = { workspace = true } base64 = { workspace = true }
clap = { workspace = true, features = ["derive", "env"], optional = true } clap = { workspace = true, features = ["derive", "env"], optional = true }
http = { workspace = true }
once_cell = { workspace = true }
reqwest = { workspace = true } reqwest = { workspace = true }
reqwest-middleware = { workspace = true } reqwest-middleware = { workspace = true }
rust-netrc = { workspace = true } rust-netrc = { workspace = true }
task-local-extensions = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
url = { workspace = true } url = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
once_cell = { workspace = true }
[dev-dependencies] [dev-dependencies]
tempfile = { workspace = true } tempfile = { workspace = true }

2
crates/uv-auth/src/middleware.rs
View File

@ -1,9 +1,9 @@
use http::Extensions;
use std::path::Path; use std::path::Path;
use netrc::Netrc; use netrc::Netrc;
use reqwest::{header::HeaderValue, Request, Response}; use reqwest::{header::HeaderValue, Request, Response};
use reqwest_middleware::{Middleware, Next}; use reqwest_middleware::{Middleware, Next};
use task_local_extensions::Extensions;
use tracing::{debug, warn}; use tracing::{debug, warn};
use crate::{ use crate::{

1
crates/uv-build/Cargo.toml
View File

@ -20,6 +20,7 @@ pep508_rs = { workspace = true }
uv-fs = { workspace = true } uv-fs = { workspace = true }
uv-interpreter = { workspace = true } uv-interpreter = { workspace = true }
uv-types = { workspace = true, features = ["serde"] } uv-types = { workspace = true, features = ["serde"] }
uv-configuration = { workspace = true, features = ["serde"] }
uv-virtualenv = { workspace = true } uv-virtualenv = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }

53
crates/uv-build/src/lib.rs
View File

@ -28,11 +28,10 @@ use tracing::{debug, info_span, instrument, Instrument};
use distribution_types::Resolution; use distribution_types::Resolution;
use pep440_rs::Version; use pep440_rs::Version;
use pep508_rs::{PackageName, Requirement}; use pep508_rs::{PackageName, Requirement};
use uv_configuration::{BuildKind, ConfigSettings, SetupPyStrategy};
use uv_fs::{PythonExt, Simplified}; use uv_fs::{PythonExt, Simplified};
use uv_interpreter::{Interpreter, PythonEnvironment}; use uv_interpreter::{Interpreter, PythonEnvironment};
use uv_types::{ use uv_types::{BuildContext, BuildIsolation, SourceBuildTrait};
BuildContext, BuildIsolation, BuildKind, ConfigSettings, SetupPyStrategy, SourceBuildTrait,
};
/// e.g. `pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory` /// e.g. `pygraphviz/graphviz_wrap.c:3020:10: fatal error: graphviz/cgraph.h: No such file or directory`
static MISSING_HEADER_RE: Lazy<Regex> = Lazy::new(|| { static MISSING_HEADER_RE: Lazy<Regex> = Lazy::new(|| {
@ -113,7 +112,7 @@ pub enum MissingLibrary {
#[derive(Debug, Error)] #[derive(Debug, Error)]
pub struct MissingHeaderCause { pub struct MissingHeaderCause {
missing_library: MissingLibrary, missing_library: MissingLibrary,
package_id: String, version_id: String,
} }
impl Display for MissingHeaderCause { impl Display for MissingHeaderCause {
@ -123,22 +122,22 @@ impl Display for MissingHeaderCause {
write!( write!(
f, f,
"This error likely indicates that you need to install a library that provides \"{}\" for {}", "This error likely indicates that you need to install a library that provides \"{}\" for {}",
header, self.package_id header, self.version_id
) )
} }
MissingLibrary::Linker(library) => { MissingLibrary::Linker(library) => {
write!( write!(
f, f,
"This error likely indicates that you need to install the library that provides a shared library \ "This error likely indicates that you need to install the library that provides a shared library \
for {library} for {package_id} (e.g. lib{library}-dev)", for {library} for {version_id} (e.g. lib{library}-dev)",
library = library, package_id = self.package_id library = library, version_id = self.version_id
) )
} }
MissingLibrary::PythonPackage(package) => { MissingLibrary::PythonPackage(package) => {
write!( write!(
f, f,
"This error likely indicates that you need to `uv pip install {package}` into the build environment for {package_id}", "This error likely indicates that you need to `uv pip install {package}` into the build environment for {version_id}",
package = package, package_id = self.package_id package = package, version_id = self.version_id
) )
} }
} }
@ -149,7 +148,7 @@ impl Error {
fn from_command_output( fn from_command_output(
message: String, message: String,
output: &Output, output: &Output,
package_id: impl Into<String>, version_id: impl Into<String>,
) -> Self { ) -> Self {
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
@ -179,7 +178,7 @@ impl Error {
stderr, stderr,
missing_header_cause: MissingHeaderCause { missing_header_cause: MissingHeaderCause {
missing_library, missing_library,
package_id: package_id.into(), version_id: version_id.into(),
}, },
}; };
} }
@ -365,7 +364,7 @@ pub struct SourceBuild {
/// > it created. /// > it created.
metadata_directory: Option<PathBuf>, metadata_directory: Option<PathBuf>,
/// Package id such as `foo-1.2.3`, for error reporting /// Package id such as `foo-1.2.3`, for error reporting
package_id: String, version_id: String,
/// Whether we do a regular PEP 517 build or an PEP 660 editable build /// Whether we do a regular PEP 517 build or an PEP 660 editable build
build_kind: BuildKind, build_kind: BuildKind,
/// Modified PATH that contains the `venv_bin`, `user_path` and `system_path` variables in that order /// Modified PATH that contains the `venv_bin`, `user_path` and `system_path` variables in that order
@ -386,7 +385,7 @@ impl SourceBuild {
interpreter: &Interpreter, interpreter: &Interpreter,
build_context: &impl BuildContext, build_context: &impl BuildContext,
source_build_context: SourceBuildContext, source_build_context: SourceBuildContext,
package_id: String, version_id: String,
setup_py: SetupPyStrategy, setup_py: SetupPyStrategy,
config_settings: ConfigSettings, config_settings: ConfigSettings,
build_isolation: BuildIsolation<'_>, build_isolation: BuildIsolation<'_>,
@ -478,7 +477,7 @@ impl SourceBuild {
&venv, &venv,
pep517_backend, pep517_backend,
build_context, build_context,
&package_id, &version_id,
build_kind, build_kind,
&config_settings, &config_settings,
&environment_variables, &environment_variables,
@ -498,7 +497,7 @@ impl SourceBuild {
build_kind, build_kind,
config_settings, config_settings,
metadata_directory: None, metadata_directory: None,
package_id, version_id,
environment_variables, environment_variables,
modified_path, modified_path,
}) })
@ -696,7 +695,7 @@ impl SourceBuild {
return Err(Error::from_command_output( return Err(Error::from_command_output(
"Build backend failed to determine metadata through `prepare_metadata_for_build_wheel`".to_string(), "Build backend failed to determine metadata through `prepare_metadata_for_build_wheel`".to_string(),
&output, &output,
&self.package_id, &self.version_id,
)); ));
} }
@ -715,8 +714,8 @@ impl SourceBuild {
/// dir. /// dir.
/// ///
/// <https://packaging.python.org/en/latest/specifications/source-distribution-format/> /// <https://packaging.python.org/en/latest/specifications/source-distribution-format/>
#[instrument(skip_all, fields(package_id = self.package_id))] #[instrument(skip_all, fields(version_id = self.version_id))]
pub async fn build(&self, wheel_dir: &Path) -> Result<String, Error> { pub async fn build_wheel(&self, wheel_dir: &Path) -> Result<String, Error> {
// The build scripts run with the extracted root as cwd, so they need the absolute path. // The build scripts run with the extracted root as cwd, so they need the absolute path.
let wheel_dir = fs::canonicalize(wheel_dir)?; let wheel_dir = fs::canonicalize(wheel_dir)?;
@ -751,7 +750,7 @@ impl SourceBuild {
return Err(Error::from_command_output( return Err(Error::from_command_output(
"Failed building wheel through setup.py".to_string(), "Failed building wheel through setup.py".to_string(),
&output, &output,
&self.package_id, &self.version_id,
)); ));
} }
let dist = fs::read_dir(self.source_tree.join("dist"))?; let dist = fs::read_dir(self.source_tree.join("dist"))?;
@ -762,7 +761,7 @@ impl SourceBuild {
"Expected exactly wheel in `dist/` after invoking setup.py, found {dist_dir:?}" "Expected exactly wheel in `dist/` after invoking setup.py, found {dist_dir:?}"
), ),
&output, &output,
&self.package_id) &self.version_id)
); );
}; };
@ -832,7 +831,7 @@ impl SourceBuild {
self.build_kind self.build_kind
), ),
&output, &output,
&self.package_id, &self.version_id,
)); ));
} }
@ -844,7 +843,7 @@ impl SourceBuild {
self.build_kind self.build_kind
), ),
&output, &output,
&self.package_id, &self.version_id,
)); ));
} }
Ok(distribution_filename) Ok(distribution_filename)
@ -857,7 +856,7 @@ impl SourceBuildTrait for SourceBuild {
} }
async fn wheel<'a>(&'a self, wheel_dir: &'a Path) -> anyhow::Result<String> { async fn wheel<'a>(&'a self, wheel_dir: &'a Path) -> anyhow::Result<String> {
Ok(self.build(wheel_dir).await?) Ok(self.build_wheel(wheel_dir).await?)
} }
} }
@ -874,7 +873,7 @@ async fn create_pep517_build_environment(
venv: &PythonEnvironment, venv: &PythonEnvironment,
pep517_backend: &Pep517Backend, pep517_backend: &Pep517Backend,
build_context: &impl BuildContext, build_context: &impl BuildContext,
package_id: &str, version_id: &str,
build_kind: BuildKind, build_kind: BuildKind,
config_settings: &ConfigSettings, config_settings: &ConfigSettings,
environment_variables: &FxHashMap<OsString, OsString>, environment_variables: &FxHashMap<OsString, OsString>,
@ -928,7 +927,7 @@ async fn create_pep517_build_environment(
return Err(Error::from_command_output( return Err(Error::from_command_output(
format!("Build backend failed to determine extra requires with `build_{build_kind}()`"), format!("Build backend failed to determine extra requires with `build_{build_kind}()`"),
&output, &output,
package_id, version_id,
)); ));
} }
@ -939,7 +938,7 @@ async fn create_pep517_build_environment(
"Build backend failed to read extra requires from `get_requires_for_build_{build_kind}`: {err}" "Build backend failed to read extra requires from `get_requires_for_build_{build_kind}`: {err}"
), ),
&output, &output,
package_id, version_id,
) )
})?; })?;
@ -950,7 +949,7 @@ async fn create_pep517_build_environment(
"Build backend failed to return extra requires with `get_requires_for_build_{build_kind}`: {err}" "Build backend failed to return extra requires with `get_requires_for_build_{build_kind}`: {err}"
), ),
&output, &output,
package_id, version_id,
) )
})?; })?;

24
crates/uv-cache/src/archive.rs Normal file
View File

@ -0,0 +1,24 @@
use std::path::Path;
/// A unique identifier for an archive (unzipped wheel) in the cache.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ArchiveId(String);
impl Default for ArchiveId {
fn default() -> Self {
Self::new()
}
}
impl ArchiveId {
/// Generate a new unique identifier for an archive.
pub fn new() -> Self {
Self(nanoid::nanoid!())
}
}
impl AsRef<Path> for ArchiveId {
fn as_ref(&self) -> &Path {
self.0.as_ref()
}
}

35
crates/uv-cache/src/lib.rs
View File

@ -23,7 +23,9 @@ use crate::removal::{rm_rf, Removal};
pub use crate::timestamp::Timestamp; pub use crate::timestamp::Timestamp;
pub use crate::wheel::WheelCache; pub use crate::wheel::WheelCache;
use crate::wheel::WheelCacheKind; use crate::wheel::WheelCacheKind;
pub use archive::ArchiveId;
mod archive;
mod by_timestamp; mod by_timestamp;
#[cfg(feature = "clap")] #[cfg(feature = "clap")]
mod cli; mod cli;
@ -71,6 +73,12 @@ impl CacheEntry {
} }
} }
impl AsRef<Path> for CacheEntry {
fn as_ref(&self) -> &Path {
&self.0
}
}
/// A subdirectory within the cache. /// A subdirectory within the cache.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct CacheShard(PathBuf); pub struct CacheShard(PathBuf);
@ -167,6 +175,11 @@ impl Cache {
CacheEntry::new(self.bucket(cache_bucket).join(dir), file) CacheEntry::new(self.bucket(cache_bucket).join(dir), file)
} }
/// Return the path to an archive in the cache.
pub fn archive(&self, id: &ArchiveId) -> PathBuf {
self.bucket(CacheBucket::Archive).join(id)
}
/// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy. /// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
pub fn must_revalidate(&self, package: &PackageName) -> bool { pub fn must_revalidate(&self, package: &PackageName) -> bool {
match &self.refresh { match &self.refresh {
@ -208,18 +221,18 @@ impl Cache {
} }
} }
/// Persist a temporary directory to the artifact store. /// Persist a temporary directory to the artifact store, returning its unique ID.
pub async fn persist( pub async fn persist(
&self, &self,
temp_dir: impl AsRef<Path>, temp_dir: impl AsRef<Path>,
path: impl AsRef<Path>, path: impl AsRef<Path>,
) -> io::Result<PathBuf> { ) -> io::Result<ArchiveId> {
// Create a unique ID for the artifact. // Create a unique ID for the artifact.
// TODO(charlie): Support content-addressed persistence via SHAs. // TODO(charlie): Support content-addressed persistence via SHAs.
let id = nanoid::nanoid!(); let id = ArchiveId::new();
// Move the temporary directory into the directory store. // Move the temporary directory into the directory store.
let archive_entry = self.entry(CacheBucket::Archive, "", id); let archive_entry = self.entry(CacheBucket::Archive, "", &id);
fs_err::create_dir_all(archive_entry.dir())?; fs_err::create_dir_all(archive_entry.dir())?;
uv_fs::rename_with_retry(temp_dir.as_ref(), archive_entry.path()).await?; uv_fs::rename_with_retry(temp_dir.as_ref(), archive_entry.path()).await?;
@ -227,7 +240,7 @@ impl Cache {
fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?; fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
uv_fs::replace_symlink(archive_entry.path(), path.as_ref())?; uv_fs::replace_symlink(archive_entry.path(), path.as_ref())?;
Ok(archive_entry.into_path_buf()) Ok(id)
} }
/// Initialize a directory for use as a cache. /// Initialize a directory for use as a cache.
@ -594,12 +607,12 @@ pub enum CacheBucket {
impl CacheBucket { impl CacheBucket {
fn to_str(self) -> &'static str { fn to_str(self) -> &'static str {
match self { match self {
Self::BuiltWheels => "built-wheels-v2", Self::BuiltWheels => "built-wheels-v3",
Self::FlatIndex => "flat-index-v0", Self::FlatIndex => "flat-index-v0",
Self::Git => "git-v0", Self::Git => "git-v0",
Self::Interpreter => "interpreter-v0", Self::Interpreter => "interpreter-v0",
Self::Simple => "simple-v6", Self::Simple => "simple-v7",
Self::Wheels => "wheels-v0", Self::Wheels => "wheels-v1",
Self::Archive => "archive-v0", Self::Archive => "archive-v0",
} }
} }
@ -795,6 +808,12 @@ impl ArchiveTimestamp {
} }
} }
/// Return the modification timestamp for a file.
pub fn from_file(path: impl AsRef<Path>) -> Result<Self, io::Error> {
let metadata = fs_err::metadata(path.as_ref())?;
Ok(Self::Exact(Timestamp::from_metadata(&metadata)))
}
/// Return the modification timestamp for an archive. /// Return the modification timestamp for an archive.
pub fn timestamp(&self) -> Timestamp { pub fn timestamp(&self) -> Timestamp {
match self { match self {

17
crates/uv-client/Cargo.toml
View File

@ -11,13 +11,14 @@ install-wheel-rs = { workspace = true }
pep440_rs = { workspace = true } pep440_rs = { workspace = true }
pep508_rs = { workspace = true } pep508_rs = { workspace = true }
platform-tags = { workspace = true } platform-tags = { workspace = true }
pypi-types = { workspace = true }
uv-auth = { workspace = true } uv-auth = { workspace = true }
uv-cache = { workspace = true } uv-cache = { workspace = true }
uv-configuration = { workspace = true }
uv-fs = { workspace = true, features = ["tokio"] } uv-fs = { workspace = true, features = ["tokio"] }
uv-normalize = { workspace = true } uv-normalize = { workspace = true }
uv-version = { workspace = true } uv-version = { workspace = true }
uv-warnings = { workspace = true } uv-warnings = { workspace = true }
pypi-types = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
@ -33,11 +34,9 @@ reqwest-middleware = { workspace = true }
reqwest-retry = { workspace = true } reqwest-retry = { workspace = true }
rkyv = { workspace = true } rkyv = { workspace = true }
rmp-serde = { workspace = true } rmp-serde = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
sys-info = { workspace = true } sys-info = { workspace = true }
task-local-extensions = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
tl = { workspace = true } tl = { workspace = true }
@ -47,14 +46,10 @@ tracing = { workspace = true }
url = { workspace = true } url = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
# These must be kept in-sync with those used by `reqwest`.
rustls = { version = "0.21.10" }
rustls-native-certs = { version = "0.6.3" }
webpki-roots = { version = "0.25.4" }
[dev-dependencies] [dev-dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
hyper = { version = "0.14.28", features = ["server", "http1"] } http-body-util = { version = "0.1.0" }
insta = { version = "1.36.1" } hyper = { version = "1.2.0", features = ["server", "http1"] }
os_info = { version = "3.7.0", default-features = false } hyper-util = { version = "0.1.3", features = ["tokio"] }
insta = { version = "1.36.1" , features = ["filters", "json", "redactions"] }
tokio = { workspace = true, features = ["fs", "macros"] } tokio = { workspace = true, features = ["fs", "macros"] }

22
crates/uv-client/src/base_client.rs
View File

@ -16,10 +16,9 @@ use uv_warnings::warn_user_once;
use crate::linehaul::LineHaul; use crate::linehaul::LineHaul;
use crate::middleware::OfflineMiddleware; use crate::middleware::OfflineMiddleware;
use crate::tls::Roots; use crate::Connectivity;
use crate::{tls, Connectivity};
/// A builder for an [`RegistryClient`]. /// A builder for an [`BaseClient`].
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct BaseClientBuilder<'a> { pub struct BaseClientBuilder<'a> {
keyring_provider: KeyringProvider, keyring_provider: KeyringProvider,
@ -140,19 +139,20 @@ impl<'a> BaseClientBuilder<'a> {
} }
path_exists path_exists
}); });
// Load the TLS configuration.
let tls = tls::load(if self.native_tls || ssl_cert_file_exists {
Roots::Native
} else {
Roots::Webpki
})
.expect("Failed to load TLS configuration.");
// Configure the builder.
let client_core = ClientBuilder::new() let client_core = ClientBuilder::new()
.user_agent(user_agent_string) .user_agent(user_agent_string)
.pool_max_idle_per_host(20) .pool_max_idle_per_host(20)
.timeout(std::time::Duration::from_secs(timeout)) .timeout(std::time::Duration::from_secs(timeout))
.use_preconfigured_tls(tls); .tls_built_in_root_certs(false);
// Configure TLS.
let client_core = if self.native_tls || ssl_cert_file_exists {
client_core.tls_built_in_native_certs(true)
} else {
client_core.tls_built_in_webpki_certs(true)
};
client_core.build().expect("Failed to build HTTP client.") client_core.build().expect("Failed to build HTTP client.")
}); });

28
crates/uv-client/src/cached_client.rs
View File

@ -299,6 +299,34 @@ impl CachedClient {
} }
} }
/// Make a request without checking whether the cache is fresh.
pub async fn skip_cache<
Payload: Serialize + DeserializeOwned + Send + 'static,
CallBackError,
Callback,
CallbackReturn,
>(
&self,
req: Request,
cache_entry: &CacheEntry,
response_callback: Callback,
) -> Result<Payload, CachedClientError<CallBackError>>
where
Callback: FnOnce(Response) -> CallbackReturn + Send,
CallbackReturn: Future<Output = Result<Payload, CallBackError>> + Send,
{
let (response, cache_policy) = self.fresh_request(req).await?;
let payload = self
.run_response_callback(cache_entry, cache_policy, response, move |resp| async {
let payload = response_callback(resp).await?;
Ok(SerdeCacheable { inner: payload })
})
.await?;
Ok(payload)
}
async fn resend_and_heal_cache<Payload: Cacheable, CallBackError, Callback, CallbackReturn>( async fn resend_and_heal_cache<Payload: Cacheable, CallBackError, Callback, CallbackReturn>(
&self, &self,
req: Request, req: Request,

8
crates/uv-client/src/error.rs
View File

@ -132,7 +132,7 @@ pub enum ErrorKind {
/// Dist-info error /// Dist-info error
#[error(transparent)] #[error(transparent)]
InstallWheel(#[from] install_wheel_rs::Error), DistInfo(#[from] install_wheel_rs::Error),
#[error("{0} isn't available locally, but making network requests to registries was banned.")] #[error("{0} isn't available locally, but making network requests to registries was banned.")]
NoIndex(String), NoIndex(String),
@ -146,7 +146,11 @@ pub enum ErrorKind {
/// The metadata file could not be parsed. /// The metadata file could not be parsed.
#[error("Couldn't parse metadata of {0} from {1}")] #[error("Couldn't parse metadata of {0} from {1}")]
MetadataParseError(WheelFilename, String, #[source] Box<pypi_types::Error>), MetadataParseError(
WheelFilename,
String,
#[source] Box<pypi_types::MetadataError>,
),
/// The metadata file was not found in the wheel. /// The metadata file was not found in the wheel.
#[error("Metadata file `{0}` was not found in {1}")] #[error("Metadata file `{0}` was not found in {1}")]

153
crates/uv-client/src/flat_index.rs
View File

@ -1,24 +1,14 @@
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;
use std::path::PathBuf; use std::path::PathBuf;
use futures::{FutureExt, StreamExt}; use futures::{FutureExt, StreamExt};
use reqwest::Response; use reqwest::Response;
use rustc_hash::FxHashMap; use tracing::{debug, info_span, warn, Instrument};
use tracing::{debug, info_span, instrument, warn, Instrument};
use url::Url; use url::Url;
use distribution_filename::DistFilename; use distribution_filename::DistFilename;
use distribution_types::{ use distribution_types::{File, FileLocation, FlatIndexLocation, IndexUrl};
BuiltDist, Dist, File, FileLocation, FlatIndexLocation, IndexUrl, PrioritizedDist,
RegistryBuiltDist, RegistrySourceDist, SourceDist, SourceDistCompatibility,
};
use pep440_rs::Version;
use pep508_rs::VerbatimUrl; use pep508_rs::VerbatimUrl;
use platform_tags::Tags;
use pypi_types::Hashes;
use uv_cache::{Cache, CacheBucket}; use uv_cache::{Cache, CacheBucket};
use uv_normalize::PackageName;
use crate::cached_client::{CacheControl, CachedClientError}; use crate::cached_client::{CacheControl, CachedClientError};
use crate::html::SimpleHtml; use crate::html::SimpleHtml;
@ -36,10 +26,10 @@ pub enum FlatIndexError {
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct FlatIndexEntries { pub struct FlatIndexEntries {
/// The list of `--find-links` entries. /// The list of `--find-links` entries.
entries: Vec<(DistFilename, File, IndexUrl)>, pub entries: Vec<(DistFilename, File, IndexUrl)>,
/// Whether any `--find-links` entries could not be resolved due to a lack of network /// Whether any `--find-links` entries could not be resolved due to a lack of network
/// connectivity. /// connectivity.
offline: bool, pub offline: bool,
} }
impl FlatIndexEntries { impl FlatIndexEntries {
@ -215,7 +205,7 @@ impl<'a> FlatIndexClient<'a> {
fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> { fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
// Absolute paths are required for the URL conversion. // Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?; let path = fs_err::canonicalize(path)?;
let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path)); let index_url = IndexUrl::Path(VerbatimUrl::from_path(&path));
let mut dists = Vec::new(); let mut dists = Vec::new();
for entry in fs_err::read_dir(path)? { for entry in fs_err::read_dir(path)? {
@ -234,9 +224,9 @@ impl<'a> FlatIndexClient<'a> {
}; };
let file = File { let file = File {
dist_info_metadata: None, dist_info_metadata: false,
filename: filename.to_string(), filename: filename.to_string(),
hashes: Hashes::default(), hashes: Vec::new(),
requires_python: None, requires_python: None,
size: None, size: None,
upload_time_utc_ms: None, upload_time_utc_ms: None,
@ -256,132 +246,3 @@ impl<'a> FlatIndexClient<'a> {
Ok(FlatIndexEntries::from_entries(dists)) Ok(FlatIndexEntries::from_entries(dists))
} }
} }
/// A set of [`PrioritizedDist`] from a `--find-links` entry, indexed by [`PackageName`]
/// and [`Version`].
#[derive(Debug, Clone, Default)]
pub struct FlatIndex {
/// The list of [`FlatDistributions`] from the `--find-links` entries, indexed by package name.
index: FxHashMap<PackageName, FlatDistributions>,
/// Whether any `--find-links` entries could not be resolved due to a lack of network
/// connectivity.
offline: bool,
}
impl FlatIndex {
/// Collect all files from a `--find-links` target into a [`FlatIndex`].
#[instrument(skip_all)]
pub fn from_entries(entries: FlatIndexEntries, tags: &Tags) -> Self {
// Collect compatible distributions.
let mut index = FxHashMap::default();
for (filename, file, url) in entries.entries {
let distributions = index.entry(filename.name().clone()).or_default();
Self::add_file(distributions, file, filename, tags, url);
}
// Collect offline entries.
let offline = entries.offline;
Self { index, offline }
}
fn add_file(
distributions: &mut FlatDistributions,
file: File,
filename: DistFilename,
tags: &Tags,
index: IndexUrl,
) {
// No `requires-python` here: for source distributions, we don't have that information;
// for wheels, we read it lazily only when selected.
match filename {
DistFilename::WheelFilename(filename) => {
let compatibility = filename.compatibility(tags);
let version = filename.version.clone();
let dist = Dist::Built(BuiltDist::Registry(RegistryBuiltDist {
filename,
file: Box::new(file),
index,
}));
match distributions.0.entry(version) {
Entry::Occupied(mut entry) => {
entry
.get_mut()
.insert_built(dist, None, compatibility.into());
}
Entry::Vacant(entry) => {
entry.insert(PrioritizedDist::from_built(
dist,
None,
compatibility.into(),
));
}
}
}
DistFilename::SourceDistFilename(filename) => {
let dist = Dist::Source(SourceDist::Registry(RegistrySourceDist {
filename: filename.clone(),
file: Box::new(file),
index,
}));
match distributions.0.entry(filename.version) {
Entry::Occupied(mut entry) => {
entry.get_mut().insert_source(
dist,
None,
SourceDistCompatibility::Compatible,
);
}
Entry::Vacant(entry) => {
entry.insert(PrioritizedDist::from_source(
dist,
None,
SourceDistCompatibility::Compatible,
));
}
}
}
}
}
/// Get the [`FlatDistributions`] for the given package name.
pub fn get(&self, package_name: &PackageName) -> Option<&FlatDistributions> {
self.index.get(package_name)
}
/// Returns `true` if there are any offline `--find-links` entries.
pub fn offline(&self) -> bool {
self.offline
}
}
/// A set of [`PrioritizedDist`] from a `--find-links` entry for a single package, indexed
/// by [`Version`].
#[derive(Debug, Clone, Default)]
pub struct FlatDistributions(BTreeMap<Version, PrioritizedDist>);
impl FlatDistributions {
pub fn iter(&self) -> impl Iterator<Item = (&Version, &PrioritizedDist)> {
self.0.iter()
}
pub fn remove(&mut self, version: &Version) -> Option<PrioritizedDist> {
self.0.remove(version)
}
}
impl IntoIterator for FlatDistributions {
type Item = (Version, PrioritizedDist);
type IntoIter = std::collections::btree_map::IntoIter<Version, PrioritizedDist>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}
impl From<FlatDistributions> for BTreeMap<Version, PrioritizedDist> {
fn from(distributions: FlatDistributions) -> Self {
distributions.0
}
}

57
crates/uv-client/src/html.rs
View File

@ -164,6 +164,9 @@ impl SimpleHtml {
.last() .last()
.ok_or_else(|| Error::MissingFilename(href.to_string()))?; .ok_or_else(|| Error::MissingFilename(href.to_string()))?;
// Strip any query string from the filename.
let filename = filename.split('?').next().unwrap_or(filename);
// Unquote the filename. // Unquote the filename.
let filename = urlencoding::decode(filename) let filename = urlencoding::decode(filename)
.map_err(|_| Error::UnsupportedFilename(filename.to_string()))?; .map_err(|_| Error::UnsupportedFilename(filename.to_string()))?;
@ -681,6 +684,60 @@ mod tests {
"###); "###);
} }
#[test]
fn parse_query_string() {
let text = r#"
<!DOCTYPE html>
<html>
<body>
<h1>Links for jinja2</h1>
<a href="/whl/Jinja2-3.1.2-py3-none-any.whl?project=legacy">Jinja2-3.1.2-py3-none-any.whl</a><br/>
</body>
</html>
<!--TIMESTAMP 1703347410-->
"#;
let base = Url::parse("https://download.pytorch.org/whl/jinja2/").unwrap();
let result = SimpleHtml::parse(text, &base).unwrap();
insta::assert_debug_snapshot!(result, @r###"
SimpleHtml {
base: BaseUrl(
Url {
scheme: "https",
cannot_be_a_base: false,
username: "",
password: None,
host: Some(
Domain(
"download.pytorch.org",
),
),
port: None,
path: "/whl/jinja2/",
query: None,
fragment: None,
},
),
files: [
File {
dist_info_metadata: None,
filename: "Jinja2-3.1.2-py3-none-any.whl",
hashes: Hashes {
md5: None,
sha256: None,
sha384: None,
sha512: None,
},
requires_python: None,
size: None,
upload_time: None,
url: "/whl/Jinja2-3.1.2-py3-none-any.whl?project=legacy",
yanked: None,
},
],
}
"###);
}
#[test] #[test]
fn parse_missing_hash_value() { fn parse_missing_hash_value() {
let text = r#" let text = r#"

3
crates/uv-client/src/lib.rs
View File

@ -1,7 +1,7 @@
pub use base_client::{BaseClient, BaseClientBuilder}; pub use base_client::{BaseClient, BaseClientBuilder};
pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy}; pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy};
pub use error::{BetterReqwestError, Error, ErrorKind}; pub use error::{BetterReqwestError, Error, ErrorKind};
pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError}; pub use flat_index::{FlatIndexClient, FlatIndexEntries, FlatIndexError};
pub use linehaul::LineHaul; pub use linehaul::LineHaul;
pub use registry_client::{ pub use registry_client::{
Connectivity, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum, Connectivity, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum,
@ -20,4 +20,3 @@ mod middleware;
mod registry_client; mod registry_client;
mod remote_metadata; mod remote_metadata;
mod rkyvutil; mod rkyvutil;
mod tls;

2
crates/uv-client/src/middleware.rs
View File

@ -1,8 +1,8 @@
use http::Extensions;
use std::fmt::Debug; use std::fmt::Debug;
use reqwest::{Request, Response}; use reqwest::{Request, Response};
use reqwest_middleware::{Middleware, Next}; use reqwest_middleware::{Middleware, Next};
use task_local_extensions::Extensions;
use url::Url; use url::Url;
/// A custom error type for the offline middleware. /// A custom error type for the offline middleware.

53
crates/uv-client/src/registry_client.rs
View File

@ -9,7 +9,7 @@ use http::HeaderMap;
use reqwest::{Client, Response, StatusCode}; use reqwest::{Client, Response, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
use tokio_util::compat::FuturesAsyncReadCompatExt; use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
use tracing::{info_span, instrument, trace, warn, Instrument}; use tracing::{info_span, instrument, trace, warn, Instrument};
use url::Url; use url::Url;
@ -22,6 +22,7 @@ use platform_tags::Platform;
use pypi_types::{Metadata23, SimpleJson}; use pypi_types::{Metadata23, SimpleJson};
use uv_auth::KeyringProvider; use uv_auth::KeyringProvider;
use uv_cache::{Cache, CacheBucket, WheelCache}; use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_configuration::IndexStrategy;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use crate::base_client::{BaseClient, BaseClientBuilder}; use crate::base_client::{BaseClient, BaseClientBuilder};
@ -35,6 +36,7 @@ use crate::{CachedClient, CachedClientError, Error, ErrorKind};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct RegistryClientBuilder<'a> { pub struct RegistryClientBuilder<'a> {
index_urls: IndexUrls, index_urls: IndexUrls,
index_strategy: IndexStrategy,
keyring_provider: KeyringProvider, keyring_provider: KeyringProvider,
native_tls: bool, native_tls: bool,
retries: u32, retries: u32,
@ -49,6 +51,7 @@ impl RegistryClientBuilder<'_> {
pub fn new(cache: Cache) -> Self { pub fn new(cache: Cache) -> Self {
Self { Self {
index_urls: IndexUrls::default(), index_urls: IndexUrls::default(),
index_strategy: IndexStrategy::default(),
keyring_provider: KeyringProvider::default(), keyring_provider: KeyringProvider::default(),
native_tls: false, native_tls: false,
cache, cache,
@ -68,6 +71,12 @@ impl<'a> RegistryClientBuilder<'a> {
self self
} }
#[must_use]
pub fn index_strategy(mut self, index_strategy: IndexStrategy) -> Self {
self.index_strategy = index_strategy;
self
}
#[must_use] #[must_use]
pub fn keyring_provider(mut self, keyring_provider: KeyringProvider) -> Self { pub fn keyring_provider(mut self, keyring_provider: KeyringProvider) -> Self {
self.keyring_provider = keyring_provider; self.keyring_provider = keyring_provider;
@ -147,6 +156,7 @@ impl<'a> RegistryClientBuilder<'a> {
RegistryClient { RegistryClient {
index_urls: self.index_urls, index_urls: self.index_urls,
index_strategy: self.index_strategy,
cache: self.cache, cache: self.cache,
connectivity, connectivity,
client, client,
@ -160,6 +170,8 @@ impl<'a> RegistryClientBuilder<'a> {
pub struct RegistryClient { pub struct RegistryClient {
/// The index URLs to use for fetching packages. /// The index URLs to use for fetching packages.
index_urls: IndexUrls, index_urls: IndexUrls,
/// The strategy to use when fetching across multiple indexes.
index_strategy: IndexStrategy,
/// The underlying HTTP client. /// The underlying HTTP client.
client: CachedClient, client: CachedClient,
/// Used for the remote wheel METADATA cache. /// Used for the remote wheel METADATA cache.
@ -206,17 +218,23 @@ impl RegistryClient {
pub async fn simple( pub async fn simple(
&self, &self,
package_name: &PackageName, package_name: &PackageName,
) -> Result<(IndexUrl, OwnedArchive<SimpleMetadata>), Error> { ) -> Result<Vec<(IndexUrl, OwnedArchive<SimpleMetadata>)>, Error> {
let mut it = self.index_urls.indexes().peekable(); let mut it = self.index_urls.indexes().peekable();
if it.peek().is_none() { if it.peek().is_none() {
return Err(ErrorKind::NoIndex(package_name.as_ref().to_string()).into()); return Err(ErrorKind::NoIndex(package_name.as_ref().to_string()).into());
} }
let mut results = Vec::new();
for index in it { for index in it {
let result = self.simple_single_index(package_name, index).await?; match self.simple_single_index(package_name, index).await? {
Ok(metadata) => {
results.push((index.clone(), metadata));
return match result { // If we're only using the first match, we can stop here.
Ok(metadata) => Ok((index.clone(), metadata)), if self.index_strategy == IndexStrategy::FirstMatch {
break;
}
}
Err(CachedClientError::Client(err)) => match err.into_kind() { Err(CachedClientError::Client(err)) => match err.into_kind() {
ErrorKind::Offline(_) => continue, ErrorKind::Offline(_) => continue,
ErrorKind::ReqwestError(err) => { ErrorKind::ReqwestError(err) => {
@ -225,20 +243,24 @@ impl RegistryClient {
{ {
continue; continue;
} }
Err(ErrorKind::from(err).into()) return Err(ErrorKind::from(err).into());
} }
other => Err(other.into()), other => return Err(other.into()),
}, },
Err(CachedClientError::Callback(err)) => Err(err), Err(CachedClientError::Callback(err)) => return Err(err),
}; };
} }
match self.connectivity { if results.is_empty() {
return match self.connectivity {
Connectivity::Online => { Connectivity::Online => {
Err(ErrorKind::PackageNotFound(package_name.to_string()).into()) Err(ErrorKind::PackageNotFound(package_name.to_string()).into())
} }
Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()), Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()),
};
} }
Ok(results)
} }
async fn simple_single_index( async fn simple_single_index(
@ -263,6 +285,7 @@ impl RegistryClient {
Path::new(&match index { Path::new(&match index {
IndexUrl::Pypi(_) => "pypi".to_string(), IndexUrl::Pypi(_) => "pypi".to_string(),
IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)), IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
IndexUrl::Path(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
}), }),
format!("{package_name}.rkyv"), format!("{package_name}.rkyv"),
); );
@ -402,11 +425,7 @@ impl RegistryClient {
) -> Result<Metadata23, Error> { ) -> Result<Metadata23, Error> {
// If the metadata file is available at its own url (PEP 658), download it from there. // If the metadata file is available at its own url (PEP 658), download it from there.
let filename = WheelFilename::from_str(&file.filename).map_err(ErrorKind::WheelFilename)?; let filename = WheelFilename::from_str(&file.filename).map_err(ErrorKind::WheelFilename)?;
if file if file.dist_info_metadata {
.dist_info_metadata
.as_ref()
.is_some_and(pypi_types::DistInfoMetadata::is_available)
{
let mut url = url.clone(); let mut url = url.clone();
url.set_path(&format!("{}.metadata", url.path())); url.set_path(&format!("{}.metadata", url.path()));
@ -596,7 +615,8 @@ async fn read_metadata_async_seek(
debug_source: String, debug_source: String,
reader: impl tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin, reader: impl tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin,
) -> Result<Metadata23, Error> { ) -> Result<Metadata23, Error> {
let mut zip_reader = async_zip::tokio::read::seek::ZipFileReader::with_tokio(reader) let reader = futures::io::BufReader::new(reader.compat());
let mut zip_reader = async_zip::base::read::seek::ZipFileReader::new(reader)
.await .await
.map_err(|err| ErrorKind::Zip(filename.clone(), err))?; .map_err(|err| ErrorKind::Zip(filename.clone(), err))?;
@ -609,7 +629,7 @@ async fn read_metadata_async_seek(
.enumerate() .enumerate()
.filter_map(|(index, entry)| Some((index, entry.filename().as_str().ok()?))), .filter_map(|(index, entry)| Some((index, entry.filename().as_str().ok()?))),
) )
.map_err(ErrorKind::InstallWheel)?; .map_err(ErrorKind::DistInfo)?;
// Read the contents of the `METADATA` file. // Read the contents of the `METADATA` file.
let mut contents = Vec::new(); let mut contents = Vec::new();
@ -633,6 +653,7 @@ async fn read_metadata_async_stream<R: futures::AsyncRead + Unpin>(
debug_source: String, debug_source: String,
reader: R, reader: R,
) -> Result<Metadata23, Error> { ) -> Result<Metadata23, Error> {
let reader = futures::io::BufReader::with_capacity(128 * 1024, reader);
let mut zip = async_zip::base::read::stream::ZipFileReader::new(reader); let mut zip = async_zip::base::read::stream::ZipFileReader::new(reader);
while let Some(mut entry) = zip while let Some(mut entry) = zip

8
crates/uv-client/src/remote_metadata.rs
View File

@ -1,5 +1,5 @@
use async_http_range_reader::AsyncHttpRangeReader; use async_http_range_reader::AsyncHttpRangeReader;
use async_zip::tokio::read::seek::ZipFileReader; use futures::io::BufReader;
use tokio_util::compat::TokioAsyncReadCompatExt; use tokio_util::compat::TokioAsyncReadCompatExt;
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
@ -61,7 +61,8 @@ pub(crate) async fn wheel_metadata_from_remote_zip(
.await; .await;
// Construct a zip reader to uses the stream. // Construct a zip reader to uses the stream.
let mut reader = ZipFileReader::new(reader.compat()) let buf = BufReader::new(reader.compat());
let mut reader = async_zip::base::read::seek::ZipFileReader::new(buf)
.await .await
.map_err(|err| ErrorKind::Zip(filename.clone(), err))?; .map_err(|err| ErrorKind::Zip(filename.clone(), err))?;
@ -74,7 +75,7 @@ pub(crate) async fn wheel_metadata_from_remote_zip(
.enumerate() .enumerate()
.filter_map(|(idx, e)| Some(((idx, e), e.filename().as_str().ok()?))), .filter_map(|(idx, e)| Some(((idx, e), e.filename().as_str().ok()?))),
) )
.map_err(ErrorKind::InstallWheel)?; .map_err(ErrorKind::DistInfo)?;
let offset = metadata_entry.header_offset(); let offset = metadata_entry.header_offset();
let size = metadata_entry.compressed_size() let size = metadata_entry.compressed_size()
@ -90,6 +91,7 @@ pub(crate) async fn wheel_metadata_from_remote_zip(
reader reader
.inner_mut() .inner_mut()
.get_mut() .get_mut()
.get_mut()
.prefetch(offset..offset + size) .prefetch(offset..offset + size)
.await; .await;

102
crates/uv-client/src/tls.rs
View File

@ -1,102 +0,0 @@
use rustls::ClientConfig;
use tracing::warn;
#[derive(thiserror::Error, Debug)]
pub(crate) enum TlsError {
#[error(transparent)]
Rustls(#[from] rustls::Error),
#[error("zero valid certificates found in native root store")]
ZeroCertificates,
#[error("failed to load native root certificates")]
NativeCertificates(#[source] std::io::Error),
}
#[derive(Debug, Clone, Copy)]
pub(crate) enum Roots {
/// Use reqwest's `rustls-tls-webpki-roots` behavior for loading root certificates.
Webpki,
/// Use reqwest's `rustls-tls-native-roots` behavior for loading root certificates.
Native,
}
/// Initialize a TLS configuration for the client.
///
/// This is equivalent to the TLS initialization `reqwest` when `rustls-tls` is enabled,
/// with two notable changes:
///
/// 1. It enables _either_ the `webpki-roots` or the `native-certs` feature, but not both.
/// 2. It assumes the following builder settings (which match the defaults):
/// - `root_certs: vec![]`
/// - `min_tls_version: None`
/// - `max_tls_version: None`
/// - `identity: None`
/// - `certs_verification: false`
/// - `tls_sni: true`
/// - `http_version_pref: HttpVersionPref::All`
///
/// See: <https://github.com/seanmonstar/reqwest/blob/e3192638518d577759dd89da489175b8f992b12f/src/async_impl/client.rs#L498>
pub(crate) fn load(roots: Roots) -> Result<ClientConfig, TlsError> {
// Set root certificates.
let mut root_cert_store = rustls::RootCertStore::empty();
match roots {
Roots::Webpki => {
// Use `rustls-tls-webpki-roots`
use rustls::OwnedTrustAnchor;
let trust_anchors = webpki_roots::TLS_SERVER_ROOTS.iter().map(|trust_anchor| {
OwnedTrustAnchor::from_subject_spki_name_constraints(
trust_anchor.subject,
trust_anchor.spki,
trust_anchor.name_constraints,
)
});
root_cert_store.add_trust_anchors(trust_anchors);
}
Roots::Native => {
// Use: `rustls-tls-native-roots`
let mut valid_count = 0;
let mut invalid_count = 0;
for cert in
rustls_native_certs::load_native_certs().map_err(TlsError::NativeCertificates)?
{
let cert = rustls::Certificate(cert.0);
// Continue on parsing errors, as native stores often include ancient or syntactically
// invalid certificates, like root certificates without any X509 extensions.
// Inspiration: https://github.com/rustls/rustls/blob/633bf4ba9d9521a95f68766d04c22e2b01e68318/rustls/src/anchors.rs#L105-L112
match root_cert_store.add(&cert) {
Ok(_) => valid_count += 1,
Err(err) => {
invalid_count += 1;
warn!(
"rustls failed to parse DER certificate {:?} {:?}",
&err, &cert
);
}
}
}
if valid_count == 0 && invalid_count > 0 {
return Err(TlsError::ZeroCertificates);
}
}
}
// Build TLS config
let config_builder = ClientConfig::builder()
.with_safe_default_cipher_suites()
.with_safe_default_kx_groups()
.with_protocol_versions(rustls::ALL_VERSIONS)?
.with_root_certificates(root_cert_store);
// Finalize TLS config
let mut tls = config_builder.with_no_client_auth();
// Enable SNI
tls.enable_sni = true;
// ALPN protocol
tls.alpn_protocols = vec!["h2".into(), "http/1.1".into()];
Ok(tls)
}

22
crates/uv-client/tests/netrc_auth.rs
View File

@ -3,10 +3,13 @@ use std::io::Write;
use anyhow::Result; use anyhow::Result;
use futures::future; use futures::future;
use hyper::header::AUTHORIZATION; use http::header::AUTHORIZATION;
use hyper::server::conn::Http; use http_body_util::Full;
use hyper::body::Bytes;
use hyper::server::conn::http1;
use hyper::service::service_fn; use hyper::service::service_fn;
use hyper::{Body, Request, Response}; use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
use tokio::net::TcpListener; use tokio::net::TcpListener;
@ -21,7 +24,7 @@ async fn test_client_with_netrc_credentials() -> Result<()> {
// Spawn the server loop in a background task // Spawn the server loop in a background task
tokio::spawn(async move { tokio::spawn(async move {
let svc = service_fn(move |req: Request<Body>| { let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
// Get User Agent Header and send it back in the response // Get User Agent Header and send it back in the response
let auth = req let auth = req
.headers() .headers()
@ -29,17 +32,20 @@ async fn test_client_with_netrc_credentials() -> Result<()> {
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.map(|s| s.to_string()) .map(|s| s.to_string())
.unwrap_or_default(); // Empty Default .unwrap_or_default(); // Empty Default
future::ok::<_, hyper::Error>(Response::new(Body::from(auth))) future::ok::<_, hyper::Error>(Response::new(Full::new(Bytes::from(auth))))
}); });
// Start Hyper Server // Start Server (not wrapped in loop {} since we want a single response server)
// If you want server to accept multiple connections, wrap it in loop {}
let (socket, _) = listener.accept().await.unwrap(); let (socket, _) = listener.accept().await.unwrap();
Http::new() let socket = TokioIo::new(socket);
.http1_keep_alive(false) tokio::task::spawn(async move {
http1::Builder::new()
.serve_connection(socket, svc) .serve_connection(socket, svc)
.with_upgrades() .with_upgrades()
.await .await
.expect("Server Started"); .expect("Server Started");
}); });
});
// Create a netrc file // Create a netrc file
let mut netrc_file = NamedTempFile::new()?; let mut netrc_file = NamedTempFile::new()?;

170
crates/uv-client/tests/user_agent_version.rs
View File

@ -1,9 +1,13 @@
use anyhow::Result; use anyhow::Result;
use futures::future; use futures::future;
use http_body_util::Full;
use hyper::body::Bytes;
use hyper::header::USER_AGENT; use hyper::header::USER_AGENT;
use hyper::server::conn::Http; use hyper::server::conn::http1;
use hyper::service::service_fn; use hyper::service::service_fn;
use hyper::{Body, Request, Response}; use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use insta::{assert_json_snapshot, assert_snapshot, with_settings};
use pep508_rs::{MarkerEnvironment, StringVersion}; use pep508_rs::{MarkerEnvironment, StringVersion};
use platform_tags::{Arch, Os, Platform}; use platform_tags::{Arch, Os, Platform};
use tokio::net::TcpListener; use tokio::net::TcpListener;
@ -19,8 +23,8 @@ async fn test_user_agent_has_version() -> Result<()> {
let addr = listener.local_addr()?; let addr = listener.local_addr()?;
// Spawn the server loop in a background task // Spawn the server loop in a background task
tokio::spawn(async move { let server_task = tokio::spawn(async move {
let svc = service_fn(move |req: Request<Body>| { let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
// Get User Agent Header and send it back in the response // Get User Agent Header and send it back in the response
let user_agent = req let user_agent = req
.headers() .headers()
@ -28,17 +32,20 @@ async fn test_user_agent_has_version() -> Result<()> {
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.map(|s| s.to_string()) .map(|s| s.to_string())
.unwrap_or_default(); // Empty Default .unwrap_or_default(); // Empty Default
future::ok::<_, hyper::Error>(Response::new(Body::from(user_agent))) future::ok::<_, hyper::Error>(Response::new(Full::new(Bytes::from(user_agent))))
}); });
// Start Hyper Server // Start Server (not wrapped in loop {} since we want a single response server)
// If you want server to accept multiple connections, wrap it in loop {}
let (socket, _) = listener.accept().await.unwrap(); let (socket, _) = listener.accept().await.unwrap();
Http::new() let socket = TokioIo::new(socket);
.http1_keep_alive(false) tokio::task::spawn(async move {
http1::Builder::new()
.serve_connection(socket, svc) .serve_connection(socket, svc)
.with_upgrades() .with_upgrades()
.await .await
.expect("Server Started"); .expect("Server Started");
}); });
});
// Initialize uv-client // Initialize uv-client
let cache = Cache::temp()?; let cache = Cache::temp()?;
@ -46,7 +53,8 @@ async fn test_user_agent_has_version() -> Result<()> {
// Send request to our dummy server // Send request to our dummy server
let res = client let res = client
.uncached_client() .cached_client()
.uncached()
.get(format!("http://{addr}")) .get(format!("http://{addr}"))
.send() .send()
.await?; .await?;
@ -60,6 +68,9 @@ async fn test_user_agent_has_version() -> Result<()> {
// Verify body matches regex // Verify body matches regex
assert_eq!(body, format!("uv/{}", version())); assert_eq!(body, format!("uv/{}", version()));
// Wait for the server task to complete, to be a good citizen.
server_task.await?;
Ok(()) Ok(())
} }
@ -70,8 +81,8 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
let addr = listener.local_addr()?; let addr = listener.local_addr()?;
// Spawn the server loop in a background task // Spawn the server loop in a background task
tokio::spawn(async move { let server_task = tokio::spawn(async move {
let svc = service_fn(move |req: Request<Body>| { let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
// Get User Agent Header and send it back in the response // Get User Agent Header and send it back in the response
let user_agent = req let user_agent = req
.headers() .headers()
@ -79,17 +90,20 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.map(|s| s.to_string()) .map(|s| s.to_string())
.unwrap_or_default(); // Empty Default .unwrap_or_default(); // Empty Default
future::ok::<_, hyper::Error>(Response::new(Body::from(user_agent))) future::ok::<_, hyper::Error>(Response::new(Full::new(Bytes::from(user_agent))))
}); });
// Start Hyper Server // Start Server (not wrapped in loop {} since we want a single response server)
// If you want server to accept multiple connections, wrap it in loop {}
let (socket, _) = listener.accept().await.unwrap(); let (socket, _) = listener.accept().await.unwrap();
Http::new() let socket = TokioIo::new(socket);
.http1_keep_alive(false) tokio::task::spawn(async move {
http1::Builder::new()
.serve_connection(socket, svc) .serve_connection(socket, svc)
.with_upgrades() .with_upgrades()
.await .await
.expect("Server Started"); .expect("Server Started");
}); });
});
// Add some representative markers for an Ubuntu CI runner // Add some representative markers for an Ubuntu CI runner
let markers = MarkerEnvironment { let markers = MarkerEnvironment {
@ -142,7 +156,8 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
// Send request to our dummy server // Send request to our dummy server
let res = client let res = client
.uncached_client() .cached_client()
.uncached()
.get(format!("http://{addr}")) .get(format!("http://{addr}"))
.send() .send()
.await?; .await?;
@ -153,6 +168,9 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
// Check User Agent // Check User Agent
let body = res.text().await?; let body = res.text().await?;
// Wait for the server task to complete, to be a good citizen.
server_task.await?;
// Unpack User-Agent with linehaul // Unpack User-Agent with linehaul
let (uv_version, uv_linehaul) = body let (uv_version, uv_linehaul) = body
.split_once(' ') .split_once(' ')
@ -161,62 +179,82 @@ async fn test_user_agent_has_linehaul() -> Result<()> {
// Deserializing Linehaul // Deserializing Linehaul
let linehaul: LineHaul = serde_json::from_str(uv_linehaul)?; let linehaul: LineHaul = serde_json::from_str(uv_linehaul)?;
// Assert linehaul user agent
let filters = vec![(version(), "[VERSION]")];
with_settings!({
filters => filters
}, {
// Assert uv version // Assert uv version
assert_eq!(uv_version, format!("uv/{}", version())); assert_snapshot!(uv_version, @"uv/[VERSION]");
// Assert linehaul json
// Assert linehaul assert_json_snapshot!(&linehaul, {
let installer_info = linehaul.installer.unwrap(); ".distro" => "[distro]",
let system_info = linehaul.system.unwrap(); ".ci" => "[ci]"
let impl_info = linehaul.implementation.unwrap(); }, @r###"
{
assert_eq!(installer_info.name.unwrap(), "uv".to_string()); "installer": {
assert_eq!(installer_info.version.unwrap(), version()); "name": "uv",
"version": "[VERSION]"
assert_eq!(system_info.name.unwrap(), markers.platform_system); },
assert_eq!(system_info.release.unwrap(), markers.platform_release); "python": "3.12.2",
"implementation": {
assert_eq!( "name": "CPython",
impl_info.name.unwrap(), "version": "3.12.2"
markers.platform_python_implementation },
); "distro": "[distro]",
assert_eq!( "system": {
impl_info.version.unwrap(), "name": "Linux",
markers.python_full_version.version.to_string() "release": "6.5.0-1016-azure"
); },
"cpu": "x86_64",
assert_eq!( "openssl_version": null,
linehaul.python.unwrap(), "setuptools_version": null,
markers.python_full_version.version.to_string() "rustc_version": null,
); "ci": "[ci]"
assert_eq!(linehaul.cpu.unwrap(), markers.platform_machine); }
"###);
assert_eq!(linehaul.openssl_version, None); });
assert_eq!(linehaul.setuptools_version, None);
assert_eq!(linehaul.rustc_version, None);
// Assert distro
if cfg!(windows) { if cfg!(windows) {
assert_eq!(linehaul.distro, None); assert_json_snapshot!(&linehaul.distro, @"null");
} else if cfg!(target_os = "linux") { } else if cfg!(target_os = "linux") {
// Using `os_info` to confirm our values are as expected in Linux assert_json_snapshot!(&linehaul.distro, {
let info = os_info::get(); ".id" => "[distro.id]",
let Some(distro_info) = linehaul.distro else { ".name" => "[distro.name]",
panic!("got no distro, but expected one in linehaul") ".version" => "[distro.version]"
}; // We mock the libc version already
assert_eq!(distro_info.id.as_deref(), info.codename()); }, @r###"
if let Some(ref name) = distro_info.name { {
assert_eq!(name, &info.os_type().to_string()); "name": "[distro.name]",
"version": "[distro.version]",
"id": "[distro.id]",
"libc": {
"lib": "glibc",
"version": "2.38"
} }
if let Some(ref version) = distro_info.version { }"###
assert_eq!(version, &info.version().to_string()); );
} // Check dynamic values
assert!(distro_info.libc.is_some()); let distro_info = linehaul
.distro
.expect("got no distro, but expected one in linehaul");
// Gather distribution info from /etc/os-release.
let release_info = sys_info::linux_os_release()
.expect("got no os release info, but expected one in linux");
assert_eq!(distro_info.id, release_info.version_codename);
assert_eq!(distro_info.name, release_info.name);
assert_eq!(distro_info.version, release_info.version_id);
} else if cfg!(target_os = "macos") { } else if cfg!(target_os = "macos") {
// We mock the macOS version // We mock the macOS distro
let distro_info = linehaul.distro.unwrap(); assert_json_snapshot!(&linehaul.distro, @r###"
assert_eq!(distro_info.id, None); {
assert_eq!(distro_info.name.unwrap(), "macOS"); "name": "macOS",
assert_eq!(distro_info.version, Some("14.4".to_string())); "version": "14.4",
assert_eq!(distro_info.libc, None); "id": null,
"libc": null
}"###
);
} }
Ok(()) Ok(())

28
crates/uv-configuration/Cargo.toml Normal file
View File

@ -0,0 +1,28 @@
[package]
name = "uv-configuration"
version = "0.0.1"
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
[lints]
workspace = true
[dependencies]
pep508_rs = { workspace = true }
uv-normalize = { workspace = true }
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
itertools = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
[features]
default = []
serde = ["dep:serde", "dep:serde_json"]

38
crates/uv-types/src/build_options.rs → crates/uv-configuration/src/build_options.rs
View File

@ -1,24 +1,9 @@
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use pep508_rs::PackageName; use pep508_rs::PackageName;
use uv_interpreter::PythonEnvironment;
use crate::{PackageNameSpecifier, PackageNameSpecifiers}; use crate::{PackageNameSpecifier, PackageNameSpecifiers};
/// Whether to enforce build isolation when building source distributions.
#[derive(Debug, Copy, Clone)]
pub enum BuildIsolation<'a> {
Isolated,
Shared(&'a PythonEnvironment),
}
impl<'a> BuildIsolation<'a> {
/// Returns `true` if build isolation is enforced.
pub fn is_isolated(&self) -> bool {
matches!(self, Self::Isolated)
}
}
/// The strategy to use when building source distributions that lack a `pyproject.toml`. /// The strategy to use when building source distributions that lack a `pyproject.toml`.
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
pub enum SetupPyStrategy { pub enum SetupPyStrategy {
@ -211,6 +196,29 @@ impl NoBuild {
} }
} }
#[derive(Debug, Default, Clone, Hash, Eq, PartialEq)]
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
pub enum IndexStrategy {
/// Only use results from the first index that returns a match for a given package name.
///
/// While this differs from pip's behavior, it's the default index strategy as it's the most
/// secure.
#[default]
FirstMatch,
/// Search for every package name across all indexes, exhausting the versions from the first
/// index before moving on to the next.
///
/// In this strategy, we look for every package across all indexes. When resolving, we attempt
/// to use versions from the indexes in order, such that we exhaust all available versions from
/// the first index before moving on to the next. Further, if a version is found to be
/// incompatible in the first index, we do not reconsider that version in subsequent indexes,
/// even if the secondary index might contain compatible versions (e.g., variants of the same
/// versions with different ABI tags or Python version constraints).
///
/// See: https://peps.python.org/pep-0708/
UnsafeAnyMatch,
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::str::FromStr; use std::str::FromStr;

1
crates/uv-types/src/config_settings.rs → crates/uv-configuration/src/config_settings.rs
View File

@ -40,6 +40,7 @@ enum ConfigSettingValue {
/// ///
/// See: <https://peps.python.org/pep-0517/#config-settings> /// See: <https://peps.python.org/pep-0517/#config-settings>
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
#[cfg_attr(not(feature = "serde"), allow(dead_code))]
pub struct ConfigSettings(BTreeMap<String, ConfigSettingValue>); pub struct ConfigSettings(BTreeMap<String, ConfigSettingValue>);
impl FromIterator<ConfigSettingEntry> for ConfigSettings { impl FromIterator<ConfigSettingEntry> for ConfigSettings {

0
crates/uv-types/src/constraints.rs → crates/uv-configuration/src/constraints.rs
View File

13
crates/uv-configuration/src/lib.rs Normal file
View File

@ -0,0 +1,13 @@
pub use build_options::*;
pub use config_settings::*;
pub use constraints::*;
pub use name_specifiers::*;
pub use overrides::*;
pub use package_options::*;
mod build_options;
mod config_settings;
mod constraints;
mod name_specifiers;
mod overrides;
mod package_options;

0
crates/uv-types/src/name_specifiers.rs → crates/uv-configuration/src/name_specifiers.rs
View File

0
crates/uv-types/src/overrides.rs → crates/uv-configuration/src/overrides.rs
View File

0
crates/uv-types/src/package_options.rs → crates/uv-configuration/src/package_options.rs
View File

3
crates/uv-dev/Cargo.toml
View File

@ -23,11 +23,14 @@ pep508_rs = { workspace = true }
uv-build = { workspace = true } uv-build = { workspace = true }
uv-cache = { workspace = true, features = ["clap"] } uv-cache = { workspace = true, features = ["clap"] }
uv-client = { workspace = true } uv-client = { workspace = true }
uv-configuration = { workspace = true }
uv-dispatch = { workspace = true } uv-dispatch = { workspace = true }
uv-fs = { workspace = true }
uv-installer = { workspace = true } uv-installer = { workspace = true }
uv-interpreter = { workspace = true } uv-interpreter = { workspace = true }
uv-normalize = { workspace = true } uv-normalize = { workspace = true }
uv-resolver = { workspace = true } uv-resolver = { workspace = true }
uv-toolchain = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
# Any dependencies that are exclusively used in `uv-dev` should be listed as non-workspace # Any dependencies that are exclusively used in `uv-dev` should be listed as non-workspace

12
crates/uv-dev/src/build.rs
View File

@ -9,14 +9,12 @@ use distribution_types::IndexLocations;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use uv_build::{SourceBuild, SourceBuildContext}; use uv_build::{SourceBuild, SourceBuildContext};
use uv_cache::{Cache, CacheArgs}; use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, RegistryClientBuilder}; use uv_client::RegistryClientBuilder;
use uv_configuration::{BuildKind, ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch; use uv_dispatch::BuildDispatch;
use uv_interpreter::PythonEnvironment; use uv_interpreter::PythonEnvironment;
use uv_resolver::InMemoryIndex; use uv_resolver::{FlatIndex, InMemoryIndex};
use uv_types::NoBinary; use uv_types::{BuildContext, BuildIsolation, InFlight};
use uv_types::{
BuildContext, BuildIsolation, BuildKind, ConfigSettings, InFlight, NoBuild, SetupPyStrategy,
};
#[derive(Parser)] #[derive(Parser)]
pub(crate) struct BuildArgs { pub(crate) struct BuildArgs {
@ -93,5 +91,5 @@ pub(crate) async fn build(args: BuildArgs) -> Result<PathBuf> {
FxHashMap::default(), FxHashMap::default(),
) )
.await?; .await?;
Ok(wheel_dir.join(builder.build(&wheel_dir).await?)) Ok(wheel_dir.join(builder.build_wheel(&wheel_dir).await?))
} }

142
crates/uv-dev/src/fetch_python.rs Normal file
View File

@ -0,0 +1,142 @@
use anyhow::Result;
use clap::Parser;
use fs_err as fs;
#[cfg(unix)]
use fs_err::tokio::symlink;
use futures::StreamExt;
#[cfg(unix)]
use itertools::Itertools;
use std::str::FromStr;
#[cfg(unix)]
use std::{collections::HashMap, path::PathBuf};
use tokio::time::Instant;
use tracing::{info, info_span, Instrument};
use uv_fs::Simplified;
use uv_toolchain::{
DownloadResult, Error, PythonDownload, PythonDownloadRequest, TOOLCHAIN_DIRECTORY,
};
#[derive(Parser, Debug)]
pub(crate) struct FetchPythonArgs {
versions: Vec<String>,
}
pub(crate) async fn fetch_python(args: FetchPythonArgs) -> Result<()> {
let start = Instant::now();
let bootstrap_dir = &*TOOLCHAIN_DIRECTORY;
fs_err::create_dir_all(bootstrap_dir)?;
let versions = if args.versions.is_empty() {
info!("Reading versions from file...");
read_versions_file().await?
} else {
args.versions
};
let requests = versions
.iter()
.map(|version| {
PythonDownloadRequest::from_str(version).and_then(PythonDownloadRequest::fill)
})
.collect::<Result<Vec<_>, Error>>()?;
let downloads = requests
.iter()
.map(|request| match PythonDownload::from_request(request) {
Some(download) => download,
None => panic!("No download found for request {request:?}"),
})
.collect::<Vec<_>>();
let client = uv_client::BaseClientBuilder::new().build();
info!("Fetching requested versions...");
let mut tasks = futures::stream::iter(downloads.iter())
.map(|download| {
async {
let result = download.fetch(&client, bootstrap_dir).await;
(download.python_version(), result)
}
.instrument(info_span!("download", key = %download))
})
.buffered(4);
let mut results = Vec::new();
let mut downloaded = 0;
while let Some(task) = tasks.next().await {
let (version, result) = task;
let path = match result? {
DownloadResult::AlreadyAvailable(path) => {
info!("Found existing download for v{}", version);
path
}
DownloadResult::Fetched(path) => {
info!("Downloaded v{} to {}", version, path.user_display());
downloaded += 1;
path
}
};
results.push((version, path));
}
if downloaded > 0 {
let s = if downloaded == 1 { "" } else { "s" };
info!(
"Fetched {} in {}s",
format!("{} version{}", downloaded, s),
start.elapsed().as_secs()
);
} else {
info!("All versions downloaded already.");
};
// Order matters here, as we overwrite previous links
info!("Installing to `{}`...", bootstrap_dir.user_display());
// On Windows, linking the executable generally results in broken installations
// and each toolchain path will need to be added to the PATH separately in the
// desired order
#[cfg(unix)]
{
let mut links: HashMap<PathBuf, PathBuf> = HashMap::new();
for (version, path) in results {
// TODO(zanieb): This path should be a part of the download metadata
let executable = path.join("install").join("bin").join("python3");
for target in [
bootstrap_dir.join(format!("python{}", version.python_full_version())),
bootstrap_dir.join(format!("python{}.{}", version.major(), version.minor())),
bootstrap_dir.join(format!("python{}", version.major())),
bootstrap_dir.join("python"),
] {
// Attempt to remove it, we'll fail on link if we couldn't remove it for some reason
// but if it's missing we don't want to error
let _ = fs::remove_file(&target);
symlink(&executable, &target).await?;
links.insert(target, executable.clone());
}
}
for (target, executable) in links.iter().sorted() {
info!(
"Linked `{}` to `{}`",
target.user_display(),
executable.user_display()
);
}
};
info!("Installed {} versions", requests.len());
Ok(())
}
async fn read_versions_file() -> Result<Vec<String>> {
let lines: Vec<String> = fs::tokio::read_to_string(".python-versions")
.await?
.lines()
.map(ToString::to_string)
.collect();
Ok(lines)
}

5
crates/uv-dev/src/main.rs
View File

@ -21,6 +21,7 @@ use resolve_many::ResolveManyArgs;
use crate::build::{build, BuildArgs}; use crate::build::{build, BuildArgs};
use crate::clear_compile::ClearCompileArgs; use crate::clear_compile::ClearCompileArgs;
use crate::compile::CompileArgs; use crate::compile::CompileArgs;
use crate::fetch_python::FetchPythonArgs;
use crate::render_benchmarks::RenderBenchmarksArgs; use crate::render_benchmarks::RenderBenchmarksArgs;
use crate::resolve_cli::ResolveCliArgs; use crate::resolve_cli::ResolveCliArgs;
use crate::wheel_metadata::WheelMetadataArgs; use crate::wheel_metadata::WheelMetadataArgs;
@ -44,6 +45,7 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
mod build; mod build;
mod clear_compile; mod clear_compile;
mod compile; mod compile;
mod fetch_python;
mod render_benchmarks; mod render_benchmarks;
mod resolve_cli; mod resolve_cli;
mod resolve_many; mod resolve_many;
@ -72,6 +74,8 @@ enum Cli {
Compile(CompileArgs), Compile(CompileArgs),
/// Remove all `.pyc` in the tree. /// Remove all `.pyc` in the tree.
ClearCompile(ClearCompileArgs), ClearCompile(ClearCompileArgs),
/// Fetch Python versions for testing
FetchPython(FetchPythonArgs),
} }
#[instrument] // Anchor span to check for overhead #[instrument] // Anchor span to check for overhead
@ -92,6 +96,7 @@ async fn run() -> Result<()> {
Cli::RenderBenchmarks(args) => render_benchmarks::render_benchmarks(&args)?, Cli::RenderBenchmarks(args) => render_benchmarks::render_benchmarks(&args)?,
Cli::Compile(args) => compile::compile(args).await?, Cli::Compile(args) => compile::compile(args).await?,
Cli::ClearCompile(args) => clear_compile::clear_compile(&args)?, Cli::ClearCompile(args) => clear_compile::clear_compile(&args)?,
Cli::FetchPython(args) => fetch_python::fetch_python(args).await?,
} }
Ok(()) Ok(())
} }

30
crates/uv-dev/src/resolve_cli.rs
View File

@ -12,12 +12,13 @@ use petgraph::dot::{Config as DotConfig, Dot};
use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Resolution}; use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Resolution};
use pep508_rs::Requirement; use pep508_rs::Requirement;
use uv_cache::{Cache, CacheArgs}; use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder}; use uv_client::{FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch; use uv_dispatch::BuildDispatch;
use uv_installer::SitePackages; use uv_installer::SitePackages;
use uv_interpreter::PythonEnvironment; use uv_interpreter::PythonEnvironment;
use uv_resolver::{InMemoryIndex, Manifest, Options, Resolver}; use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, Resolver};
use uv_types::{BuildIsolation, ConfigSettings, InFlight, NoBinary, NoBuild, SetupPyStrategy}; use uv_types::{BuildIsolation, HashStrategy, InFlight};
#[derive(ValueEnum, Default, Clone)] #[derive(ValueEnum, Default, Clone)]
pub(crate) enum ResolveCliFormat { pub(crate) enum ResolveCliFormat {
@ -56,14 +57,6 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
let venv = PythonEnvironment::from_virtualenv(&cache)?; let venv = PythonEnvironment::from_virtualenv(&cache)?;
let index_locations = let index_locations =
IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false); IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false);
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.build();
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(entries, venv.interpreter().tags()?)
};
let index = InMemoryIndex::default(); let index = InMemoryIndex::default();
let in_flight = InFlight::default(); let in_flight = InFlight::default();
let no_build = if args.no_build { let no_build = if args.no_build {
@ -71,6 +64,20 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
} else { } else {
NoBuild::None NoBuild::None
}; };
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_locations.index_urls())
.build();
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(
entries,
venv.interpreter().tags()?,
&HashStrategy::None,
&no_build,
&NoBinary::None,
)
};
let config_settings = ConfigSettings::default(); let config_settings = ConfigSettings::default();
let build_dispatch = BuildDispatch::new( let build_dispatch = BuildDispatch::new(
@ -101,6 +108,7 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
&client, &client,
&flat_index, &flat_index,
&index, &index,
&HashStrategy::None,
&build_dispatch, &build_dispatch,
&site_packages, &site_packages,
)?; )?;

21
crates/uv-dev/src/resolve_many.rs
View File

@ -14,13 +14,13 @@ use distribution_types::IndexLocations;
use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::{Requirement, VersionOrUrl}; use pep508_rs::{Requirement, VersionOrUrl};
use uv_cache::{Cache, CacheArgs}; use uv_cache::{Cache, CacheArgs};
use uv_client::{FlatIndex, OwnedArchive, RegistryClient, RegistryClientBuilder}; use uv_client::{OwnedArchive, RegistryClient, RegistryClientBuilder};
use uv_configuration::{ConfigSettings, NoBinary, NoBuild, SetupPyStrategy};
use uv_dispatch::BuildDispatch; use uv_dispatch::BuildDispatch;
use uv_interpreter::PythonEnvironment; use uv_interpreter::PythonEnvironment;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_resolver::InMemoryIndex; use uv_resolver::{FlatIndex, InMemoryIndex};
use uv_types::NoBinary; use uv_types::{BuildContext, BuildIsolation, InFlight};
use uv_types::{BuildContext, BuildIsolation, ConfigSettings, InFlight, NoBuild, SetupPyStrategy};
#[derive(Parser)] #[derive(Parser)]
pub(crate) struct ResolveManyArgs { pub(crate) struct ResolveManyArgs {
@ -47,10 +47,17 @@ async fn find_latest_version(
client: &RegistryClient, client: &RegistryClient,
package_name: &PackageName, package_name: &PackageName,
) -> Option<Version> { ) -> Option<Version> {
let (_, raw_simple_metadata) = client.simple(package_name).await.ok()?; client
.simple(package_name)
.await
.ok()
.into_iter()
.flatten()
.filter_map(|(_index, raw_simple_metadata)| {
let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata); let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata);
let version = simple_metadata.into_iter().next()?.version; Some(simple_metadata.into_iter().next()?.version)
Some(version) })
.max()
} }
pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> { pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {

2
crates/uv-dispatch/Cargo.toml
View File

@ -21,9 +21,9 @@ uv-cache = { workspace = true }
uv-client = { workspace = true } uv-client = { workspace = true }
uv-installer = { workspace = true } uv-installer = { workspace = true }
uv-interpreter = { workspace = true } uv-interpreter = { workspace = true }
uv-requirements = { workspace = true }
uv-resolver = { workspace = true } uv-resolver = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
uv-configuration = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }
futures = { workspace = true } futures = { workspace = true }

31
crates/uv-dispatch/src/lib.rs
View File

@ -3,8 +3,8 @@
//! implementing [`BuildContext`]. //! implementing [`BuildContext`].
use std::ffi::OsStr; use std::ffi::OsStr;
use std::ffi::OsString;
use std::path::Path; use std::path::Path;
use std::{ffi::OsString, future::Future};
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use futures::FutureExt; use futures::FutureExt;
@ -16,14 +16,12 @@ use distribution_types::{IndexLocations, Name, Resolution, SourceDist};
use pep508_rs::Requirement; use pep508_rs::Requirement;
use uv_build::{SourceBuild, SourceBuildContext}; use uv_build::{SourceBuild, SourceBuildContext};
use uv_cache::Cache; use uv_cache::Cache;
use uv_client::{FlatIndex, RegistryClient}; use uv_client::RegistryClient;
use uv_configuration::{BuildKind, ConfigSettings, NoBinary, NoBuild, Reinstall, SetupPyStrategy};
use uv_installer::{Downloader, Installer, Plan, Planner, SitePackages}; use uv_installer::{Downloader, Installer, Plan, Planner, SitePackages};
use uv_interpreter::{Interpreter, PythonEnvironment}; use uv_interpreter::{Interpreter, PythonEnvironment};
use uv_resolver::{InMemoryIndex, Manifest, Options, Resolver}; use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, Resolver};
use uv_types::{ use uv_types::{BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
BuildContext, BuildIsolation, BuildKind, ConfigSettings, EmptyInstalledPackages, InFlight,
NoBinary, NoBuild, Reinstall, SetupPyStrategy,
};
/// The main implementation of [`BuildContext`], used by the CLI, see [`BuildContext`] /// The main implementation of [`BuildContext`], used by the CLI, see [`BuildContext`]
/// documentation. /// documentation.
@ -145,6 +143,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
self.client, self.client,
self.flat_index, self.flat_index,
self.index, self.index,
&HashStrategy::None,
self, self,
&EmptyInstalledPackages, &EmptyInstalledPackages,
)?; )?;
@ -157,7 +156,6 @@ impl<'a> BuildContext for BuildDispatch<'a> {
Ok(Resolution::from(graph)) Ok(Resolution::from(graph))
} }
#[allow(clippy::manual_async_fn)] // TODO(konstin): rustc 1.75 gets into a type inference cycle with async fn
#[instrument( #[instrument(
skip(self, resolution, venv), skip(self, resolution, venv),
fields( fields(
@ -165,12 +163,11 @@ impl<'a> BuildContext for BuildDispatch<'a> {
venv = ?venv.root() venv = ?venv.root()
) )
)] )]
fn install<'data>( async fn install<'data>(
&'data self, &'data self,
resolution: &'data Resolution, resolution: &'data Resolution,
venv: &'data PythonEnvironment, venv: &'data PythonEnvironment,
) -> impl Future<Output = Result<()>> + Send + 'data { ) -> Result<()> {
async move {
debug!( debug!(
"Installing in {} in {}", "Installing in {} in {}",
resolution resolution
@ -196,6 +193,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
site_packages, site_packages,
&Reinstall::None, &Reinstall::None,
&NoBinary::None, &NoBinary::None,
&HashStrategy::None,
self.index_locations, self.index_locations,
self.cache(), self.cache(),
venv, venv,
@ -224,7 +222,8 @@ impl<'a> BuildContext for BuildDispatch<'a> {
vec![] vec![]
} else { } else {
// TODO(konstin): Check that there is no endless recursion. // TODO(konstin): Check that there is no endless recursion.
let downloader = Downloader::new(self.cache, tags, self.client, self); let downloader =
Downloader::new(self.cache, tags, &HashStrategy::None, self.client, self);
debug!( debug!(
"Downloading and building requirement{} for build: {}", "Downloading and building requirement{} for build: {}",
if remote.len() == 1 { "" } else { "s" }, if remote.len() == 1 { "" } else { "s" },
@ -269,15 +268,13 @@ impl<'a> BuildContext for BuildDispatch<'a> {
Ok(()) Ok(())
} }
}
#[allow(clippy::manual_async_fn)] // TODO(konstin): rustc 1.75 gets into a type inference cycle with async fn #[instrument(skip_all, fields(version_id = version_id, subdirectory = ?subdirectory))]
#[instrument(skip_all, fields(package_id = package_id, subdirectory = ?subdirectory))]
async fn setup_build<'data>( async fn setup_build<'data>(
&'data self, &'data self,
source: &'data Path, source: &'data Path,
subdirectory: Option<&'data Path>, subdirectory: Option<&'data Path>,
package_id: &'data str, version_id: &'data str,
dist: Option<&'data SourceDist>, dist: Option<&'data SourceDist>,
build_kind: BuildKind, build_kind: BuildKind,
) -> Result<SourceBuild> { ) -> Result<SourceBuild> {
@ -307,7 +304,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
self.interpreter, self.interpreter,
self, self,
self.source_build_context.clone(), self.source_build_context.clone(),
package_id.to_string(), version_id.to_string(),
self.setup_py, self.setup_py,
self.config_settings.clone(), self.config_settings.clone(),
self.build_isolation, self.build_isolation,

4
crates/uv-distribution/Cargo.toml
View File

@ -28,10 +28,12 @@ uv-fs = { workspace = true, features = ["tokio"] }
uv-git = { workspace = true, features = ["vendored-openssl"] } uv-git = { workspace = true, features = ["vendored-openssl"] }
uv-normalize = { workspace = true } uv-normalize = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
uv-configuration = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
md-5 = { workspace = true }
nanoid = { workspace = true } nanoid = { workspace = true }
once_cell = { workspace = true } once_cell = { workspace = true }
reqwest = { workspace = true } reqwest = { workspace = true }
@ -39,6 +41,7 @@ reqwest-middleware = { workspace = true }
rmp-serde = { workspace = true } rmp-serde = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
sha2 = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
@ -46,4 +49,3 @@ tokio-util = { workspace = true, features = ["compat"] }
tracing = { workspace = true } tracing = { workspace = true }
url = { workspace = true } url = { workspace = true }
zip = { workspace = true } zip = { workspace = true }

25
crates/uv-distribution/src/archive.rs Normal file
View File

@ -0,0 +1,25 @@
use distribution_types::Hashed;
use pypi_types::HashDigest;
use uv_cache::ArchiveId;
/// An archive (unzipped wheel) that exists in the local cache.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Archive {
/// The unique ID of the entry in the wheel's archive bucket.
pub id: ArchiveId,
/// The computed hashes of the archive.
pub hashes: Vec<HashDigest>,
}
impl Archive {
/// Create a new [`Archive`] with the given ID and hashes.
pub(crate) fn new(id: ArchiveId, hashes: Vec<HashDigest>) -> Self {
Self { id, hashes }
}
}
impl Hashed for Archive {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}

580
crates/uv-distribution/src/distribution_database.rs
View File

@ -1,8 +1,9 @@
use std::io; use std::io;
use std::path::{Path, PathBuf}; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use futures::{FutureExt, TryStreamExt}; use futures::{FutureExt, TryStreamExt};
use tempfile::TempDir;
use tokio::io::AsyncSeekExt; use tokio::io::AsyncSeekExt;
use tokio_util::compat::FuturesAsyncReadCompatExt; use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info_span, instrument, warn, Instrument}; use tracing::{info_span, instrument, warn, Instrument};
@ -10,17 +11,23 @@ use url::Url;
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
use distribution_types::{ use distribution_types::{
BuildableSource, BuiltDist, Dist, FileLocation, IndexLocations, LocalEditable, Name, SourceDist, BuildableSource, BuiltDist, Dist, FileLocation, HashPolicy, Hashed, IndexLocations,
LocalEditable, Name, SourceDist,
}; };
use platform_tags::Tags; use platform_tags::Tags;
use pypi_types::Metadata23; use pypi_types::{HashDigest, Metadata23};
use uv_cache::{ArchiveTarget, ArchiveTimestamp, CacheBucket, CacheEntry, WheelCache}; use uv_cache::{ArchiveId, ArchiveTimestamp, CacheBucket, CacheEntry, Timestamp, WheelCache};
use uv_client::{CacheControl, CachedClientError, Connectivity, RegistryClient}; use uv_client::{
use uv_types::{BuildContext, NoBinary, NoBuild}; CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient,
};
use uv_configuration::{NoBinary, NoBuild};
use uv_extract::hash::Hasher;
use uv_fs::write_atomic;
use uv_types::BuildContext;
use crate::download::{BuiltWheel, UnzippedWheel}; use crate::archive::Archive;
use crate::locks::Locks; use crate::locks::Locks;
use crate::{DiskWheel, Error, LocalWheel, Reporter, SourceDistributionBuilder}; use crate::{ArchiveMetadata, Error, LocalWheel, Reporter, SourceDistributionBuilder};
/// A cached high-level interface to convert distributions (a requirement resolved to a location) /// A cached high-level interface to convert distributions (a requirement resolved to a location)
/// to a wheel or wheel metadata. /// to a wheel or wheel metadata.
@ -77,28 +84,38 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
/// Either fetch the wheel or fetch and build the source distribution /// Either fetch the wheel or fetch and build the source distribution
/// ///
/// If `no_remote_wheel` is set, the wheel will be built from a source distribution /// Returns a wheel that's compliant with the given platform tags.
/// even if compatible pre-built wheels are available. ///
/// While hashes will be generated in some cases, hash-checking is only enforced for source
/// distributions, and should be enforced by the caller for wheels.
#[instrument(skip_all, fields(%dist))] #[instrument(skip_all, fields(%dist))]
pub async fn get_or_build_wheel(&self, dist: &Dist, tags: &Tags) -> Result<LocalWheel, Error> { pub async fn get_or_build_wheel(
&self,
dist: &Dist,
tags: &Tags,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
match dist { match dist {
Dist::Built(built) => self.get_wheel(built).await, Dist::Built(built) => self.get_wheel(built, hashes).await,
Dist::Source(source) => self.build_wheel(source, tags).await, Dist::Source(source) => self.build_wheel(source, tags, hashes).await,
} }
} }
/// Either fetch the only wheel metadata (directly from the index or with range requests) or /// Either fetch the only wheel metadata (directly from the index or with range requests) or
/// fetch and build the source distribution. /// fetch and build the source distribution.
/// ///
/// Returns the [`Metadata23`], along with a "precise" URL for the source distribution, if /// While hashes will be generated in some cases, hash-checking is only enforced for source
/// possible. For example, given a Git dependency with a reference to a branch or tag, return a /// distributions, and should be enforced by the caller for wheels.
/// URL with a precise reference to the current commit of that branch or tag.
#[instrument(skip_all, fields(%dist))] #[instrument(skip_all, fields(%dist))]
pub async fn get_or_build_wheel_metadata(&self, dist: &Dist) -> Result<Metadata23, Error> { pub async fn get_or_build_wheel_metadata(
&self,
dist: &Dist,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
match dist { match dist {
Dist::Built(built) => self.get_wheel_metadata(built).await, Dist::Built(built) => self.get_wheel_metadata(built, hashes).await,
Dist::Source(source) => { Dist::Source(source) => {
self.build_wheel_metadata(&BuildableSource::Dist(source)) self.build_wheel_metadata(&BuildableSource::Dist(source), hashes)
.await .await
} }
} }
@ -110,22 +127,35 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
editable: &LocalEditable, editable: &LocalEditable,
editable_wheel_dir: &Path, editable_wheel_dir: &Path,
) -> Result<(LocalWheel, Metadata23), Error> { ) -> Result<(LocalWheel, Metadata23), Error> {
// Build the wheel.
let (dist, disk_filename, filename, metadata) = self let (dist, disk_filename, filename, metadata) = self
.builder .builder
.build_editable(editable, editable_wheel_dir) .build_editable(editable, editable_wheel_dir)
.await?; .await?;
let built_wheel = BuiltWheel { // Unzip into the editable wheel directory.
let path = editable_wheel_dir.join(&disk_filename);
let target = editable_wheel_dir.join(cache_key::digest(&editable.path));
let id = self.unzip_wheel(&path, &target).await?;
let wheel = LocalWheel {
dist, dist,
filename, filename,
path: editable_wheel_dir.join(disk_filename), archive: self.build_context.cache().archive(&id),
target: editable_wheel_dir.join(cache_key::digest(&editable.path)), hashes: vec![],
}; };
Ok((LocalWheel::Built(built_wheel), metadata))
Ok((wheel, metadata))
} }
/// Fetch a wheel from the cache or download it from the index. /// Fetch a wheel from the cache or download it from the index.
async fn get_wheel(&self, dist: &BuiltDist) -> Result<LocalWheel, Error> { ///
/// While hashes will be generated in all cases, hash-checking is _not_ enforced and should
/// instead be enforced by the caller.
async fn get_wheel(
&self,
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
let no_binary = match self.build_context.no_binary() { let no_binary = match self.build_context.no_binary() {
NoBinary::None => false, NoBinary::None => false,
NoBinary::All => true, NoBinary::All => true,
@ -145,41 +175,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
Url::parse(url).map_err(|err| Error::Url(url.clone(), err))? Url::parse(url).map_err(|err| Error::Url(url.clone(), err))?
} }
FileLocation::Path(path) => { FileLocation::Path(path) => {
let url = Url::from_file_path(path).expect("path is absolute");
let cache_entry = self.build_context.cache().entry( let cache_entry = self.build_context.cache().entry(
CacheBucket::Wheels, CacheBucket::Wheels,
WheelCache::Url(&url).wheel_dir(wheel.name().as_ref()), WheelCache::Index(&wheel.index).wheel_dir(wheel.name().as_ref()),
wheel.filename.stem(), wheel.filename.stem(),
); );
return self
// If the file is already unzipped, and the unzipped directory is fresh, .load_wheel(path, &wheel.filename, cache_entry, dist, hashes)
// return it. .await;
match cache_entry.path().canonicalize() {
Ok(archive) => {
if ArchiveTimestamp::up_to_date_with(
path,
ArchiveTarget::Cache(&archive),
)
.map_err(Error::CacheRead)?
{
return Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: Dist::Built(dist.clone()),
archive,
filename: wheel.filename.clone(),
}));
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
// Otherwise, unzip the file.
return Ok(LocalWheel::Disk(DiskWheel {
dist: Dist::Built(dist.clone()),
path: path.clone(),
target: cache_entry.into_path_buf(),
filename: wheel.filename.clone(),
}));
} }
}; };
@ -192,14 +195,15 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
// Download and unzip. // Download and unzip.
match self match self
.stream_wheel(url.clone(), &wheel.filename, &wheel_entry, dist) .stream_wheel(url.clone(), &wheel.filename, &wheel_entry, dist, hashes)
.await .await
{ {
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel { Ok(archive) => Ok(LocalWheel {
dist: Dist::Built(dist.clone()), dist: Dist::Built(dist.clone()),
archive, archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(), filename: wheel.filename.clone(),
})), }),
Err(Error::Extract(err)) if err.is_http_streaming_unsupported() => { Err(Error::Extract(err)) if err.is_http_streaming_unsupported() => {
warn!( warn!(
"Streaming unsupported for {dist}; downloading wheel to disk ({err})" "Streaming unsupported for {dist}; downloading wheel to disk ({err})"
@ -208,13 +212,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
// If the request failed because streaming is unsupported, download the // If the request failed because streaming is unsupported, download the
// wheel directly. // wheel directly.
let archive = self let archive = self
.download_wheel(url, &wheel.filename, &wheel_entry, dist) .download_wheel(url, &wheel.filename, &wheel_entry, dist, hashes)
.await?; .await?;
Ok(LocalWheel::Unzipped(UnzippedWheel { Ok(LocalWheel {
dist: Dist::Built(dist.clone()), dist: Dist::Built(dist.clone()),
archive, archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(), filename: wheel.filename.clone(),
})) })
} }
Err(err) => Err(err), Err(err) => Err(err),
} }
@ -230,14 +235,21 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
// Download and unzip. // Download and unzip.
match self match self
.stream_wheel(wheel.url.raw().clone(), &wheel.filename, &wheel_entry, dist) .stream_wheel(
wheel.url.raw().clone(),
&wheel.filename,
&wheel_entry,
dist,
hashes,
)
.await .await
{ {
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel { Ok(archive) => Ok(LocalWheel {
dist: Dist::Built(dist.clone()), dist: Dist::Built(dist.clone()),
archive, archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(), filename: wheel.filename.clone(),
})), }),
Err(Error::Client(err)) if err.is_http_streaming_unsupported() => { Err(Error::Client(err)) if err.is_http_streaming_unsupported() => {
warn!( warn!(
"Streaming unsupported for {dist}; downloading wheel to disk ({err})" "Streaming unsupported for {dist}; downloading wheel to disk ({err})"
@ -251,13 +263,15 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
&wheel.filename, &wheel.filename,
&wheel_entry, &wheel_entry,
dist, dist,
hashes,
) )
.await?; .await?;
Ok(LocalWheel::Unzipped(UnzippedWheel { Ok(LocalWheel {
dist: Dist::Built(dist.clone()), dist: Dist::Built(dist.clone()),
archive, archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: wheel.filename.clone(), filename: wheel.filename.clone(),
})) })
} }
Err(err) => Err(err), Err(err) => Err(err),
} }
@ -270,90 +284,107 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
wheel.filename.stem(), wheel.filename.stem(),
); );
// If the file is already unzipped, and the unzipped directory is fresh, self.load_wheel(&wheel.path, &wheel.filename, cache_entry, dist, hashes)
// return it. .await
match cache_entry.path().canonicalize() {
Ok(archive) => {
if ArchiveTimestamp::up_to_date_with(
&wheel.path,
ArchiveTarget::Cache(&archive),
)
.map_err(Error::CacheRead)?
{
return Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: Dist::Built(dist.clone()),
archive,
filename: wheel.filename.clone(),
}));
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
Ok(LocalWheel::Disk(DiskWheel {
dist: Dist::Built(dist.clone()),
path: wheel.path.clone(),
target: cache_entry.into_path_buf(),
filename: wheel.filename.clone(),
}))
} }
} }
} }
/// Convert a source distribution into a wheel, fetching it from the cache or building it if /// Convert a source distribution into a wheel, fetching it from the cache or building it if
/// necessary. /// necessary.
async fn build_wheel(&self, dist: &SourceDist, tags: &Tags) -> Result<LocalWheel, Error> { ///
/// The returned wheel is guaranteed to come from a distribution with a matching hash, and
/// no build processes will be executed for distributions with mismatched hashes.
async fn build_wheel(
&self,
dist: &SourceDist,
tags: &Tags,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
let lock = self.locks.acquire(&Dist::Source(dist.clone())).await; let lock = self.locks.acquire(&Dist::Source(dist.clone())).await;
let _guard = lock.lock().await; let _guard = lock.lock().await;
let built_wheel = self let built_wheel = self
.builder .builder
.download_and_build(&BuildableSource::Dist(dist), tags) .download_and_build(&BuildableSource::Dist(dist), tags, hashes)
.boxed() .boxed()
.await?; .await?;
// If the wheel was unzipped previously, respect it. Source distributions are // If the wheel was unzipped previously, respect it. Source distributions are
// cached under a unique build ID, so unzipped directories are never stale. // cached under a unique revision ID, so unzipped directories are never stale.
match built_wheel.target.canonicalize() { match built_wheel.target.canonicalize() {
Ok(archive) => Ok(LocalWheel::Unzipped(UnzippedWheel { Ok(archive) => {
return Ok(LocalWheel {
dist: Dist::Source(dist.clone()), dist: Dist::Source(dist.clone()),
archive, archive,
filename: built_wheel.filename, filename: built_wheel.filename,
})), hashes: built_wheel.hashes,
Err(err) if err.kind() == io::ErrorKind::NotFound => { });
Ok(LocalWheel::Built(BuiltWheel { }
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(Error::CacheRead(err)),
}
// Otherwise, unzip the wheel.
let id = self
.unzip_wheel(&built_wheel.path, &built_wheel.target)
.await?;
Ok(LocalWheel {
dist: Dist::Source(dist.clone()), dist: Dist::Source(dist.clone()),
path: built_wheel.path, archive: self.build_context.cache().archive(&id),
target: built_wheel.target, hashes: built_wheel.hashes,
filename: built_wheel.filename, filename: built_wheel.filename,
})) })
}
Err(err) => Err(Error::CacheRead(err)),
}
} }
/// Fetch the wheel metadata from the index, or from the cache if possible. /// Fetch the wheel metadata from the index, or from the cache if possible.
pub async fn get_wheel_metadata(&self, dist: &BuiltDist) -> Result<Metadata23, Error> { ///
/// While hashes will be generated in some cases, hash-checking is _not_ enforced and should
/// instead be enforced by the caller.
pub async fn get_wheel_metadata(
&self,
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
// If hash generation is enabled, and the distribution isn't hosted on an index, get the
// entire wheel to ensure that the hashes are included in the response. If the distribution
// is hosted on an index, the hashes will be included in the simple metadata response.
// For hash _validation_, callers are expected to enforce the policy when retrieving the
// wheel.
// TODO(charlie): Request the hashes via a separate method, to reduce the coupling in this API.
if hashes.is_generate() && matches!(dist, BuiltDist::DirectUrl(_) | BuiltDist::Path(_)) {
let wheel = self.get_wheel(dist, hashes).await?;
let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
return Ok(ArchiveMetadata { metadata, hashes });
}
match self.client.wheel_metadata(dist).boxed().await { match self.client.wheel_metadata(dist).boxed().await {
Ok(metadata) => Ok(metadata), Ok(metadata) => Ok(ArchiveMetadata::from(metadata)),
Err(err) if err.is_http_streaming_unsupported() => { Err(err) if err.is_http_streaming_unsupported() => {
warn!("Streaming unsupported when fetching metadata for {dist}; downloading wheel directly ({err})"); warn!("Streaming unsupported when fetching metadata for {dist}; downloading wheel directly ({err})");
// If the request failed due to an error that could be resolved by // If the request failed due to an error that could be resolved by
// downloading the wheel directly, try that. // downloading the wheel directly, try that.
let wheel = self.get_wheel(dist).await?; let wheel = self.get_wheel(dist, hashes).await?;
Ok(wheel.metadata()?) let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
Ok(ArchiveMetadata { metadata, hashes })
} }
Err(err) => Err(err.into()), Err(err) => Err(err.into()),
} }
} }
/// Build the wheel metadata for a source distribution, or fetch it from the cache if possible. /// Build the wheel metadata for a source distribution, or fetch it from the cache if possible.
///
/// The returned metadata is guaranteed to come from a distribution with a matching hash, and
/// no build processes will be executed for distributions with mismatched hashes.
pub async fn build_wheel_metadata( pub async fn build_wheel_metadata(
&self, &self,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
) -> Result<Metadata23, Error> { hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
let no_build = match self.build_context.no_build() { let no_build = match self.build_context.no_build() {
NoBuild::All => true, NoBuild::All => true,
NoBuild::None => false, NoBuild::None => false,
@ -372,7 +403,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
let metadata = self let metadata = self
.builder .builder
.download_and_build_metadata(source) .download_and_build_metadata(source, hashes)
.boxed() .boxed()
.await?; .await?;
Ok(metadata) Ok(metadata)
@ -385,7 +416,8 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
filename: &WheelFilename, filename: &WheelFilename,
wheel_entry: &CacheEntry, wheel_entry: &CacheEntry,
dist: &BuiltDist, dist: &BuiltDist,
) -> Result<PathBuf, Error> { hashes: HashPolicy<'_>,
) -> Result<Archive, Error> {
// Create an entry for the HTTP cache. // Create an entry for the HTTP cache.
let http_entry = wheel_entry.with_file(format!("{}.http", filename.stem())); let http_entry = wheel_entry.with_file(format!("{}.http", filename.stem()));
@ -396,35 +428,39 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
.map_err(|err| self.handle_response_errors(err)) .map_err(|err| self.handle_response_errors(err))
.into_async_read(); .into_async_read();
// Create a hasher for each hash algorithm.
let algorithms = hashes.algorithms();
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);
// Download and unzip the wheel to a temporary directory. // Download and unzip the wheel to a temporary directory.
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root()) let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
.map_err(Error::CacheWrite)?; .map_err(Error::CacheWrite)?;
uv_extract::stream::unzip(reader.compat(), temp_dir.path()).await?; uv_extract::stream::unzip(&mut hasher, temp_dir.path()).await?;
// If necessary, exhaust the reader to compute the hash.
if !hashes.is_none() {
hasher.finish().await.map_err(Error::HashExhaustion)?;
}
// Persist the temporary directory to the directory store. // Persist the temporary directory to the directory store.
let archive = self let id = self
.build_context .build_context
.cache() .cache()
.persist(temp_dir.into_path(), wheel_entry.path()) .persist(temp_dir.into_path(), wheel_entry.path())
.await .await
.map_err(Error::CacheRead)?; .map_err(Error::CacheRead)?;
Ok(archive)
Ok(Archive::new(
id,
hashers.into_iter().map(HashDigest::from).collect(),
))
} }
.instrument(info_span!("wheel", wheel = %dist)) .instrument(info_span!("wheel", wheel = %dist))
}; };
let req = self // Fetch the archive from the cache, or download it if necessary.
.client let req = self.request(url.clone())?;
.uncached_client()
.get(url)
.header(
// `reqwest` defaults to accepting compressed responses.
// Specify identity encoding to get consistent .whl downloading
// behavior from servers. ref: https://github.com/pypa/pip/pull/1688
"accept-encoding",
reqwest::header::HeaderValue::from_static("identity"),
)
.build()?;
let cache_control = match self.client.connectivity() { let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from( Connectivity::Online => CacheControl::from(
self.build_context self.build_context
@ -434,7 +470,6 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
), ),
Connectivity::Offline => CacheControl::AllowStale, Connectivity::Offline => CacheControl::AllowStale,
}; };
let archive = self let archive = self
.client .client
.cached_client() .cached_client()
@ -445,6 +480,20 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
CachedClientError::Client(err) => Error::Client(err), CachedClientError::Client(err) => Error::Client(err),
})?; })?;
// If the archive is missing the required hashes, force a refresh.
let archive = if archive.has_digests(hashes) {
archive
} else {
self.client
.cached_client()
.skip_cache(self.request(url)?, &http_entry, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
CachedClientError::Client(err) => Error::Client(err),
})?
};
Ok(archive) Ok(archive)
} }
@ -455,7 +504,8 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
filename: &WheelFilename, filename: &WheelFilename,
wheel_entry: &CacheEntry, wheel_entry: &CacheEntry,
dist: &BuiltDist, dist: &BuiltDist,
) -> Result<PathBuf, Error> { hashes: HashPolicy<'_>,
) -> Result<Archive, Error> {
// Create an entry for the HTTP cache. // Create an entry for the HTTP cache.
let http_entry = wheel_entry.with_file(format!("{}.http", filename.stem())); let http_entry = wheel_entry.with_file(format!("{}.http", filename.stem()));
@ -481,33 +531,48 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
file.seek(io::SeekFrom::Start(0)) file.seek(io::SeekFrom::Start(0))
.await .await
.map_err(Error::CacheWrite)?; .map_err(Error::CacheWrite)?;
let reader = tokio::io::BufReader::new(file);
uv_extract::seek::unzip(reader, temp_dir.path()).await?; // If no hashes are required, parallelize the unzip operation.
let hashes = if hashes.is_none() {
let file = file.into_std().await;
tokio::task::spawn_blocking({
let target = temp_dir.path().to_owned();
move || -> Result<(), uv_extract::Error> {
// Unzip the wheel into a temporary directory.
uv_extract::unzip(file, &target)?;
Ok(())
}
})
.await??;
vec![]
} else {
// Create a hasher for each hash algorithm.
let algorithms = hashes.algorithms();
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers);
uv_extract::stream::unzip(&mut hasher, temp_dir.path()).await?;
// If necessary, exhaust the reader to compute the hash.
hasher.finish().await.map_err(Error::HashExhaustion)?;
hashers.into_iter().map(HashDigest::from).collect()
};
// Persist the temporary directory to the directory store. // Persist the temporary directory to the directory store.
let archive = self let id = self
.build_context .build_context
.cache() .cache()
.persist(temp_dir.into_path(), wheel_entry.path()) .persist(temp_dir.into_path(), wheel_entry.path())
.await .await
.map_err(Error::CacheRead)?; .map_err(Error::CacheRead)?;
Ok(archive)
Ok(Archive::new(id, hashes))
} }
.instrument(info_span!("wheel", wheel = %dist)) .instrument(info_span!("wheel", wheel = %dist))
}; };
let req = self let req = self.request(url.clone())?;
.client
.uncached_client()
.get(url)
.header(
// `reqwest` defaults to accepting compressed responses.
// Specify identity encoding to get consistent .whl downloading
// behavior from servers. ref: https://github.com/pypa/pip/pull/1688
"accept-encoding",
reqwest::header::HeaderValue::from_static("identity"),
)
.build()?;
let cache_control = match self.client.connectivity() { let cache_control = match self.client.connectivity() {
Connectivity::Online => CacheControl::from( Connectivity::Online => CacheControl::from(
self.build_context self.build_context
@ -517,7 +582,6 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
), ),
Connectivity::Offline => CacheControl::AllowStale, Connectivity::Offline => CacheControl::AllowStale,
}; };
let archive = self let archive = self
.client .client
.cached_client() .cached_client()
@ -528,11 +592,225 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
CachedClientError::Client(err) => Error::Client(err), CachedClientError::Client(err) => Error::Client(err),
})?; })?;
// If the archive is missing the required hashes, force a refresh.
let archive = if archive.has_digests(hashes) {
archive
} else {
self.client
.cached_client()
.skip_cache(self.request(url)?, &http_entry, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
CachedClientError::Client(err) => Error::Client(err),
})?
};
Ok(archive) Ok(archive)
} }
/// Load a wheel from a local path.
async fn load_wheel(
&self,
path: &Path,
filename: &WheelFilename,
wheel_entry: CacheEntry,
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<LocalWheel, Error> {
// Determine the last-modified time of the wheel.
let modified = ArchiveTimestamp::from_file(path).map_err(Error::CacheRead)?;
// Attempt to read the archive pointer from the cache.
let pointer_entry = wheel_entry.with_file(format!("{}.rev", filename.stem()));
let pointer = LocalArchivePointer::read_from(&pointer_entry)?;
// Extract the archive from the pointer.
let archive = pointer
.filter(|pointer| pointer.is_up_to_date(modified))
.map(LocalArchivePointer::into_archive)
.filter(|archive| archive.has_digests(hashes));
// If the file is already unzipped, and the cache is up-to-date, return it.
if let Some(archive) = archive {
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: filename.clone(),
})
} else if hashes.is_none() {
// Otherwise, unzip the wheel.
let archive = Archive::new(self.unzip_wheel(path, wheel_entry.path()).await?, vec![]);
// Write the archive pointer to the cache.
let pointer = LocalArchivePointer {
timestamp: modified.timestamp(),
archive: archive.clone(),
};
pointer.write_to(&pointer_entry).await?;
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: filename.clone(),
})
} else {
// If necessary, compute the hashes of the wheel.
let file = fs_err::tokio::File::open(path)
.await
.map_err(Error::CacheRead)?;
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
.map_err(Error::CacheWrite)?;
// Create a hasher for each hash algorithm.
let algorithms = hashes.algorithms();
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers);
// Unzip the wheel to a temporary directory.
uv_extract::stream::unzip(&mut hasher, temp_dir.path()).await?;
// Exhaust the reader to compute the hash.
hasher.finish().await.map_err(Error::HashExhaustion)?;
let hashes = hashers.into_iter().map(HashDigest::from).collect();
// Persist the temporary directory to the directory store.
let id = self
.build_context
.cache()
.persist(temp_dir.into_path(), wheel_entry.path())
.await
.map_err(Error::CacheWrite)?;
// Create an archive.
let archive = Archive::new(id, hashes);
// Write the archive pointer to the cache.
let pointer = LocalArchivePointer {
timestamp: modified.timestamp(),
archive: archive.clone(),
};
pointer.write_to(&pointer_entry).await?;
Ok(LocalWheel {
dist: Dist::Built(dist.clone()),
archive: self.build_context.cache().archive(&archive.id),
hashes: archive.hashes,
filename: filename.clone(),
})
}
}
/// Unzip a wheel into the cache, returning the path to the unzipped directory.
async fn unzip_wheel(&self, path: &Path, target: &Path) -> Result<ArchiveId, Error> {
let temp_dir = tokio::task::spawn_blocking({
let path = path.to_owned();
let root = self.build_context.cache().root().to_path_buf();
move || -> Result<TempDir, uv_extract::Error> {
// Unzip the wheel into a temporary directory.
let temp_dir = tempfile::tempdir_in(root)?;
uv_extract::unzip(fs_err::File::open(path)?, temp_dir.path())?;
Ok(temp_dir)
}
})
.await??;
// Persist the temporary directory to the directory store.
let id = self
.build_context
.cache()
.persist(temp_dir.into_path(), target)
.await
.map_err(Error::CacheWrite)?;
Ok(id)
}
/// Returns a GET [`reqwest::Request`] for the given URL.
fn request(&self, url: Url) -> Result<reqwest::Request, reqwest::Error> {
self.client
.uncached_client()
.get(url)
.header(
// `reqwest` defaults to accepting compressed responses.
// Specify identity encoding to get consistent .whl downloading
// behavior from servers. ref: https://github.com/pypa/pip/pull/1688
"accept-encoding",
reqwest::header::HeaderValue::from_static("identity"),
)
.build()
}
/// Return the [`IndexLocations`] used by this resolver. /// Return the [`IndexLocations`] used by this resolver.
pub fn index_locations(&self) -> &IndexLocations { pub fn index_locations(&self) -> &IndexLocations {
self.build_context.index_locations() self.build_context.index_locations()
} }
} }
/// A pointer to an archive in the cache, fetched from an HTTP archive.
///
/// Encoded with `MsgPack`, and represented on disk by a `.http` file.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct HttpArchivePointer {
archive: Archive,
}
impl HttpArchivePointer {
/// Read an [`HttpArchivePointer`] from the cache.
pub fn read_from(path: impl AsRef<Path>) -> Result<Option<Self>, Error> {
match fs_err::File::open(path.as_ref()) {
Ok(file) => {
let data = DataWithCachePolicy::from_reader(file)?.data;
let archive = rmp_serde::from_slice::<Archive>(&data)?;
Ok(Some(Self { archive }))
}
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(Error::CacheRead(err)),
}
}
/// Return the [`Archive`] from the pointer.
pub fn into_archive(self) -> Archive {
self.archive
}
}
/// A pointer to an archive in the cache, fetched from a local path.
///
/// Encoded with `MsgPack`, and represented on disk by a `.rev` file.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct LocalArchivePointer {
timestamp: Timestamp,
archive: Archive,
}
impl LocalArchivePointer {
/// Read an [`LocalArchivePointer`] from the cache.
pub fn read_from(path: impl AsRef<Path>) -> Result<Option<Self>, Error> {
match fs_err::read(path) {
Ok(cached) => Ok(Some(rmp_serde::from_slice::<LocalArchivePointer>(&cached)?)),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(Error::CacheRead(err)),
}
}
/// Write an [`LocalArchivePointer`] to the cache.
pub async fn write_to(&self, entry: &CacheEntry) -> Result<(), Error> {
write_atomic(entry.path(), rmp_serde::to_vec(&self)?)
.await
.map_err(Error::CacheWrite)
}
/// Returns `true` if the archive is up-to-date with the given modified timestamp.
pub fn is_up_to_date(&self, modified: ArchiveTimestamp) -> bool {
self.timestamp == modified.timestamp()
}
/// Return the [`Archive`] from the pointer.
pub fn into_archive(self) -> Archive {
self.archive
}
}

119
crates/uv-distribution/src/download.rs
View File

@ -1,14 +1,14 @@
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
use distribution_types::{CachedDist, Dist}; use distribution_types::{CachedDist, Dist, Hashed};
use pypi_types::Metadata23; use pypi_types::{HashDigest, Metadata23};
use crate::Error; use crate::Error;
/// A wheel that's been unzipped while downloading /// A locally available wheel.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct UnzippedWheel { pub struct LocalWheel {
/// The remote distribution from which this wheel was downloaded. /// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist, pub(crate) dist: Dist,
/// The parsed filename. /// The parsed filename.
@ -16,115 +16,42 @@ pub struct UnzippedWheel {
/// The canonicalized path in the cache directory to which the wheel was downloaded. /// The canonicalized path in the cache directory to which the wheel was downloaded.
/// Typically, a directory within the archive bucket. /// Typically, a directory within the archive bucket.
pub(crate) archive: PathBuf, pub(crate) archive: PathBuf,
} /// The computed hashes of the wheel.
pub(crate) hashes: Vec<HashDigest>,
/// A downloaded wheel that's stored on-disk.
#[derive(Debug, Clone)]
pub struct DiskWheel {
/// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The path to the downloaded wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
/// Typically, a symlink within the wheels or built wheels bucket.
pub(crate) target: PathBuf,
}
/// A wheel built from a source distribution that's stored on-disk.
#[derive(Debug, Clone)]
pub struct BuiltWheel {
/// The remote source distribution from which this wheel was built.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The path to the built wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
/// Typically, a symlink within the wheels or built wheels bucket.
pub(crate) target: PathBuf,
}
/// A downloaded or built wheel.
#[derive(Debug, Clone)]
pub enum LocalWheel {
Unzipped(UnzippedWheel),
Disk(DiskWheel),
Built(BuiltWheel),
} }
impl LocalWheel { impl LocalWheel {
/// Return the path to the downloaded wheel's entry in the cache. /// Return the path to the downloaded wheel's entry in the cache.
pub fn target(&self) -> &Path { pub fn target(&self) -> &Path {
match self { &self.archive
Self::Unzipped(wheel) => &wheel.archive,
Self::Disk(wheel) => &wheel.target,
Self::Built(wheel) => &wheel.target,
}
} }
/// Return the [`Dist`] from which this wheel was downloaded. /// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist { pub fn remote(&self) -> &Dist {
match self { &self.dist
Self::Unzipped(wheel) => wheel.remote(),
Self::Disk(wheel) => wheel.remote(),
Self::Built(wheel) => wheel.remote(),
}
} }
/// Return the [`WheelFilename`] of this wheel. /// Return the [`WheelFilename`] of this wheel.
pub fn filename(&self) -> &WheelFilename { pub fn filename(&self) -> &WheelFilename {
match self { &self.filename
Self::Unzipped(wheel) => &wheel.filename,
Self::Disk(wheel) => &wheel.filename,
Self::Built(wheel) => &wheel.filename,
}
}
/// Convert a [`LocalWheel`] into a [`CachedDist`].
pub fn into_cached_dist(self, archive: PathBuf) -> CachedDist {
match self {
Self::Unzipped(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
Self::Disk(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
Self::Built(wheel) => CachedDist::from_remote(wheel.dist, wheel.filename, archive),
}
} }
/// Read the [`Metadata23`] from a wheel. /// Read the [`Metadata23`] from a wheel.
pub fn metadata(&self) -> Result<Metadata23, Error> { pub fn metadata(&self) -> Result<Metadata23, Error> {
match self { read_flat_wheel_metadata(&self.filename, &self.archive)
Self::Unzipped(wheel) => read_flat_wheel_metadata(&wheel.filename, &wheel.archive),
Self::Disk(wheel) => read_built_wheel_metadata(&wheel.filename, &wheel.path),
Self::Built(wheel) => read_built_wheel_metadata(&wheel.filename, &wheel.path),
}
} }
} }
impl UnzippedWheel { impl Hashed for LocalWheel {
/// Return the [`Dist`] from which this wheel was downloaded. fn hashes(&self) -> &[HashDigest] {
pub fn remote(&self) -> &Dist { &self.hashes
&self.dist
}
/// Convert an [`UnzippedWheel`] into a [`CachedDist`].
pub fn into_cached_dist(self) -> CachedDist {
CachedDist::from_remote(self.dist, self.filename, self.archive)
} }
} }
impl DiskWheel { /// Convert a [`LocalWheel`] into a [`CachedDist`].
/// Return the [`Dist`] from which this wheel was downloaded. impl From<LocalWheel> for CachedDist {
pub fn remote(&self) -> &Dist { fn from(wheel: LocalWheel) -> CachedDist {
&self.dist CachedDist::from_remote(wheel.dist, wheel.filename, wheel.hashes, wheel.archive)
}
}
impl BuiltWheel {
/// Return the [`Dist`] from which this source distribution that this wheel was built from was
/// downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
} }
} }
@ -134,18 +61,6 @@ impl std::fmt::Display for LocalWheel {
} }
} }
/// Read the [`Metadata23`] from a built wheel.
fn read_built_wheel_metadata(
filename: &WheelFilename,
wheel: impl AsRef<Path>,
) -> Result<Metadata23, Error> {
let file = fs_err::File::open(wheel.as_ref()).map_err(Error::CacheRead)?;
let reader = std::io::BufReader::new(file);
let mut archive = zip::ZipArchive::new(reader)?;
let metadata = install_wheel_rs::metadata::read_archive_metadata(filename, &mut archive)?;
Ok(Metadata23::parse_metadata(&metadata)?)
}
/// Read the [`Metadata23`] from an unzipped wheel. /// Read the [`Metadata23`] from an unzipped wheel.
fn read_flat_wheel_metadata( fn read_flat_wheel_metadata(
filename: &WheelFilename, filename: &WheelFilename,

100
crates/uv-distribution/src/error.rs
View File

@ -3,6 +3,8 @@ use tokio::task::JoinError;
use zip::result::ZipError; use zip::result::ZipError;
use distribution_filename::WheelFilenameError; use distribution_filename::WheelFilenameError;
use pep440_rs::Version;
use pypi_types::HashDigest;
use uv_client::BetterReqwestError; use uv_client::BetterReqwestError;
use uv_normalize::PackageName; use uv_normalize::PackageName;
@ -47,8 +49,10 @@ pub enum Error {
given: PackageName, given: PackageName,
metadata: PackageName, metadata: PackageName,
}, },
#[error("Package metadata version `{metadata}` does not match given version `{given}`")]
VersionMismatch { given: Version, metadata: Version },
#[error("Failed to parse metadata from built wheel")] #[error("Failed to parse metadata from built wheel")]
Metadata(#[from] pypi_types::Error), Metadata(#[from] pypi_types::MetadataError),
#[error("Failed to read `dist-info` metadata from built wheel")] #[error("Failed to read `dist-info` metadata from built wheel")]
DistInfo(#[from] install_wheel_rs::Error), DistInfo(#[from] install_wheel_rs::Error),
#[error("Failed to read zip archive from built wheel")] #[error("Failed to read zip archive from built wheel")]
@ -62,11 +66,11 @@ pub enum Error {
#[error("The source distribution is missing a `PKG-INFO` file")] #[error("The source distribution is missing a `PKG-INFO` file")]
MissingPkgInfo, MissingPkgInfo,
#[error("The source distribution does not support static metadata in `PKG-INFO`")] #[error("The source distribution does not support static metadata in `PKG-INFO`")]
DynamicPkgInfo(#[source] pypi_types::Error), DynamicPkgInfo(#[source] pypi_types::MetadataError),
#[error("The source distribution is missing a `pyproject.toml` file")] #[error("The source distribution is missing a `pyproject.toml` file")]
MissingPyprojectToml, MissingPyprojectToml,
#[error("The source distribution does not support static metadata in `pyproject.toml`")] #[error("The source distribution does not support static metadata in `pyproject.toml`")]
DynamicPyprojectToml(#[source] pypi_types::Error), DynamicPyprojectToml(#[source] pypi_types::MetadataError),
#[error("Unsupported scheme in URL: {0}")] #[error("Unsupported scheme in URL: {0}")]
UnsupportedScheme(String), UnsupportedScheme(String),
@ -78,6 +82,40 @@ pub enum Error {
/// Should not occur; only seen when another task panicked. /// Should not occur; only seen when another task panicked.
#[error("The task executor is broken, did some other task panic?")] #[error("The task executor is broken, did some other task panic?")]
Join(#[from] JoinError), Join(#[from] JoinError),
/// An I/O error that occurs while exhausting a reader to compute a hash.
#[error("Failed to hash distribution")]
HashExhaustion(#[source] std::io::Error),
#[error("Hash mismatch for {distribution}\n\nExpected:\n{expected}\n\nComputed:\n{actual}")]
MismatchedHashes {
distribution: String,
expected: String,
actual: String,
},
#[error(
"Hash-checking is enabled, but no hashes were provided or computed for: {distribution}"
)]
MissingHashes { distribution: String },
#[error("Hash-checking is enabled, but no hashes were computed for: {distribution}\n\nExpected:\n{expected}")]
MissingActualHashes {
distribution: String,
expected: String,
},
#[error("Hash-checking is enabled, but no hashes were provided for: {distribution}\n\nComputed:\n{actual}")]
MissingExpectedHashes {
distribution: String,
actual: String,
},
#[error("Hash-checking is not supported for local directories: {0}")]
HashesNotSupportedSourceTree(String),
#[error("Hash-checking is not supported for Git repositories: {0}")]
HashesNotSupportedGit(String),
} }
impl From<reqwest::Error> for Error { impl From<reqwest::Error> for Error {
@ -96,3 +134,59 @@ impl From<reqwest_middleware::Error> for Error {
} }
} }
} }
impl Error {
/// Construct a hash mismatch error.
pub fn hash_mismatch(
distribution: String,
expected: &[HashDigest],
actual: &[HashDigest],
) -> Error {
match (expected.is_empty(), actual.is_empty()) {
(true, true) => Self::MissingHashes { distribution },
(true, false) => {
let actual = actual
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
Self::MissingExpectedHashes {
distribution,
actual,
}
}
(false, true) => {
let expected = expected
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
Self::MissingActualHashes {
distribution,
expected,
}
}
(false, false) => {
let expected = expected
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
let actual = actual
.iter()
.map(|hash| format!(" {hash}"))
.collect::<Vec<_>>()
.join("\n");
Self::MismatchedHashes {
distribution,
expected,
actual,
}
}
}
}
}

97
crates/uv-distribution/src/index/built_wheel_index.rs
View File

@ -1,52 +1,72 @@
use distribution_types::{git_reference, DirectUrlSourceDist, GitSourceDist, PathSourceDist}; use distribution_types::{
git_reference, DirectUrlSourceDist, GitSourceDist, Hashed, PathSourceDist,
};
use platform_tags::Tags; use platform_tags::Tags;
use uv_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheShard, WheelCache}; use uv_cache::{ArchiveTimestamp, Cache, CacheBucket, CacheShard, WheelCache};
use uv_fs::symlinks; use uv_fs::symlinks;
use uv_types::HashStrategy;
use crate::index::cached_wheel::CachedWheel; use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, read_timestamp_manifest, MANIFEST}; use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
use crate::Error; use crate::Error;
/// A local index of built distributions for a specific source distribution. /// A local index of built distributions for a specific source distribution.
pub struct BuiltWheelIndex; #[derive(Debug)]
pub struct BuiltWheelIndex<'a> {
cache: &'a Cache,
tags: &'a Tags,
hasher: &'a HashStrategy,
}
impl<'a> BuiltWheelIndex<'a> {
/// Initialize an index of built distributions.
pub fn new(cache: &'a Cache, tags: &'a Tags, hasher: &'a HashStrategy) -> Self {
Self {
cache,
tags,
hasher,
}
}
impl BuiltWheelIndex {
/// Return the most compatible [`CachedWheel`] for a given source distribution at a direct URL. /// Return the most compatible [`CachedWheel`] for a given source distribution at a direct URL.
/// ///
/// This method does not perform any freshness checks and assumes that the source distribution /// This method does not perform any freshness checks and assumes that the source distribution
/// is already up-to-date. /// is already up-to-date.
pub fn url( pub fn url(&self, source_dist: &DirectUrlSourceDist) -> Result<Option<CachedWheel>, Error> {
source_dist: &DirectUrlSourceDist,
cache: &Cache,
tags: &Tags,
) -> Result<Option<CachedWheel>, Error> {
// For direct URLs, cache directly under the hash of the URL itself. // For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = cache.shard( let cache_shard = self.cache.shard(
CacheBucket::BuiltWheels, CacheBucket::BuiltWheels,
WheelCache::Url(source_dist.url.raw()).root(), WheelCache::Url(source_dist.url.raw()).root(),
); );
// Read the manifest from the cache. There's no need to enforce freshness, since we // Read the revision from the cache.
// enforce freshness on the entries. let Some(pointer) = HttpRevisionPointer::read_from(cache_shard.entry(HTTP_REVISION))?
let manifest_entry = cache_shard.entry(MANIFEST); else {
let Some(manifest) = read_http_manifest(&manifest_entry)? else {
return Ok(None); return Ok(None);
}; };
Ok(Self::find(&cache_shard.shard(manifest.id()), tags)) // Enforce hash-checking by omitting any wheels that don't satisfy the required hashes.
let revision = pointer.into_revision();
if !revision.satisfies(self.hasher.get(source_dist)) {
return Ok(None);
}
Ok(self.find(&cache_shard.shard(revision.id())))
} }
/// Return the most compatible [`CachedWheel`] for a given source distribution at a local path. /// Return the most compatible [`CachedWheel`] for a given source distribution at a local path.
pub fn path( pub fn path(&self, source_dist: &PathSourceDist) -> Result<Option<CachedWheel>, Error> {
source_dist: &PathSourceDist, let cache_shard = self.cache.shard(
cache: &Cache,
tags: &Tags,
) -> Result<Option<CachedWheel>, Error> {
let cache_shard = cache.shard(
CacheBucket::BuiltWheels, CacheBucket::BuiltWheels,
WheelCache::Path(&source_dist.url).root(), WheelCache::Path(&source_dist.url).root(),
); );
// Read the revision from the cache.
let Some(pointer) = LocalRevisionPointer::read_from(cache_shard.entry(LOCAL_REVISION))?
else {
return Ok(None);
};
// Determine the last-modified time of the source distribution. // Determine the last-modified time of the source distribution.
let Some(modified) = let Some(modified) =
ArchiveTimestamp::from_path(&source_dist.path).map_err(Error::CacheRead)? ArchiveTimestamp::from_path(&source_dist.path).map_err(Error::CacheRead)?
@ -54,28 +74,37 @@ impl BuiltWheelIndex {
return Err(Error::DirWithoutEntrypoint); return Err(Error::DirWithoutEntrypoint);
}; };
// Read the manifest from the cache. There's no need to enforce freshness, since we // If the distribution is stale, omit it from the index.
// enforce freshness on the entries. if !pointer.is_up_to_date(modified) {
let manifest_entry = cache_shard.entry(MANIFEST);
let Some(manifest) = read_timestamp_manifest(&manifest_entry, modified)? else {
return Ok(None); return Ok(None);
}; }
Ok(Self::find(&cache_shard.shard(manifest.id()), tags)) // Enforce hash-checking by omitting any wheels that don't satisfy the required hashes.
let revision = pointer.into_revision();
if !revision.satisfies(self.hasher.get(source_dist)) {
return Ok(None);
}
Ok(self.find(&cache_shard.shard(revision.id())))
} }
/// Return the most compatible [`CachedWheel`] for a given source distribution at a git URL. /// Return the most compatible [`CachedWheel`] for a given source distribution at a git URL.
pub fn git(source_dist: &GitSourceDist, cache: &Cache, tags: &Tags) -> Option<CachedWheel> { pub fn git(&self, source_dist: &GitSourceDist) -> Option<CachedWheel> {
// Enforce hash-checking, which isn't supported for Git distributions.
if self.hasher.get(source_dist).is_validate() {
return None;
}
let Ok(Some(git_sha)) = git_reference(&source_dist.url) else { let Ok(Some(git_sha)) = git_reference(&source_dist.url) else {
return None; return None;
}; };
let cache_shard = cache.shard( let cache_shard = self.cache.shard(
CacheBucket::BuiltWheels, CacheBucket::BuiltWheels,
WheelCache::Git(&source_dist.url, &git_sha.to_short_string()).root(), WheelCache::Git(&source_dist.url, &git_sha.to_short_string()).root(),
); );
Self::find(&cache_shard, tags) self.find(&cache_shard)
} }
/// Find the "best" distribution in the index for a given source distribution. /// Find the "best" distribution in the index for a given source distribution.
@ -94,16 +123,16 @@ impl BuiltWheelIndex {
/// ``` /// ```
/// ///
/// The `shard` should be `built-wheels-v0/pypi/django-allauth-0.51.0.tar.gz`. /// The `shard` should be `built-wheels-v0/pypi/django-allauth-0.51.0.tar.gz`.
fn find(shard: &CacheShard, tags: &Tags) -> Option<CachedWheel> { fn find(&self, shard: &CacheShard) -> Option<CachedWheel> {
let mut candidate: Option<CachedWheel> = None; let mut candidate: Option<CachedWheel> = None;
// Unzipped wheels are stored as symlinks into the archive directory. // Unzipped wheels are stored as symlinks into the archive directory.
for subdir in symlinks(shard) { for subdir in symlinks(shard) {
match CachedWheel::from_path(&subdir) { match CachedWheel::from_built_source(&subdir) {
None => {} None => {}
Some(dist_info) => { Some(dist_info) => {
// Pick the wheel with the highest priority // Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags); let compatibility = dist_info.filename.compatibility(self.tags);
// Only consider wheels that are compatible with our tags. // Only consider wheels that are compatible with our tags.
if !compatibility.is_compatible() { if !compatibility.is_compatible() {
@ -113,7 +142,7 @@ impl BuiltWheelIndex {
if let Some(existing) = candidate.as_ref() { if let Some(existing) = candidate.as_ref() {
// Override if the wheel is newer, or "more" compatible. // Override if the wheel is newer, or "more" compatible.
if dist_info.filename.version > existing.filename.version if dist_info.filename.version > existing.filename.version
|| compatibility > existing.filename.compatibility(tags) || compatibility > existing.filename.compatibility(self.tags)
{ {
candidate = Some(dist_info); candidate = Some(dist_info);
} }

73
crates/uv-distribution/src/index/cached_wheel.rs
View File

@ -1,9 +1,12 @@
use std::path::Path; use std::path::Path;
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
use distribution_types::{CachedDirectUrlDist, CachedRegistryDist}; use distribution_types::{CachedDirectUrlDist, CachedRegistryDist, Hashed};
use pep508_rs::VerbatimUrl; use pep508_rs::VerbatimUrl;
use uv_cache::CacheEntry; use pypi_types::HashDigest;
use uv_cache::{Cache, CacheBucket, CacheEntry};
use crate::{Archive, HttpArchivePointer, LocalArchivePointer};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct CachedWheel { pub struct CachedWheel {
@ -11,16 +14,28 @@ pub struct CachedWheel {
pub filename: WheelFilename, pub filename: WheelFilename,
/// The [`CacheEntry`] for the wheel. /// The [`CacheEntry`] for the wheel.
pub entry: CacheEntry, pub entry: CacheEntry,
/// The [`HashDigest`]s for the wheel.
pub hashes: Vec<HashDigest>,
} }
impl CachedWheel { impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`). /// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_path(path: &Path) -> Option<Self> { pub fn from_built_source(path: impl AsRef<Path>) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?; let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?; let filename = WheelFilename::from_stem(filename).ok()?;
// Convert to a cached wheel.
let archive = path.canonicalize().ok()?; let archive = path.canonicalize().ok()?;
let entry = CacheEntry::from_path(archive); let entry = CacheEntry::from_path(archive);
Some(Self { filename, entry }) let hashes = Vec::new();
Some(Self {
filename,
entry,
hashes,
})
} }
/// Convert a [`CachedWheel`] into a [`CachedRegistryDist`]. /// Convert a [`CachedWheel`] into a [`CachedRegistryDist`].
@ -28,6 +43,7 @@ impl CachedWheel {
CachedRegistryDist { CachedRegistryDist {
filename: self.filename, filename: self.filename,
path: self.entry.into_path_buf(), path: self.entry.into_path_buf(),
hashes: self.hashes,
} }
} }
@ -38,6 +54,55 @@ impl CachedWheel {
url, url,
path: self.entry.into_path_buf(), path: self.entry.into_path_buf(),
editable: false, editable: false,
hashes: self.hashes,
} }
} }
/// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`).
pub fn from_http_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
// Read the pointer.
let pointer = HttpArchivePointer::read_from(path).ok()??;
let Archive { id, hashes } = pointer.into_archive();
// Convert to a cached wheel.
let entry = cache.entry(CacheBucket::Archive, "", id);
Some(Self {
filename,
entry,
hashes,
})
}
/// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`).
pub fn from_local_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();
// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
// Read the pointer.
let pointer = LocalArchivePointer::read_from(path).ok()??;
let Archive { id, hashes } = pointer.into_archive();
// Convert to a cached wheel.
let entry = cache.entry(CacheBucket::Archive, "", id);
Some(Self {
filename,
entry,
hashes,
})
}
}
impl Hashed for CachedWheel {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
} }

120
crates/uv-distribution/src/index/registry_wheel_index.rs
View File

@ -1,19 +1,19 @@
use std::collections::hash_map::Entry; use std::collections::hash_map::Entry;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::path::Path;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use distribution_types::{CachedRegistryDist, FlatIndexLocation, IndexLocations, IndexUrl}; use distribution_types::{CachedRegistryDist, FlatIndexLocation, Hashed, IndexLocations, IndexUrl};
use pep440_rs::Version; use pep440_rs::Version;
use pep508_rs::VerbatimUrl; use pep508_rs::VerbatimUrl;
use platform_tags::Tags; use platform_tags::Tags;
use uv_cache::{Cache, CacheBucket, WheelCache}; use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_fs::{directories, symlinks}; use uv_fs::{directories, files, symlinks};
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_types::HashStrategy;
use crate::index::cached_wheel::CachedWheel; use crate::index::cached_wheel::CachedWheel;
use crate::source::{read_http_manifest, MANIFEST}; use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
/// A local index of distributions that originate from a registry, like `PyPI`. /// A local index of distributions that originate from a registry, like `PyPI`.
#[derive(Debug)] #[derive(Debug)]
@ -21,16 +21,23 @@ pub struct RegistryWheelIndex<'a> {
cache: &'a Cache, cache: &'a Cache,
tags: &'a Tags, tags: &'a Tags,
index_locations: &'a IndexLocations, index_locations: &'a IndexLocations,
hasher: &'a HashStrategy,
index: FxHashMap<&'a PackageName, BTreeMap<Version, CachedRegistryDist>>, index: FxHashMap<&'a PackageName, BTreeMap<Version, CachedRegistryDist>>,
} }
impl<'a> RegistryWheelIndex<'a> { impl<'a> RegistryWheelIndex<'a> {
/// Initialize an index of cached distributions from a directory. /// Initialize an index of registry distributions.
pub fn new(cache: &'a Cache, tags: &'a Tags, index_locations: &'a IndexLocations) -> Self { pub fn new(
cache: &'a Cache,
tags: &'a Tags,
index_locations: &'a IndexLocations,
hasher: &'a HashStrategy,
) -> Self {
Self { Self {
cache, cache,
tags, tags,
index_locations, index_locations,
hasher,
index: FxHashMap::default(), index: FxHashMap::default(),
} }
} }
@ -65,6 +72,7 @@ impl<'a> RegistryWheelIndex<'a> {
self.cache, self.cache,
self.tags, self.tags,
self.index_locations, self.index_locations,
self.hasher,
)), )),
}; };
versions versions
@ -76,14 +84,18 @@ impl<'a> RegistryWheelIndex<'a> {
cache: &Cache, cache: &Cache,
tags: &Tags, tags: &Tags,
index_locations: &IndexLocations, index_locations: &IndexLocations,
hasher: &HashStrategy,
) -> BTreeMap<Version, CachedRegistryDist> { ) -> BTreeMap<Version, CachedRegistryDist> {
let mut versions = BTreeMap::new(); let mut versions = BTreeMap::new();
// Collect into owned `IndexUrl` // Collect into owned `IndexUrl`.
let flat_index_urls: Vec<IndexUrl> = index_locations let flat_index_urls: Vec<IndexUrl> = index_locations
.flat_index() .flat_index()
.filter_map(|flat_index| match flat_index { .filter_map(|flat_index| match flat_index {
FlatIndexLocation::Path(_) => None, FlatIndexLocation::Path(path) => {
let path = fs_err::canonicalize(path).ok()?;
Some(IndexUrl::Path(VerbatimUrl::from_path(path)))
}
FlatIndexLocation::Url(url) => { FlatIndexLocation::Url(url) => {
Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone()))) Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone())))
} }
@ -97,7 +109,44 @@ impl<'a> RegistryWheelIndex<'a> {
WheelCache::Index(index_url).wheel_dir(package.to_string()), WheelCache::Index(index_url).wheel_dir(package.to_string()),
); );
Self::add_directory(&wheel_dir, tags, &mut versions); // For registry wheels, the cache structure is: `<index>/<package-name>/<wheel>.http`
// or `<index>/<package-name>/<version>/<wheel>.rev`.
for file in files(&wheel_dir) {
match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("http"))
{
if let Some(wheel) =
CachedWheel::from_http_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get_package(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("rev"))
{
if let Some(wheel) =
CachedWheel::from_local_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get_package(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
}
}
// Index all the built wheels, created by downloading and building source distributions // Index all the built wheels, created by downloading and building source distributions
// from the registry. // from the registry.
@ -110,30 +159,52 @@ impl<'a> RegistryWheelIndex<'a> {
for shard in directories(&cache_shard) { for shard in directories(&cache_shard) {
// Read the existing metadata from the cache, if it exists. // Read the existing metadata from the cache, if it exists.
let cache_shard = cache_shard.shard(shard); let cache_shard = cache_shard.shard(shard);
let manifest_entry = cache_shard.entry(MANIFEST);
if let Ok(Some(manifest)) = read_http_manifest(&manifest_entry) { // Read the revision from the cache.
Self::add_directory(cache_shard.join(manifest.id()), tags, &mut versions); let revision = match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
let revision_entry = cache_shard.entry(LOCAL_REVISION);
if let Ok(Some(pointer)) = LocalRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
}; };
if let Some(revision) = revision {
// Enforce hash-checking based on the source distribution.
if revision.satisfies(hasher.get_package(package)) {
for wheel_dir in symlinks(cache_shard.join(revision.id())) {
if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
} }
} }
versions versions
} }
/// Add the wheels in a given directory to the index. /// Add the [`CachedWheel`] to the index.
/// fn add_wheel(
/// Each subdirectory in the given path is expected to be that of an unzipped wheel. wheel: CachedWheel,
fn add_directory(
path: impl AsRef<Path>,
tags: &Tags, tags: &Tags,
versions: &mut BTreeMap<Version, CachedRegistryDist>, versions: &mut BTreeMap<Version, CachedRegistryDist>,
) { ) {
// Unzipped wheels are stored as symlinks into the archive directory. let dist_info = wheel.into_registry_dist();
for wheel_dir in symlinks(path.as_ref()) {
match CachedWheel::from_path(&wheel_dir) {
None => {}
Some(dist_info) => {
let dist_info = dist_info.into_registry_dist();
// Pick the wheel with the highest priority // Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags); let compatibility = dist_info.filename.compatibility(tags);
@ -146,7 +217,4 @@ impl<'a> RegistryWheelIndex<'a> {
versions.insert(dist_info.filename.version.clone(), dist_info); versions.insert(dist_info.filename.version.clone(), dist_info);
} }
} }
}
}
}
} }

27
crates/uv-distribution/src/lib.rs
View File

@ -1,12 +1,14 @@
pub use distribution_database::DistributionDatabase; pub use archive::Archive;
pub use download::{BuiltWheel, DiskWheel, LocalWheel}; pub use distribution_database::{DistributionDatabase, HttpArchivePointer, LocalArchivePointer};
pub use download::LocalWheel;
pub use error::Error; pub use error::Error;
pub use git::{is_same_reference, to_precise}; pub use git::{is_same_reference, to_precise};
pub use index::{BuiltWheelIndex, RegistryWheelIndex}; pub use index::{BuiltWheelIndex, RegistryWheelIndex};
use pypi_types::{HashDigest, Metadata23};
pub use reporter::Reporter; pub use reporter::Reporter;
pub use source::SourceDistributionBuilder; pub use source::SourceDistributionBuilder;
pub use unzip::Unzip;
mod archive;
mod distribution_database; mod distribution_database;
mod download; mod download;
mod error; mod error;
@ -15,4 +17,21 @@ mod index;
mod locks; mod locks;
mod reporter; mod reporter;
mod source; mod source;
mod unzip;
/// The metadata associated with an archive.
#[derive(Debug, Clone)]
pub struct ArchiveMetadata {
/// The [`Metadata23`] for the underlying distribution.
pub metadata: Metadata23,
/// The hashes of the source or built archive.
pub hashes: Vec<HashDigest>,
}
impl From<Metadata23> for ArchiveMetadata {
fn from(metadata: Metadata23) -> Self {
Self {
metadata,
hashes: vec![],
}
}
}

22
crates/uv-distribution/src/source/built_wheel_metadata.rs
View File

@ -2,23 +2,27 @@ use std::path::PathBuf;
use std::str::FromStr; use std::str::FromStr;
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
use distribution_types::Hashed;
use platform_tags::Tags; use platform_tags::Tags;
use pypi_types::HashDigest;
use uv_cache::CacheShard; use uv_cache::CacheShard;
use uv_fs::files; use uv_fs::files;
/// The information about the wheel we either just built or got from the cache. /// The information about the wheel we either just built or got from the cache.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct BuiltWheelMetadata { pub(crate) struct BuiltWheelMetadata {
/// The path to the built wheel. /// The path to the built wheel.
pub(crate) path: PathBuf, pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache. /// The expected path to the downloaded wheel's entry in the cache.
pub(crate) target: PathBuf, pub(crate) target: PathBuf,
/// The parsed filename. /// The parsed filename.
pub(crate) filename: WheelFilename, pub(crate) filename: WheelFilename,
/// The computed hashes of the source distribution from which the wheel was built.
pub(crate) hashes: Vec<HashDigest>,
} }
impl BuiltWheelMetadata { impl BuiltWheelMetadata {
/// Find a compatible wheel in the cache based on the given manifest. /// Find a compatible wheel in the cache.
pub(crate) fn find_in_cache(tags: &Tags, cache_shard: &CacheShard) -> Option<Self> { pub(crate) fn find_in_cache(tags: &Tags, cache_shard: &CacheShard) -> Option<Self> {
for directory in files(cache_shard) { for directory in files(cache_shard) {
if let Some(metadata) = Self::from_path(directory, cache_shard) { if let Some(metadata) = Self::from_path(directory, cache_shard) {
@ -39,6 +43,20 @@ impl BuiltWheelMetadata {
target: cache_shard.join(filename.stem()), target: cache_shard.join(filename.stem()),
path, path,
filename, filename,
hashes: vec![],
}) })
} }
/// Set the computed hashes of the wheel.
#[must_use]
pub(crate) fn with_hashes(mut self, hashes: Vec<HashDigest>) -> Self {
self.hashes = hashes;
self
}
}
impl Hashed for BuiltWheelMetadata {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
} }

17
crates/uv-distribution/src/source/manifest.rs
View File

@ -1,17 +0,0 @@
use serde::{Deserialize, Serialize};
/// The [`Manifest`] is a thin wrapper around a unique identifier for the source distribution.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct Manifest(String);
impl Manifest {
/// Initialize a new [`Manifest`] with a random UUID.
pub(crate) fn new() -> Self {
Self(nanoid::nanoid!())
}
/// Return the unique ID of the manifest.
pub(crate) fn id(&self) -> &str {
&self.0
}
}

1302
crates/uv-distribution/src/source/mod.rs
View File

File diff suppressed because it is too large Load Diff

72
crates/uv-distribution/src/source/revision.rs Normal file
View File

@ -0,0 +1,72 @@
use distribution_types::Hashed;
use serde::{Deserialize, Serialize};
use std::path::Path;
use pypi_types::HashDigest;
/// The [`Revision`] is a thin wrapper around a unique identifier for the source distribution.
///
/// A revision represents a unique version of a source distribution, at a level more granular than
/// (e.g.) the version number of the distribution itself. For example, a source distribution hosted
/// at a URL or a local file path may have multiple revisions, each representing a unique state of
/// the distribution, despite the reported version number remaining the same.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct Revision {
id: RevisionId,
hashes: Vec<HashDigest>,
}
impl Revision {
/// Initialize a new [`Revision`] with a random UUID.
pub(crate) fn new() -> Self {
Self {
id: RevisionId::new(),
hashes: vec![],
}
}
/// Return the unique ID of the manifest.
pub(crate) fn id(&self) -> &RevisionId {
&self.id
}
/// Return the computed hashes of the archive.
pub(crate) fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
/// Return the computed hashes of the archive.
pub(crate) fn into_hashes(self) -> Vec<HashDigest> {
self.hashes
}
/// Set the computed hashes of the archive.
#[must_use]
pub(crate) fn with_hashes(mut self, hashes: Vec<HashDigest>) -> Self {
self.hashes = hashes;
self
}
}
impl Hashed for Revision {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}
/// A unique identifier for a revision of a source distribution.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct RevisionId(String);
impl RevisionId {
/// Generate a new unique identifier for an archive.
fn new() -> Self {
Self(nanoid::nanoid!())
}
}
impl AsRef<Path> for RevisionId {
fn as_ref(&self) -> &Path {
self.0.as_ref()
}
}

36
crates/uv-distribution/src/unzip.rs
View File

@ -1,36 +0,0 @@
use std::path::Path;
use tracing::instrument;
use uv_extract::Error;
use crate::download::BuiltWheel;
use crate::{DiskWheel, LocalWheel};
pub trait Unzip {
/// Unzip a wheel into the target directory.
fn unzip(&self, target: &Path) -> Result<(), Error>;
}
impl Unzip for DiskWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
uv_extract::unzip(fs_err::File::open(&self.path)?, target)
}
}
impl Unzip for BuiltWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
uv_extract::unzip(fs_err::File::open(&self.path)?, target)
}
}
impl Unzip for LocalWheel {
#[instrument(skip_all, fields(filename=self.filename().to_string()))]
fn unzip(&self, target: &Path) -> Result<(), Error> {
match self {
Self::Unzipped(_) => Ok(()),
Self::Disk(wheel) => wheel.unzip(target),
Self::Built(wheel) => wheel.unzip(target),
}
}
}

6
crates/uv-extract/Cargo.toml
View File

@ -13,12 +13,16 @@ license = { workspace = true }
workspace = true workspace = true
[dependencies] [dependencies]
async-compression = { workspace = true, features = ["gzip"] } pypi-types = { workspace = true }
async-compression = { workspace = true, features = ["gzip", "zstd"] }
async_zip = { workspace = true, features = ["tokio"] } async_zip = { workspace = true, features = ["tokio"] }
fs-err = { workspace = true, features = ["tokio"] } fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true } futures = { workspace = true }
md-5.workspace = true
rayon = { workspace = true } rayon = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
tokio = { workspace = true, features = ["io-util"] } tokio = { workspace = true, features = ["io-util"] }
tokio-tar = { workspace = true } tokio-tar = { workspace = true }

146
crates/uv-extract/src/hash.rs Normal file
View File

@ -0,0 +1,146 @@
use std::pin::Pin;
use std::task::{Context, Poll};
use sha2::Digest;
use tokio::io::{AsyncReadExt, ReadBuf};
use pypi_types::{HashAlgorithm, HashDigest};
pub struct Sha256Reader<'a, R> {
reader: R,
hasher: &'a mut sha2::Sha256,
}
impl<'a, R> Sha256Reader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
pub fn new(reader: R, hasher: &'a mut sha2::Sha256) -> Self {
Sha256Reader { reader, hasher }
}
}
impl<'a, R> tokio::io::AsyncRead for Sha256Reader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let reader = Pin::new(&mut self.reader);
match reader.poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
self.hasher.update(buf.filled());
Poll::Ready(Ok(()))
}
other => other,
}
}
}
#[derive(Debug)]
pub enum Hasher {
Md5(md5::Md5),
Sha256(sha2::Sha256),
Sha384(sha2::Sha384),
Sha512(sha2::Sha512),
}
impl Hasher {
pub fn update(&mut self, data: &[u8]) {
match self {
Hasher::Md5(hasher) => hasher.update(data),
Hasher::Sha256(hasher) => hasher.update(data),
Hasher::Sha384(hasher) => hasher.update(data),
Hasher::Sha512(hasher) => hasher.update(data),
}
}
pub fn finalize(self) -> Vec<u8> {
match self {
Hasher::Md5(hasher) => hasher.finalize().to_vec(),
Hasher::Sha256(hasher) => hasher.finalize().to_vec(),
Hasher::Sha384(hasher) => hasher.finalize().to_vec(),
Hasher::Sha512(hasher) => hasher.finalize().to_vec(),
}
}
}
impl From<HashAlgorithm> for Hasher {
fn from(algorithm: HashAlgorithm) -> Self {
match algorithm {
HashAlgorithm::Md5 => Hasher::Md5(md5::Md5::new()),
HashAlgorithm::Sha256 => Hasher::Sha256(sha2::Sha256::new()),
HashAlgorithm::Sha384 => Hasher::Sha384(sha2::Sha384::new()),
HashAlgorithm::Sha512 => Hasher::Sha512(sha2::Sha512::new()),
}
}
}
impl From<Hasher> for HashDigest {
fn from(hasher: Hasher) -> Self {
match hasher {
Hasher::Md5(hasher) => HashDigest {
algorithm: HashAlgorithm::Md5,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
Hasher::Sha256(hasher) => HashDigest {
algorithm: HashAlgorithm::Sha256,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
Hasher::Sha384(hasher) => HashDigest {
algorithm: HashAlgorithm::Sha384,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
Hasher::Sha512(hasher) => HashDigest {
algorithm: HashAlgorithm::Sha512,
digest: format!("{:x}", hasher.finalize()).into_boxed_str(),
},
}
}
}
pub struct HashReader<'a, R> {
reader: R,
hashers: &'a mut [Hasher],
}
impl<'a, R> HashReader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
pub fn new(reader: R, hashers: &'a mut [Hasher]) -> Self {
HashReader { reader, hashers }
}
/// Exhaust the underlying reader.
pub async fn finish(&mut self) -> Result<(), std::io::Error> {
while self.read(&mut vec![0; 8192]).await? > 0 {}
Ok(())
}
}
impl<'a, R> tokio::io::AsyncRead for HashReader<'a, R>
where
R: tokio::io::AsyncRead + Unpin,
{
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let reader = Pin::new(&mut self.reader);
match reader.poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
for hasher in self.hashers.iter_mut() {
hasher.update(buf.filled());
}
Poll::Ready(Ok(()))
}
other => other,
}
}
}

Some files were not shown because too many files have changed in this diff Show More