diff --git a/Cargo.lock b/Cargo.lock index a2296dc10..35be07837 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,6 +203,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-recursion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "async-trait" version = "0.1.77" @@ -2785,6 +2796,7 @@ version = "0.0.1" dependencies = [ "anyhow", "assert_fs", + "async-recursion", "fs-err", "indoc", "insta", @@ -2793,14 +2805,19 @@ dependencies = [ "pep440_rs", "pep508_rs", "regex", + "reqwest", + "reqwest-middleware", "serde", "serde_json", "tempfile", "test-case", "thiserror", + "tokio", "tracing", "unscanny", "url", + "uv-cache", + "uv-client", "uv-fs", "uv-normalize", "uv-warnings", diff --git a/Cargo.toml b/Cargo.toml index 1d27788e7..e5fbf49de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crates/*"] exclude = [ "scripts", # Needs nightly - "crates/uv-trampoline" + "crates/uv-trampoline", ] resolver = "2" @@ -22,6 +22,7 @@ anyhow = { version = "1.0.80" } async-compression = { version = "0.4.6" } async-channel = { version = "2.2.0" } async-trait = { version = "0.1.77" } +async-recursion = { version = "1.0.5" } async_http_range_reader = { version = "0.7.0" } async_zip = { git = "https://github.com/charliermarsh/rs-async-zip", rev = "d76801da0943de985254fc6255c0e476b57c5836", features = ["deflate"] } base64 = { version = "0.21.7" } diff --git a/crates/requirements-txt/Cargo.toml b/crates/requirements-txt/Cargo.toml index 0a3f24c2a..6ee3d3098 100644 --- a/crates/requirements-txt/Cargo.toml +++ b/crates/requirements-txt/Cargo.toml @@ -15,15 +15,21 @@ workspace = true [dependencies] pep440_rs = { path = "../pep440-rs", features = ["rkyv", "serde"] } pep508_rs = { path = "../pep508-rs", features = ["rkyv", "serde", "non-pep508-extensions"] } +uv-cache = { path = "../uv-cache" } +uv-client = { path = "../uv-client" } uv-fs = { path = "../uv-fs" } uv-normalize = { path = "../uv-normalize" } uv-warnings = { path = "../uv-warnings" } +async-recursion = { workspace = true } fs-err = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } unscanny = { workspace = true } url = { workspace = true } diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs index bd4cf7efe..c14911c01 100644 --- a/crates/requirements-txt/src/lib.rs +++ b/crates/requirements-txt/src/lib.rs @@ -40,11 +40,13 @@ use std::io; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; -use tracing::{instrument, warn}; +use tracing::instrument; use unscanny::{Pattern, Scanner}; use url::Url; +use uv_client::RegistryClient; use uv_warnings::warn_user; +use async_recursion::async_recursion; use pep508_rs::{ expand_path_vars, split_scheme, Extras, Pep508Error, Pep508ErrorSource, Requirement, Scheme, VerbatimUrl, @@ -321,28 +323,38 @@ pub struct RequirementsTxt { impl RequirementsTxt { /// See module level documentation #[instrument(skip_all, fields(requirements_txt = requirements_txt.as_ref().as_os_str().to_str()))] - pub fn parse( + pub async fn parse( requirements_txt: impl AsRef, working_dir: impl AsRef, + client: &RegistryClient, ) -> Result { + let requirements_txt = requirements_txt.as_ref(); + let working_dir = working_dir.as_ref(); + let content = - uv_fs::read_to_string(&requirements_txt).map_err(|err| RequirementsTxtFileError { - file: requirements_txt.as_ref().to_path_buf(), - error: RequirementsTxtParserError::IO(err), + if requirements_txt.starts_with("http://") | requirements_txt.starts_with("https://") { + read_url_to_string(&requirements_txt, client).await + } else { + uv_fs::read_to_string(&requirements_txt) + .await + .map_err(RequirementsTxtParserError::IO) + } + .map_err(|err| RequirementsTxtFileError { + file: requirements_txt.to_path_buf(), + error: err, })?; - let working_dir = working_dir.as_ref(); - let requirements_dir = requirements_txt.as_ref().parent().unwrap_or(working_dir); - let data = Self::parse_inner(&content, working_dir, requirements_dir).map_err(|err| { - RequirementsTxtFileError { - file: requirements_txt.as_ref().to_path_buf(), + let requirements_dir = requirements_txt.parent().unwrap_or(working_dir); + let data = Self::parse_inner(&content, working_dir, requirements_dir, client) + .await + .map_err(|err| RequirementsTxtFileError { + file: requirements_txt.to_path_buf(), error: err, - } - })?; + })?; if data == Self::default() { warn_user!( "Requirements file {} does not contain any dependencies", - requirements_txt.as_ref().display() + requirements_txt.simplified_display() ); } @@ -355,10 +367,12 @@ impl RequirementsTxt { /// the current working directory. However, relative paths to sub-files (e.g., `-r ../requirements.txt`) /// are resolved against the directory of the containing `requirements.txt` file, to match /// `pip`'s behavior. - pub fn parse_inner( + #[async_recursion] + pub async fn parse_inner( content: &str, working_dir: &Path, requirements_dir: &Path, + client: &RegistryClient, ) -> Result { let mut s = Scanner::new(content); @@ -370,14 +384,20 @@ impl RequirementsTxt { start, end, } => { - let sub_file = requirements_dir.join(expand_path_vars(&filename).as_ref()); - let sub_requirements = Self::parse(&sub_file, working_dir).map_err(|err| { - RequirementsTxtParserError::Subfile { + let filename = expand_path_vars(&filename); + let sub_file = + if filename.starts_with("http://") || filename.starts_with("https://") { + PathBuf::from(filename.as_ref()) + } else { + requirements_dir.join(filename.as_ref()) + }; + let sub_requirements = Self::parse(&sub_file, working_dir, client) + .await + .map_err(|err| RequirementsTxtParserError::Subfile { source: Box::new(err), start, end, - } - })?; + })?; // Disallow conflicting `--index-url` in nested `requirements` files. if sub_requirements.index_url.is_some() @@ -402,14 +422,20 @@ impl RequirementsTxt { start, end, } => { - let sub_file = requirements_dir.join(expand_path_vars(&filename).as_ref()); - let sub_constraints = Self::parse(&sub_file, working_dir).map_err(|err| { - RequirementsTxtParserError::Subfile { + let filename = expand_path_vars(&filename); + let sub_file = + if filename.starts_with("http://") || filename.starts_with("https://") { + PathBuf::from(filename.as_ref()) + } else { + requirements_dir.join(filename.as_ref()) + }; + let sub_constraints = Self::parse(&sub_file, working_dir, client) + .await + .map_err(|err| RequirementsTxtParserError::Subfile { source: Box::new(err), start, end, - } - })?; + })?; // Treat any nested requirements or constraints as constraints. This differs // from `pip`, which seems to treat `-r` requirements in constraints files as // _requirements_, but we don't want to support that. @@ -767,6 +793,31 @@ fn parse_value<'a, T>( } } +/// Fetch the contents of a URL and return them as a string. +async fn read_url_to_string( + path: impl AsRef, + client: &RegistryClient, +) -> Result { + // pip would URL-encode the non-UTF-8 bytes of the string; we just don't support them. + let path_utf8 = + path.as_ref() + .to_str() + .ok_or_else(|| RequirementsTxtParserError::NonUnicodeUrl { + url: path.as_ref().to_owned(), + })?; + Ok(client + .cached_client() + .uncached() + .get(path_utf8) + .send() + .await? + .error_for_status() + .map_err(reqwest_middleware::Error::Reqwest)? + .text() + .await + .map_err(reqwest_middleware::Error::Reqwest)?) +} + /// Error parsing requirements.txt, wrapper with filename #[derive(Debug)] pub struct RequirementsTxtFileError { @@ -808,6 +859,10 @@ pub enum RequirementsTxtParserError { start: usize, end: usize, }, + Reqwest(reqwest_middleware::Error), + NonUnicodeUrl { + url: PathBuf, + }, } impl RequirementsTxtParserError { @@ -855,6 +910,8 @@ impl RequirementsTxtParserError { start: start + offset, end: end + offset, }, + Self::Reqwest(err) => Self::Reqwest(err), + Self::NonUnicodeUrl { url } => Self::NonUnicodeUrl { url }, } } } @@ -897,6 +954,16 @@ impl Display for RequirementsTxtParserError { Self::Subfile { start, .. } => { write!(f, "Error parsing included file at position {start}") } + Self::Reqwest(err) => { + write!(f, "Error while accessing remote requirements file {err}") + } + Self::NonUnicodeUrl { url } => { + write!( + f, + "Remote requirements URL contains non-unicode characters: {}", + url.display(), + ) + } } } } @@ -914,6 +981,8 @@ impl std::error::Error for RequirementsTxtParserError { Self::Pep508 { source, .. } => Some(source), Self::Subfile { source, .. } => Some(source.as_ref()), Self::Parser { .. } => None, + Self::Reqwest(err) => err.source(), + Self::NonUnicodeUrl { .. } => None, } } } @@ -989,6 +1058,21 @@ impl Display for RequirementsTxtFileError { self.file.simplified_display(), ) } + RequirementsTxtParserError::Reqwest(err) => { + write!( + f, + "Error while accessing remote requirements file {}: {err}", + self.file.simplified_display(), + ) + } + + RequirementsTxtParserError::NonUnicodeUrl { url } => { + write!( + f, + "Remote requirements URL contains non-unicode characters: {}", + url.display(), + ) + } } } } @@ -1005,6 +1089,12 @@ impl From for RequirementsTxtParserError { } } +impl From for RequirementsTxtParserError { + fn from(err: reqwest_middleware::Error) -> Self { + Self::Reqwest(err) + } +} + /// Calculates the column and line offset of a given cursor based on the /// number of Unicode codepoints. fn calculate_row_column(content: &str, position: usize) -> (usize, usize) { @@ -1057,6 +1147,7 @@ mod test { use tempfile::tempdir; use test_case::test_case; use unscanny::Scanner; + use uv_client::{RegistryClient, RegistryClientBuilder}; use uv_fs::Simplified; use crate::{calculate_row_column, EditableRequirement, RequirementsTxt}; @@ -1065,6 +1156,12 @@ mod test { PathBuf::from("./test-data") } + fn registry_client() -> RegistryClient { + RegistryClientBuilder::new(uv_cache::Cache::temp().unwrap()) + .connectivity(uv_client::Connectivity::Online) + .build() + } + #[test_case(Path::new("basic.txt"))] #[test_case(Path::new("constraints-a.txt"))] #[test_case(Path::new("constraints-b.txt"))] @@ -1075,11 +1172,14 @@ mod test { #[test_case(Path::new("poetry-with-hashes.txt"))] #[test_case(Path::new("small.txt"))] #[test_case(Path::new("whitespace.txt"))] - fn parse(path: &Path) { + #[tokio::test] + async fn parse(path: &Path) { let working_dir = workspace_test_data_dir().join("requirements-txt"); let requirements_txt = working_dir.join(path); - let actual = RequirementsTxt::parse(requirements_txt, &working_dir).unwrap(); + let actual = RequirementsTxt::parse(requirements_txt, &working_dir, ®istry_client()) + .await + .unwrap(); let snapshot = format!("parse-{}", path.to_string_lossy()); insta::assert_debug_snapshot!(snapshot, actual); @@ -1096,7 +1196,8 @@ mod test { #[test_case(Path::new("small.txt"))] #[test_case(Path::new("whitespace.txt"))] #[test_case(Path::new("editable.txt"))] - fn line_endings(path: &Path) { + #[tokio::test] + async fn line_endings(path: &Path) { let working_dir = workspace_test_data_dir().join("requirements-txt"); let requirements_txt = working_dir.join(path); @@ -1120,14 +1221,16 @@ mod test { let requirements_txt = temp_dir.path().join(path); fs::write(&requirements_txt, contents).unwrap(); - let actual = RequirementsTxt::parse(&requirements_txt, &working_dir).unwrap(); + let actual = RequirementsTxt::parse(&requirements_txt, &working_dir, ®istry_client()) + .await + .unwrap(); let snapshot = format!("line-endings-{}", path.to_string_lossy()); insta::assert_debug_snapshot!(snapshot, actual); } - #[test] - fn invalid_include_missing_file() -> Result<()> { + #[tokio::test] + async fn invalid_include_missing_file() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let missing_txt = temp_dir.child("missing.txt"); let requirements_txt = temp_dir.child("requirements.txt"); @@ -1135,7 +1238,10 @@ mod test { -r missing.txt "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error) .chain() // The last error is operating-system specific. @@ -1155,22 +1261,25 @@ mod test { }, { insta::assert_snapshot!(errors, @r###" Error parsing included file in `` at position 0 - failed to open file `` + failed to read from file `` "###); }); Ok(()) } - #[test] - fn invalid_requirement() -> Result<()> { + #[tokio::test] + async fn invalid_requirement() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {" numpy[ö]==1.29 "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = @@ -1193,15 +1302,18 @@ mod test { Ok(()) } - #[test] - fn unsupported_editable() -> Result<()> { + #[tokio::test] + async fn unsupported_editable() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {" -e http://localhost:8080/ "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = @@ -1219,15 +1331,18 @@ mod test { Ok(()) } - #[test] - fn invalid_editable_extra() -> Result<()> { + #[tokio::test] + async fn invalid_editable_extra() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {" -e black[,abcdef] "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = @@ -1247,15 +1362,18 @@ mod test { Ok(()) } - #[test] - fn invalid_index_url() -> Result<()> { + #[tokio::test] + async fn invalid_index_url() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {" --index-url 123 "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = @@ -1276,8 +1394,8 @@ mod test { Ok(()) } - #[test] - fn missing_r() -> Result<()> { + #[tokio::test] + async fn missing_r() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let file_txt = temp_dir.child("file.txt"); @@ -1289,7 +1407,10 @@ mod test { file.txt "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = @@ -1307,8 +1428,8 @@ mod test { Ok(()) } - #[test] - fn relative_requirement() -> Result<()> { + #[tokio::test] + async fn relative_requirement() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; // Create a requirements file with a relative entry, in a subdirectory. @@ -1330,7 +1451,10 @@ mod test { -r subdir/child.txt "})?; - let requirements = RequirementsTxt::parse(parent_txt.path(), temp_dir.path()).unwrap(); + let requirements = + RequirementsTxt::parse(parent_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap(); insta::assert_debug_snapshot!(requirements, @r###" RequirementsTxt { requirements: [ @@ -1359,9 +1483,9 @@ mod test { Ok(()) } - #[test] + #[tokio::test] #[cfg(not(windows))] - fn nested_editable() -> Result<()> { + async fn nested_editable() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); @@ -1381,7 +1505,9 @@ mod test { "})?; let requirements = - RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap(); + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap(); insta::assert_debug_snapshot!(requirements, @r###" RequirementsTxt { @@ -1419,8 +1545,8 @@ mod test { Ok(()) } - #[test] - fn nested_conflicting_index_url() -> Result<()> { + #[tokio::test] + async fn nested_conflicting_index_url() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); @@ -1439,7 +1565,10 @@ mod test { --index-url https://fake.pypi.org/simple "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = @@ -1477,8 +1606,8 @@ mod test { ); } - #[test] - fn parser_error_line_and_column() -> Result<()> { + #[tokio::test] + async fn parser_error_line_and_column() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let requirements_txt = temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {" @@ -1487,7 +1616,10 @@ mod test { tqdm "})?; - let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let error = + RequirementsTxt::parse(requirements_txt.path(), temp_dir.path(), ®istry_client()) + .await + .unwrap_err(); let errors = anyhow::Error::new(error).chain().join("\n"); let requirement_txt = diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 7ae8c95b0..5adc72e51 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -177,6 +177,12 @@ impl RegistryClient { self.timeout } + /// Set the index URLs to use for fetching packages. + #[must_use] + pub fn with_index_url(self, index_urls: IndexUrls) -> Self { + Self { index_urls, ..self } + } + /// Fetch a package from the `PyPI` simple API. /// /// "simple" here refers to [PEP 503 – Simple Repository API](https://peps.python.org/pep-0503/) diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index 11db64995..3598c419b 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -1,4 +1,5 @@ use std::fmt::Display; +use std::io::Read; use std::path::{Path, PathBuf}; use fs2::FileExt; @@ -12,35 +13,31 @@ pub use crate::path::*; mod path; -/// Reads the contents of the file path given into memory. +/// Reads the contents of the file path into memory. /// /// If the file path is `-`, then contents are read from stdin instead. -pub fn read(path: impl AsRef) -> std::io::Result> { - use std::io::Read; - +pub async fn read(path: impl AsRef) -> std::io::Result> { let path = path.as_ref(); if path == Path::new("-") { let mut buf = Vec::with_capacity(1024); std::io::stdin().read_to_end(&mut buf)?; Ok(buf) } else { - fs::read(path) + fs_err::tokio::read(path).await } } -/// Reads the contents of the file path given into memory as a `String`. +/// Reads the contents of the file path into memory as a `String`. /// /// If the file path is `-`, then contents are read from stdin instead. -pub fn read_to_string(path: impl AsRef) -> std::io::Result { - use std::io::Read; - +pub async fn read_to_string(path: impl AsRef) -> std::io::Result { let path = path.as_ref(); if path == Path::new("-") { let mut buf = String::with_capacity(1024); std::io::stdin().read_to_string(&mut buf)?; Ok(buf) } else { - fs::read_to_string(path) + fs_err::tokio::read_to_string(path).await } } diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 5ba3b6db6..68a1e1ed1 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -9,6 +9,7 @@ use std::str::FromStr; use anstream::{eprint, AutoStream, StripStream}; use anyhow::{anyhow, Context, Result}; use chrono::{DateTime, Utc}; +use futures::future::OptionFuture; use itertools::Itertools; use owo_colors::OwoColorize; use rustc_hash::FxHashSet; @@ -82,6 +83,11 @@ pub(crate) async fn pip_compile( )); } + // Initialize the registry client. + let client = RegistryClientBuilder::new(cache.clone()) + .connectivity(connectivity) + .build(); + // Read all requirements from the provided sources. let RequirementsSpecification { project, @@ -94,11 +100,14 @@ pub(crate) async fn pip_compile( no_index, find_links, extras: used_extras, - } = RequirementsSpecification::from_sources(requirements, constraints, overrides, &extras)?; - - // Incorporate any index locations from the provided sources. - let index_locations = - index_locations.combine(index_url, extra_index_urls, find_links, no_index); + } = RequirementsSpecification::from_sources( + requirements, + constraints, + overrides, + &extras, + &client, + ) + .await?; // Check that all provided extras are used if let ExtrasSpecification::Some(extras) = extras { @@ -117,28 +126,33 @@ pub(crate) async fn pip_compile( } } - let preferences: Vec = output_file - // As an optimization, skip reading the lockfile is we're upgrading all packages anyway. - .filter(|_| !upgrade.is_all()) - .filter(|output_file| output_file.exists()) - .map(Path::to_path_buf) - .map(RequirementsSource::from_path) - .as_ref() - .map(|source| RequirementsSpecification::from_source(source, &extras)) - .transpose()? - .map(|spec| spec.requirements) - .map(|requirements| match upgrade { - // Respect all pinned versions from the existing lockfile. - Upgrade::None => requirements, - // Ignore all pinned versions from the existing lockfile. - Upgrade::All => vec![], - // Ignore pinned versions for the specified packages. - Upgrade::Packages(packages) => requirements - .into_iter() - .filter(|requirement| !packages.contains(&requirement.name)) - .collect(), - }) - .unwrap_or_default(); + let preferences: Vec = OptionFuture::from( + output_file + // As an optimization, skip reading the lockfile is we're upgrading all packages anyway. + .filter(|_| !upgrade.is_all()) + .filter(|output_file| output_file.exists()) + .map(Path::to_path_buf) + .map(RequirementsSource::from_path) + .as_ref() + .map(|source| async { + RequirementsSpecification::from_source(source, &extras, &client).await + }), + ) + .await + .transpose()? + .map(|spec| spec.requirements) + .map(|requirements| match upgrade { + // Respect all pinned versions from the existing lockfile. + Upgrade::None => requirements, + // Ignore all pinned versions from the existing lockfile. + Upgrade::All => vec![], + // Ignore pinned versions for the specified packages. + Upgrade::Packages(packages) => requirements + .into_iter() + .filter(|requirement| !packages.contains(&requirement.name)) + .collect(), + }) + .unwrap_or_default(); // Find an interpreter to use for building distributions let platform = Platform::current()?; @@ -196,11 +210,13 @@ pub(crate) async fn pip_compile( |python_version| Cow::Owned(python_version.markers(interpreter.markers())), ); - // Instantiate a client. - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_locations.index_urls()) - .connectivity(connectivity) - .build(); + // Incorporate any index locations from the provided sources. + let index_locations = + index_locations.combine(index_url, extra_index_urls, find_links, no_index); + + // Update the index URLs on the client, to take into account any index URLs added by the + // sources (e.g., `--index-url` in a `requirements.txt` file). + let client = client.with_index_url(index_locations.index_urls()); // Resolve the flat indexes from `--find-links`. let flat_index = { diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 53b2c57ee..ec44703d0 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -70,6 +70,11 @@ pub(crate) async fn pip_install( ) -> Result { let start = std::time::Instant::now(); + // Initialize the registry client. + let client = RegistryClientBuilder::new(cache.clone()) + .connectivity(connectivity) + .build(); + // Read all requirements from the provided sources. let RequirementsSpecification { project, @@ -82,11 +87,7 @@ pub(crate) async fn pip_install( no_index, find_links, extras: used_extras, - } = specification(requirements, constraints, overrides, extras)?; - - // Incorporate any index locations from the provided sources. - let index_locations = - index_locations.combine(index_url, extra_index_urls, find_links, no_index); + } = specification(requirements, constraints, overrides, extras, &client).await?; // Check that all provided extras are used if let ExtrasSpecification::Some(extras) = extras { @@ -169,11 +170,13 @@ pub(crate) async fn pip_install( let tags = venv.interpreter().tags()?; let markers = venv.interpreter().markers(); - // Instantiate a client. - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_locations.index_urls()) - .connectivity(connectivity) - .build(); + // Incorporate any index locations from the provided sources. + let index_locations = + index_locations.combine(index_url, extra_index_urls, find_links, no_index); + + // Update the index URLs on the client, to take into account any index URLs added by the + // sources (e.g., `--index-url` in a `requirements.txt` file). + let client = client.with_index_url(index_locations.index_urls()); // Resolve the flat indexes from `--find-links`. let flat_index = { @@ -316,11 +319,12 @@ pub(crate) async fn pip_install( } /// Consolidate the requirements for an installation. -fn specification( +async fn specification( requirements: &[RequirementsSource], constraints: &[RequirementsSource], overrides: &[RequirementsSource], extras: &ExtrasSpecification<'_>, + client: &RegistryClient, ) -> Result { // If the user requests `extras` but does not provide a pyproject toml source if !matches!(extras, ExtrasSpecification::None) @@ -332,8 +336,14 @@ fn specification( } // Read all requirements from the provided sources. - let spec = - RequirementsSpecification::from_sources(requirements, constraints, overrides, extras)?; + let spec = RequirementsSpecification::from_sources( + requirements, + constraints, + overrides, + extras, + client, + ) + .await?; // Check that all provided extras are used if let ExtrasSpecification::Some(extras) = extras { diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index 565869328..a61551537 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -48,6 +48,11 @@ pub(crate) async fn pip_sync( ) -> Result { let start = std::time::Instant::now(); + // Initialize the registry client. + let client = RegistryClientBuilder::new(cache.clone()) + .connectivity(connectivity) + .build(); + // Read all requirements from the provided sources. let RequirementsSpecification { project: _project, @@ -60,7 +65,7 @@ pub(crate) async fn pip_sync( no_index, find_links, extras: _extras, - } = RequirementsSpecification::from_simple_sources(sources)?; + } = RequirementsSpecification::from_simple_sources(sources, &client).await?; let num_requirements = requirements.len() + editables.len(); if num_requirements == 0 { @@ -68,10 +73,6 @@ pub(crate) async fn pip_sync( return Ok(ExitStatus::Success); } - // Incorporate any index locations from the provided sources. - let index_locations = - index_locations.combine(index_url, extra_index_urls, find_links, no_index); - // Detect the current Python interpreter. let platform = Platform::current()?; let venv = if let Some(python) = python.as_ref() { @@ -108,11 +109,13 @@ pub(crate) async fn pip_sync( // Determine the current environment markers. let tags = venv.interpreter().tags()?; - // Prep the registry client. - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_locations.index_urls()) - .connectivity(connectivity) - .build(); + // Incorporate any index locations from the provided sources. + let index_locations = + index_locations.combine(index_url, extra_index_urls, find_links, no_index); + + // Update the index URLs on the client, to take into account any index URLs added by the + // sources (e.g., `--index-url` in a `requirements.txt` file). + let client = client.with_index_url(index_locations.index_urls()); // Resolve the flat indexes from `--find-links`. let flat_index = { diff --git a/crates/uv/src/commands/pip_uninstall.rs b/crates/uv/src/commands/pip_uninstall.rs index 63715c06e..007915b56 100644 --- a/crates/uv/src/commands/pip_uninstall.rs +++ b/crates/uv/src/commands/pip_uninstall.rs @@ -7,6 +7,7 @@ use tracing::debug; use distribution_types::{InstalledMetadata, Name}; use platform_host::Platform; use uv_cache::Cache; +use uv_client::{Connectivity, RegistryClientBuilder}; use uv_fs::Simplified; use uv_interpreter::PythonEnvironment; @@ -20,10 +21,16 @@ pub(crate) async fn pip_uninstall( python: Option, system: bool, cache: Cache, + connectivity: Connectivity, printer: Printer, ) -> Result { let start = std::time::Instant::now(); + // Initialize the registry client. + let client: uv_client::RegistryClient = RegistryClientBuilder::new(cache.clone()) + .connectivity(connectivity) + .build(); + // Read all requirements from the provided sources. let RequirementsSpecification { project: _project, @@ -36,7 +43,7 @@ pub(crate) async fn pip_uninstall( no_index: _no_index, find_links: _find_links, extras: _extras, - } = RequirementsSpecification::from_simple_sources(sources)?; + } = RequirementsSpecification::from_simple_sources(sources, &client).await?; // Detect the current Python interpreter. let platform = Platform::current()?; diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index 92e0c1084..d0330d137 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -861,6 +861,10 @@ struct PipUninstallArgs { /// should be used with caution, as it can modify the system Python installation. #[clap(long, conflicts_with = "python")] system: bool, + + /// Run offline, i.e., without accessing the network. + #[arg(global = true, long)] + offline: bool, } #[derive(Args)] @@ -1455,7 +1459,19 @@ async fn run() -> Result { .map(RequirementsSource::from_path), ) .collect::>(); - commands::pip_uninstall(&sources, args.python, args.system, cache, printer).await + commands::pip_uninstall( + &sources, + args.python, + args.system, + cache, + if args.offline { + Connectivity::Offline + } else { + Connectivity::Online + }, + printer, + ) + .await } Commands::Pip(PipNamespace { command: PipCommand::Freeze(args), diff --git a/crates/uv/src/requirements.rs b/crates/uv/src/requirements.rs index f7d9fb0c9..31db38181 100644 --- a/crates/uv/src/requirements.rs +++ b/crates/uv/src/requirements.rs @@ -12,6 +12,7 @@ use distribution_types::{FlatIndexLocation, IndexUrl}; use pep508_rs::Requirement; use requirements_txt::{EditableRequirement, FindLink, RequirementsTxt}; use tracing::{instrument, Level}; +use uv_client::RegistryClient; use uv_fs::Simplified; use uv_normalize::{ExtraName, PackageName}; @@ -138,9 +139,10 @@ pub(crate) struct RequirementsSpecification { impl RequirementsSpecification { /// Read the requirements and constraints from a source. #[instrument(skip_all, level = Level::DEBUG, fields(source = % source))] - pub(crate) fn from_source( + pub(crate) async fn from_source( source: &RequirementsSource, - extras: &ExtrasSpecification, + extras: &ExtrasSpecification<'_>, + client: &RegistryClient, ) -> Result { Ok(match source { RequirementsSource::Package(name) => { @@ -176,7 +178,8 @@ impl RequirementsSpecification { } } RequirementsSource::RequirementsTxt(path) => { - let requirements_txt = RequirementsTxt::parse(path, std::env::current_dir()?)?; + let requirements_txt = + RequirementsTxt::parse(path, std::env::current_dir()?, client).await?; Self { project: None, requirements: requirements_txt @@ -206,7 +209,7 @@ impl RequirementsSpecification { } } RequirementsSource::PyprojectToml(path) => { - let contents = uv_fs::read_to_string(path)?; + let contents = uv_fs::read_to_string(path).await?; let pyproject_toml = toml::from_str::(&contents) .with_context(|| format!("Failed to parse `{}`", path.simplified_display()))?; let mut used_extras = FxHashSet::default(); @@ -273,11 +276,12 @@ impl RequirementsSpecification { } /// Read the combined requirements and constraints from a set of sources. - pub(crate) fn from_sources( + pub(crate) async fn from_sources( requirements: &[RequirementsSource], constraints: &[RequirementsSource], overrides: &[RequirementsSource], - extras: &ExtrasSpecification, + extras: &ExtrasSpecification<'_>, + client: &RegistryClient, ) -> Result { let mut spec = Self::default(); @@ -285,7 +289,7 @@ impl RequirementsSpecification { // A `requirements.txt` can contain a `-c constraints.txt` directive within it, so reading // a requirements file can also add constraints. for source in requirements { - let source = Self::from_source(source, extras)?; + let source = Self::from_source(source, extras, client).await?; spec.requirements.extend(source.requirements); spec.constraints.extend(source.constraints); spec.overrides.extend(source.overrides); @@ -312,7 +316,7 @@ impl RequirementsSpecification { // Read all constraints, treating _everything_ as a constraint. for source in constraints { - let source = Self::from_source(source, extras)?; + let source = Self::from_source(source, extras, client).await?; spec.constraints.extend(source.requirements); spec.constraints.extend(source.constraints); spec.constraints.extend(source.overrides); @@ -332,7 +336,7 @@ impl RequirementsSpecification { // Read all overrides, treating both requirements _and_ constraints as overrides. for source in overrides { - let source = Self::from_source(source, extras)?; + let source = Self::from_source(source, extras, client).await?; spec.overrides.extend(source.requirements); spec.overrides.extend(source.constraints); spec.overrides.extend(source.overrides); @@ -354,8 +358,11 @@ impl RequirementsSpecification { } /// Read the requirements from a set of sources. - pub(crate) fn from_simple_sources(requirements: &[RequirementsSource]) -> Result { - Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None) + pub(crate) async fn from_simple_sources( + requirements: &[RequirementsSource], + client: &RegistryClient, + ) -> Result { + Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None, client).await } } diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index d3d295de4..1eddd1e77 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -115,7 +115,7 @@ fn missing_requirements_in() { ----- stdout ----- ----- stderr ----- - error: failed to open file `requirements.in` + error: failed to read from file `requirements.in` Caused by: No such file or directory (os error 2) "### ); @@ -142,7 +142,7 @@ fn missing_venv() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: failed to open file `requirements.in` + error: failed to read from file `requirements.in` Caused by: No such file or directory (os error 2) "### ); diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs index 9bd24d3c1..527e7cda1 100644 --- a/crates/uv/tests/pip_install.rs +++ b/crates/uv/tests/pip_install.rs @@ -100,7 +100,7 @@ fn missing_requirements_txt() { ----- stdout ----- ----- stderr ----- - error: failed to open file `requirements.txt` + error: failed to read from file `requirements.txt` Caused by: No such file or directory (os error 2) "### ); @@ -1509,6 +1509,73 @@ fn install_constraints_inline() -> Result<()> { Ok(()) } +/// Install a package from a `constraints.txt` file on a remote http server. +#[test] +fn install_constraints_remote() { + let context = TestContext::new("3.12"); + + uv_snapshot!(command(&context) + .arg("-c") + .arg("https://raw.githubusercontent.com/apache/airflow/constraints-2-6/constraints-3.11.txt") + .arg("typing_extensions>=4.0"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + typing-extensions==4.7.1 + "### + ); // would yield typing-extensions==4.8.2 without constraint file +} + +/// Install a package from a `requirements.txt` file, with an inline constraint, which points +/// to a remote http server. +#[test] +fn install_constraints_inline_remote() -> Result<()> { + let context = TestContext::new("3.12"); + let requirementstxt = context.temp_dir.child("requirements.txt"); + requirementstxt.write_str("typing-extensions>=4.0\n-c https://raw.githubusercontent.com/apache/airflow/constraints-2-6/constraints-3.11.txt")?; + + uv_snapshot!(command(&context) + .arg("-r") + .arg("requirements.txt"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + typing-extensions==4.7.1 + "### // would yield typing-extensions==4.8.2 without constraint file + ); + + Ok(()) +} + +#[test] +fn install_constraints_respects_offline_mode() { + let context = TestContext::new("3.12"); + + uv_snapshot!(command(&context) + .arg("--offline") + .arg("-r") + .arg("http://example.com/requirements.txt"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Error while accessing remote requirements file http://example.com/requirements.txt: Middleware error: Network connectivity is disabled, but the requested data wasn't found in the cache for: `http://example.com/requirements.txt` + Caused by: Network connectivity is disabled, but the requested data wasn't found in the cache for: `http://example.com/requirements.txt` + "### + ); +} + /// Tests that we can install `polars==0.14.0`, which has this odd dependency /// requirement in its wheel metadata: `pyarrow>=4.0.*; extra == 'pyarrow'`. /// diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index 338a98eb5..b10c6bf8a 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -124,7 +124,7 @@ fn missing_requirements_txt() { ----- stdout ----- ----- stderr ----- - error: failed to open file `requirements.txt` + error: failed to read from file `requirements.txt` Caused by: No such file or directory (os error 2) "###); @@ -151,7 +151,7 @@ fn missing_venv() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: failed to open file `requirements.txt` + error: failed to read from file `requirements.txt` Caused by: No such file or directory (os error 2) "###); diff --git a/crates/uv/tests/pip_uninstall.rs b/crates/uv/tests/pip_uninstall.rs index 18ee8d7cf..3bb9eda52 100644 --- a/crates/uv/tests/pip_uninstall.rs +++ b/crates/uv/tests/pip_uninstall.rs @@ -75,7 +75,7 @@ fn missing_requirements_txt() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: failed to open file `requirements.txt` + error: failed to read from file `requirements.txt` Caused by: No such file or directory (os error 2) "### ); @@ -125,7 +125,7 @@ fn missing_pyproject_toml() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: failed to open file `pyproject.toml` + error: failed to read from file `pyproject.toml` Caused by: No such file or directory (os error 2) "### );