diff --git a/Cargo.lock b/Cargo.lock index 4d78cad6a..f414c6f82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2137,6 +2137,7 @@ dependencies = [ "log", "once_cell", "pep440_rs 0.3.12", + "puffin-fs", "puffin-normalize", "pyo3", "pyo3-log", diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs index 8b9f7e4b9..fa7786e15 100644 --- a/crates/distribution-types/src/index_url.rs +++ b/crates/distribution-types/src/index_url.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use pep508_rs::split_scheme; +use puffin_fs::normalize_url_path; static PYPI_URL: Lazy = Lazy::new(|| Url::parse("https://pypi.org/simple").unwrap()); @@ -89,7 +90,11 @@ impl FromStr for FlatIndexLocation { if scheme == "file" { // Ex) `file:///home/ferris/project/scripts/...` or `file:../ferris/` let path = path.strip_prefix("//").unwrap_or(path); - let path = PathBuf::from(path); + + // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`. + let path = normalize_url_path(path); + + let path = PathBuf::from(path.as_ref()); Ok(Self::Path(path)) } else { // Ex) `https://download.pytorch.org/whl/torch_stable.html` @@ -162,6 +167,35 @@ impl IndexLocations { } } } + + /// Combine a set of index locations. + /// + /// If either the current or the other index locations have `no_index` set, the result will + /// have `no_index` set. + /// + /// If the current index location has an `index` set, it will be preserved. + #[must_use] + pub fn combine( + self, + index: Option, + extra_index: Vec, + flat_index: Vec, + no_index: bool, + ) -> Self { + if no_index { + Self { + index: None, + extra_index: Vec::new(), + flat_index, + } + } else { + Self { + index: self.index.or(index), + extra_index: self.extra_index.into_iter().chain(extra_index).collect(), + flat_index: self.flat_index.into_iter().chain(flat_index).collect(), + } + } + } } impl<'a> IndexLocations { diff --git a/crates/pep508-rs/Cargo.toml b/crates/pep508-rs/Cargo.toml index 92fdc9b14..080c403f4 100644 --- a/crates/pep508-rs/Cargo.toml +++ b/crates/pep508-rs/Cargo.toml @@ -18,6 +18,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] pep440_rs = { path = "../pep440-rs" } +puffin-fs = { path = "../puffin-fs" } puffin-normalize = { path = "../puffin-normalize" } derivative = { workspace = true } diff --git a/crates/pep508-rs/src/lib.rs b/crates/pep508-rs/src/lib.rs index fd6937a1a..d24148dc2 100644 --- a/crates/pep508-rs/src/lib.rs +++ b/crates/pep508-rs/src/lib.rs @@ -16,7 +16,6 @@ #![deny(missing_docs)] -use std::borrow::Cow; #[cfg(feature = "pyo3")] use std::collections::hash_map::DefaultHasher; use std::collections::HashSet; @@ -43,6 +42,7 @@ pub use marker::{ MarkerValueString, MarkerValueVersion, MarkerWarningKind, StringVersion, }; use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; +use puffin_fs::normalize_url_path; #[cfg(feature = "pyo3")] use puffin_normalize::InvalidNameError; use puffin_normalize::{ExtraName, PackageName}; @@ -745,47 +745,26 @@ fn preprocess_url( ) -> Result { let url = if let Some((scheme, path)) = split_scheme(url) { if scheme == "file" { - if let Some(path) = path.strip_prefix("//") { - let path = if cfg!(windows) { - // Transform `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz` - Cow::Owned( - path.strip_prefix('/') - .unwrap_or(path) - .replace('/', std::path::MAIN_SEPARATOR_STR), - ) - } else { - Cow::Borrowed(path) - }; - // Ex) `file:///home/ferris/project/scripts/...` - if let Some(working_dir) = working_dir { - VerbatimUrl::from_path(path, working_dir).with_given(url.to_string()) - } else { - VerbatimUrl::from_absolute_path(path) - .map_err(|err| Pep508Error { - message: Pep508ErrorSource::UrlError(err), - start, - len, - input: cursor.to_string(), - })? - .with_given(url.to_string()) - } + // Ex) `file:///home/ferris/project/scripts/...` or `file:../editable/`. + let path = path.strip_prefix("//").unwrap_or(path); + + // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`. + let path = normalize_url_path(path); + + if let Some(working_dir) = working_dir { + VerbatimUrl::from_path(path, working_dir).with_given(url.to_string()) } else { - // Ex) `file:../editable/` - if let Some(working_dir) = working_dir { - VerbatimUrl::from_path(path, working_dir).with_given(url.to_string()) - } else { - VerbatimUrl::from_absolute_path(path) - .map_err(|err| Pep508Error { - message: Pep508ErrorSource::UrlError(err), - start, - len, - input: cursor.to_string(), - })? - .with_given(url.to_string()) - } + VerbatimUrl::from_absolute_path(path) + .map_err(|err| Pep508Error { + message: Pep508ErrorSource::UrlError(err), + start, + len, + input: cursor.to_string(), + })? + .with_given(url.to_string()) } } else { - // Ex) `https://...` + // Ex) `https://download.pytorch.org/whl/torch_stable.html` VerbatimUrl::from_str(url).map_err(|err| Pep508Error { message: Pep508ErrorSource::UrlError(err), start, diff --git a/crates/puffin-fs/src/path.rs b/crates/puffin-fs/src/path.rs index 840b902a7..d3e9a89a3 100644 --- a/crates/puffin-fs/src/path.rs +++ b/crates/puffin-fs/src/path.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::path::Path; pub trait NormalizedDisplay { @@ -13,3 +14,53 @@ impl> NormalizedDisplay for T { dunce::simplified(self.as_ref()).display() } } + +/// Normalize the `path` component of a URL for use as a file path. +/// +/// For example, on Windows, transforms `C:\Users\ferris\wheel-0.42.0.tar.gz` to +/// `/C:/Users/ferris/wheel-0.42.0.tar.gz`. +/// +/// On other platforms, this is a no-op. +pub fn normalize_url_path(path: &str) -> Cow<'_, str> { + if cfg!(windows) { + Cow::Owned( + path.strip_prefix('/') + .unwrap_or(path) + .replace('/', std::path::MAIN_SEPARATOR_STR), + ) + } else { + Cow::Borrowed(path) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize() { + if cfg!(windows) { + assert_eq!( + normalize_url_path("/C:/Users/ferris/wheel-0.42.0.tar.gz"), + "C:\\Users\\ferris\\wheel-0.42.0.tar.gz" + ); + } else { + assert_eq!( + normalize_url_path("/C:/Users/ferris/wheel-0.42.0.tar.gz"), + "/C:/Users/ferris/wheel-0.42.0.tar.gz" + ); + } + + if cfg!(windows) { + assert_eq!( + normalize_url_path("./ferris/wheel-0.42.0.tar.gz"), + ".\\ferris\\wheel-0.42.0.tar.gz" + ); + } else { + assert_eq!( + normalize_url_path("./ferris/wheel-0.42.0.tar.gz"), + "./ferris/wheel-0.42.0.tar.gz" + ); + } + } +} diff --git a/crates/puffin/src/commands/pip_compile.rs b/crates/puffin/src/commands/pip_compile.rs index 83fa73881..69eddc02b 100644 --- a/crates/puffin/src/commands/pip_compile.rs +++ b/crates/puffin/src/commands/pip_compile.rs @@ -85,9 +85,17 @@ pub(crate) async fn pip_compile( constraints, overrides, editables, + index_url, + extra_index_urls, + no_index, + find_links, extras: used_extras, } = RequirementsSpecification::from_sources(requirements, constraints, overrides, &extras)?; + // Incorporate any index locations from the provided sources. + let index_locations = + index_locations.combine(index_url, extra_index_urls, find_links, no_index); + // Check that all provided extras are used if let ExtrasSpecification::Some(extras) = extras { let mut unused_extras = extras diff --git a/crates/puffin/src/commands/pip_install.rs b/crates/puffin/src/commands/pip_install.rs index ae2336183..5d180485a 100644 --- a/crates/puffin/src/commands/pip_install.rs +++ b/crates/puffin/src/commands/pip_install.rs @@ -66,9 +66,17 @@ pub(crate) async fn pip_install( constraints, overrides, editables, + index_url, + extra_index_urls, + no_index, + find_links, extras: used_extras, } = specification(requirements, constraints, overrides, extras)?; + // Incorporate any index locations from the provided sources. + let index_locations = + index_locations.combine(index_url, extra_index_urls, find_links, no_index); + // Check that all provided extras are used if let ExtrasSpecification::Some(extras) = extras { let mut unused_extras = extras diff --git a/crates/puffin/src/commands/pip_sync.rs b/crates/puffin/src/commands/pip_sync.rs index c70bc46ce..e5143e761 100644 --- a/crates/puffin/src/commands/pip_sync.rs +++ b/crates/puffin/src/commands/pip_sync.rs @@ -44,13 +44,29 @@ pub(crate) async fn pip_sync( let start = std::time::Instant::now(); // Read all requirements from the provided sources. - let (requirements, editables) = RequirementsSpecification::requirements_and_editables(sources)?; + let RequirementsSpecification { + project: _project, + requirements, + constraints: _constraints, + overrides: _overrides, + editables, + index_url, + extra_index_urls, + no_index, + find_links, + extras: _extras, + } = RequirementsSpecification::from_simple_sources(sources)?; + let num_requirements = requirements.len() + editables.len(); if num_requirements == 0 { writeln!(printer, "No requirements found")?; return Ok(ExitStatus::Success); } + // Incorporate any index locations from the provided sources. + let index_locations = + index_locations.combine(index_url, extra_index_urls, find_links, no_index); + // Detect the current Python interpreter. let platform = Platform::current()?; let venv = Virtualenv::from_env(platform, &cache)?; diff --git a/crates/puffin/src/commands/pip_uninstall.rs b/crates/puffin/src/commands/pip_uninstall.rs index ba1c42c88..6db4d9ae6 100644 --- a/crates/puffin/src/commands/pip_uninstall.rs +++ b/crates/puffin/src/commands/pip_uninstall.rs @@ -23,7 +23,18 @@ pub(crate) async fn pip_uninstall( let start = std::time::Instant::now(); // Read all requirements from the provided sources. - let (requirements, editables) = RequirementsSpecification::requirements_and_editables(sources)?; + let RequirementsSpecification { + project: _project, + requirements, + constraints: _constraints, + overrides: _overrides, + editables, + index_url: _index_url, + extra_index_urls: _extra_index_urls, + no_index: _no_index, + find_links: _find_links, + extras: _extras, + } = RequirementsSpecification::from_simple_sources(sources)?; // Detect the current Python interpreter. let platform = Platform::current()?; diff --git a/crates/puffin/src/requirements.rs b/crates/puffin/src/requirements.rs index 69507c845..79b2009c7 100644 --- a/crates/puffin/src/requirements.rs +++ b/crates/puffin/src/requirements.rs @@ -7,10 +7,11 @@ use anyhow::{Context, Result}; use fs_err as fs; use rustc_hash::FxHashSet; +use distribution_types::{FlatIndexLocation, IndexUrl}; use pep508_rs::Requirement; use puffin_fs::NormalizedDisplay; use puffin_normalize::{ExtraName, PackageName}; -use requirements_txt::{EditableRequirement, RequirementsTxt}; +use requirements_txt::{EditableRequirement, FindLink, RequirementsTxt}; #[derive(Debug)] pub(crate) enum RequirementsSource { @@ -67,6 +68,14 @@ pub(crate) struct RequirementsSpecification { pub(crate) editables: Vec, /// The extras used to collect requirements. pub(crate) extras: FxHashSet, + /// The index URL to use for fetching packages. + pub(crate) index_url: Option, + /// The extra index URLs to use for fetching packages. + pub(crate) extra_index_urls: Vec, + /// Whether to disallow index usage. + pub(crate) no_index: bool, + /// The `--find-links` locations to use for fetching packages. + pub(crate) find_links: Vec, } impl RequirementsSpecification { @@ -86,6 +95,10 @@ impl RequirementsSpecification { overrides: vec![], editables: vec![], extras: FxHashSet::default(), + index_url: None, + extra_index_urls: vec![], + no_index: false, + find_links: vec![], } } RequirementsSource::Editable(name) => { @@ -98,6 +111,10 @@ impl RequirementsSpecification { overrides: vec![], editables: vec![requirement], extras: FxHashSet::default(), + index_url: None, + extra_index_urls: vec![], + no_index: false, + find_links: vec![], } } RequirementsSource::RequirementsTxt(path) => { @@ -113,6 +130,21 @@ impl RequirementsSpecification { editables: requirements_txt.editables, overrides: vec![], extras: FxHashSet::default(), + index_url: requirements_txt.index_url.map(IndexUrl::from), + extra_index_urls: requirements_txt + .extra_index_urls + .into_iter() + .map(IndexUrl::from) + .collect(), + no_index: requirements_txt.no_index, + find_links: requirements_txt + .find_links + .into_iter() + .map(|link| match link { + FindLink::Url(url) => FlatIndexLocation::Url(url), + FindLink::Path(path) => FlatIndexLocation::Path(path), + }) + .collect(), } } RequirementsSource::PyprojectToml(path) => { @@ -151,6 +183,10 @@ impl RequirementsSpecification { overrides: vec![], editables: vec![], extras: used_extras, + index_url: None, + extra_index_urls: vec![], + no_index: false, + find_links: vec![], } } }) @@ -176,10 +212,22 @@ impl RequirementsSpecification { spec.extras.extend(source.extras); spec.editables.extend(source.editables); - // Use the first project name discovered + // Use the first project name discovered. if spec.project.is_none() { spec.project = source.project; } + + if let Some(url) = source.index_url { + if let Some(existing) = spec.index_url { + return Err(anyhow::anyhow!( + "Multiple index URLs specified: `{existing}` vs.` {url}", + )); + } + spec.index_url = Some(url); + } + spec.no_index |= source.no_index; + spec.extra_index_urls.extend(source.extra_index_urls); + spec.find_links.extend(source.find_links); } // Read all constraints, treating _everything_ as a constraint. @@ -188,6 +236,18 @@ impl RequirementsSpecification { spec.constraints.extend(source.requirements); spec.constraints.extend(source.constraints); spec.constraints.extend(source.overrides); + + if let Some(url) = source.index_url { + if let Some(existing) = spec.index_url { + return Err(anyhow::anyhow!( + "Multiple index URLs specified: `{existing}` vs.` {url}", + )); + } + spec.index_url = Some(url); + } + spec.no_index |= source.no_index; + spec.extra_index_urls.extend(source.extra_index_urls); + spec.find_links.extend(source.find_links); } // Read all overrides, treating both requirements _and_ constraints as overrides. @@ -196,16 +256,25 @@ impl RequirementsSpecification { spec.overrides.extend(source.requirements); spec.overrides.extend(source.constraints); spec.overrides.extend(source.overrides); + + if let Some(url) = source.index_url { + if let Some(existing) = spec.index_url { + return Err(anyhow::anyhow!( + "Multiple index URLs specified: `{existing}` vs.` {url}", + )); + } + spec.index_url = Some(url); + } + spec.no_index |= source.no_index; + spec.extra_index_urls.extend(source.extra_index_urls); + spec.find_links.extend(source.find_links); } Ok(spec) } /// Read the requirements from a set of sources. - pub(crate) fn requirements_and_editables( - requirements: &[RequirementsSource], - ) -> Result<(Vec, Vec)> { - let specification = Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None)?; - Ok((specification.requirements, specification.editables)) + pub(crate) fn from_simple_sources(requirements: &[RequirementsSource]) -> Result { + Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None) } } diff --git a/crates/puffin/tests/pip_compile.rs b/crates/puffin/tests/pip_compile.rs index 75bbd2f74..96151be82 100644 --- a/crates/puffin/tests/pip_compile.rs +++ b/crates/puffin/tests/pip_compile.rs @@ -3350,6 +3350,49 @@ fn find_links_url() -> Result<()> { Ok(()) } +/// Compile using `--find-links` with a URL by resolving `tqdm` from the `PyTorch` wheels index, +/// with the URL itself provided in a `requirements.txt` file. +#[test] +fn find_links_requirements_txt() -> Result<()> { + let temp_dir = TempDir::new()?; + let cache_dir = TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + let requirements_in = temp_dir.child("requirements.in"); + requirements_in.write_str("-f https://download.pytorch.org/whl/torch_stable.html\ntqdm")?; + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("compile") + .arg("requirements.in") + .arg("--no-index") + .arg("--emit-find-links") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by Puffin v[VERSION] via the following command: + # puffin pip compile requirements.in --no-index --emit-find-links --cache-dir [CACHE_DIR] + --find-links https://download.pytorch.org/whl/torch_stable.html + + tqdm==4.64.1 + + ----- stderr ----- + Resolved 1 package in [TIME] + "###); + }); + + Ok(()) +} + /// Use an existing resolution for `black==23.10.1`, with stale versions of `click` and `pathspec`. /// Nothing should change. #[test] @@ -3689,7 +3732,7 @@ fn missing_package_name() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: Unsupported requirement in requirements.in position 0 to 135 + error: Unsupported requirement in requirements.in at position 0 Caused by: URL requirement must be preceded by a package name. Add the name of the package before the URL (e.g., `package_name @ https://...`). https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3974,3 +4017,119 @@ fn emit_find_links() -> Result<()> { Ok(()) } + +/// Respect the `--no-index` flag in a `requirements.txt` file. +#[test] +fn no_index_requirements_txt() -> Result<()> { + let temp_dir = TempDir::new()?; + let cache_dir = TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + let requirements_in = temp_dir.child("requirements.in"); + requirements_in.write_str("--no-index\ntqdm")?; + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("compile") + .arg("requirements.in") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: tqdm isn't available locally, but making network requests to registries was banned. + "###); + }); + + Ok(()) +} + +/// Prefer the `--index-url` from the command line over the `--index-url` in a `requirements.txt` +/// file. +#[test] +fn index_url_requirements_txt() -> Result<()> { + let temp_dir = TempDir::new()?; + let cache_dir = TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + let requirements_in = temp_dir.child("requirements.in"); + requirements_in.write_str("--index-url https://google.com\ntqdm")?; + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("compile") + .arg("requirements.in") + .arg("--index-url") + .arg("https://pypi.org/simple") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by Puffin v[VERSION] via the following command: + # puffin pip compile requirements.in --index-url https://pypi.org/simple --cache-dir [CACHE_DIR] + tqdm==4.66.1 + + ----- stderr ----- + Resolved 1 package in [TIME] + "###); + }); + + Ok(()) +} + +/// Raise an error when multiple `requirements.txt` files include `--index-url` flags. +#[test] +fn conflicting_index_urls_requirements_txt() -> Result<()> { + let temp_dir = TempDir::new()?; + let cache_dir = TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + let requirements_in = temp_dir.child("requirements.in"); + requirements_in.write_str("--index-url https://google.com\ntqdm")?; + + let constraints_in = temp_dir.child("constraints.in"); + constraints_in.write_str("--index-url https://wikipedia.org\nflask")?; + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("compile") + .arg("requirements.in") + .arg("--constraint") + .arg("constraints.in") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Multiple index URLs specified: `https://google.com/` vs.` https://wikipedia.org/ + "###); + }); + + Ok(()) +} diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs index 0b5c165a4..62109a660 100644 --- a/crates/requirements-txt/src/lib.rs +++ b/crates/requirements-txt/src/lib.rs @@ -45,7 +45,7 @@ use unscanny::{Pattern, Scanner}; use url::Url; use pep508_rs::{split_scheme, Extras, Pep508Error, Pep508ErrorSource, Requirement, VerbatimUrl}; -use puffin_fs::NormalizedDisplay; +use puffin_fs::{normalize_url_path, NormalizedDisplay}; use puffin_normalize::ExtraName; /// We emit one of those for each requirements.txt entry @@ -66,6 +66,62 @@ enum RequirementsTxtStatement { RequirementEntry(RequirementEntry), /// `-e` EditableRequirement(EditableRequirement), + /// `--index-url` + IndexUrl(Url), + /// `--extra-index-url` + ExtraIndexUrl(Url), + /// `--find-links` + FindLinks(FindLink), + /// `--no-index` + NoIndex, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FindLink { + Path(PathBuf), + Url(Url), +} + +impl FindLink { + /// Parse a raw string for a `--find-links` entry, which could be a URL or a local path. + /// + /// For example: + /// - `file:///home/ferris/project/scripts/...` + /// - `file:../ferris/` + /// - `../ferris/` + /// - `https://download.pytorch.org/whl/torch_stable.html` + pub fn parse(given: &str, working_dir: impl AsRef) -> Result { + if let Some((scheme, path)) = split_scheme(given) { + if scheme == "file" { + // Ex) `file:///home/ferris/project/scripts/...` or `file:../ferris/` + let path = path.strip_prefix("//").unwrap_or(path); + + // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`. + let path = normalize_url_path(path); + + let path = PathBuf::from(path.as_ref()); + let path = if path.is_absolute() { + path + } else { + working_dir.as_ref().join(path) + }; + Ok(Self::Path(path)) + } else { + // Ex) `https://download.pytorch.org/whl/torch_stable.html` + let url = Url::parse(given)?; + Ok(Self::Url(url)) + } + } else { + // Ex) `../ferris/` + let path = PathBuf::from(given); + let path = if path.is_absolute() { + path + } else { + working_dir.as_ref().join(path) + }; + Ok(Self::Path(path)) + } + } } #[derive(Debug, Clone, Eq, PartialEq)] @@ -134,15 +190,15 @@ impl EditableRequirement { // Create a `VerbatimUrl` to represent the editable requirement. let url = if let Some((scheme, path)) = split_scheme(requirement) { if scheme == "file" { - if let Some(path) = path.strip_prefix("//") { - // Ex) `file:///home/ferris/project/scripts/...` - VerbatimUrl::from_path(path, working_dir.as_ref()) - } else { - // Ex) `file:../editable/` - VerbatimUrl::from_path(path, working_dir.as_ref()) - } + // Ex) `file:///home/ferris/project/scripts/...` or `file:../editable/` + let path = path.strip_prefix("//").unwrap_or(path); + + // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`. + let path = normalize_url_path(path); + + VerbatimUrl::from_path(path, working_dir.as_ref()) } else { - // Ex) `https://...` + // Ex) `https://download.pytorch.org/whl/torch_stable.html` return Err(RequirementsTxtParserError::UnsupportedUrl( requirement.to_string(), )); @@ -218,14 +274,22 @@ impl Display for RequirementEntry { } /// Parsed and flattened requirements.txt with requirements and constraints -#[derive(Debug, Clone, Default, Eq, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct RequirementsTxt { - /// The actual requirements with the hashes + /// The actual requirements with the hashes. pub requirements: Vec, - /// Constraints included with `-c` + /// Constraints included with `-c`. pub constraints: Vec, - /// Editables with `-e` + /// Editables with `-e`. pub editables: Vec, + /// The index URL, specified with `--index-url`. + pub index_url: Option, + /// The extra index URLs, specified with `--extra-index-url`. + pub extra_index_urls: Vec, + /// The find links locations, specified with `--find-links`. + pub find_links: Vec, + /// Whether to ignore the index, specified with `--no-index`. + pub no_index: bool, } impl RequirementsTxt { @@ -313,6 +377,24 @@ impl RequirementsTxt { RequirementsTxtStatement::EditableRequirement(editable) => { data.editables.push(editable); } + RequirementsTxtStatement::IndexUrl(url) => { + if data.index_url.is_some() { + return Err(RequirementsTxtParserError::Parser { + message: "Multiple `--index-url` values provided".to_string(), + location: s.cursor(), + }); + } + data.index_url = Some(url); + } + RequirementsTxtStatement::ExtraIndexUrl(url) => { + data.extra_index_urls.push(url); + } + RequirementsTxtStatement::FindLinks(path_or_url) => { + data.find_links.push(path_or_url); + } + RequirementsTxtStatement::NoIndex => { + data.no_index = true; + } } } Ok(data) @@ -366,6 +448,37 @@ fn parse_entry( let editable_requirement = EditableRequirement::parse(path_or_url, working_dir) .map_err(|err| err.with_offset(start))?; RequirementsTxtStatement::EditableRequirement(editable_requirement) + } else if s.eat_if("-i") || s.eat_if("--index-url") { + let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?; + let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url { + source: err, + url: url.to_string(), + start, + end: s.cursor(), + })?; + RequirementsTxtStatement::IndexUrl(url) + } else if s.eat_if("--extra-index-url") { + let url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?; + let url = Url::parse(url).map_err(|err| RequirementsTxtParserError::Url { + source: err, + url: url.to_string(), + start, + end: s.cursor(), + })?; + RequirementsTxtStatement::ExtraIndexUrl(url) + } else if s.eat_if("--no-index") { + RequirementsTxtStatement::NoIndex + } else if s.eat_if("--find-links") || s.eat_if("-f") { + let path_or_url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?; + let path_or_url = FindLink::parse(path_or_url, working_dir).map_err(|err| { + RequirementsTxtParserError::Url { + source: err, + url: path_or_url.to_string(), + start, + end: s.cursor(), + } + })?; + RequirementsTxtStatement::FindLinks(path_or_url) } else if s.at(char::is_ascii_alphanumeric) { let (requirement, hashes) = parse_requirement_and_hashes(s, content, working_dir)?; RequirementsTxtStatement::RequirementEntry(RequirementEntry { @@ -545,6 +658,12 @@ pub struct RequirementsTxtFileError { #[derive(Debug)] pub enum RequirementsTxtParserError { IO(io::Error), + Url { + source: url::ParseError, + url: String, + start: usize, + end: usize, + }, InvalidEditablePath(String), UnsupportedUrl(String), Parser { @@ -577,6 +696,17 @@ impl RequirementsTxtParserError { RequirementsTxtParserError::InvalidEditablePath(given) => { RequirementsTxtParserError::InvalidEditablePath(given) } + RequirementsTxtParserError::Url { + source, + url, + start, + end, + } => RequirementsTxtParserError::Url { + source, + url, + start: start + offset, + end: end + offset, + }, RequirementsTxtParserError::UnsupportedUrl(url) => { RequirementsTxtParserError::UnsupportedUrl(url) } @@ -618,8 +748,11 @@ impl Display for RequirementsTxtParserError { RequirementsTxtParserError::InvalidEditablePath(given) => { write!(f, "Invalid editable path: {given}") } + RequirementsTxtParserError::Url { url, start, .. } => { + write!(f, "Invalid URL at position {start}: `{url}`") + } RequirementsTxtParserError::UnsupportedUrl(url) => { - write!(f, "Unsupported URL (expected a `file://` scheme): {url}") + write!(f, "Unsupported URL (expected a `file://` scheme): `{url}`") } RequirementsTxtParserError::Parser { message, location } => { write!(f, "{message} at position {location}") @@ -641,6 +774,7 @@ impl std::error::Error for RequirementsTxtParserError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match &self { RequirementsTxtParserError::IO(err) => err.source(), + RequirementsTxtParserError::Url { source, .. } => Some(source), RequirementsTxtParserError::InvalidEditablePath(_) => None, RequirementsTxtParserError::UnsupportedUrl(_) => None, RequirementsTxtParserError::UnsupportedRequirement { source, .. } => Some(source), @@ -655,6 +789,13 @@ impl Display for RequirementsTxtFileError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match &self.error { RequirementsTxtParserError::IO(err) => err.fmt(f), + RequirementsTxtParserError::Url { url, start, .. } => { + write!( + f, + "Invalid URL in `{}` at position {start}: `{url}`", + self.file.normalized_display(), + ) + } RequirementsTxtParserError::InvalidEditablePath(given) => { write!( f, @@ -665,7 +806,7 @@ impl Display for RequirementsTxtFileError { RequirementsTxtParserError::UnsupportedUrl(url) => { write!( f, - "Unsupported URL (expected a `file://` scheme) in `{}`: {url}", + "Unsupported URL (expected a `file://` scheme) in `{}`: `{url}`", self.file.normalized_display(), ) } @@ -676,13 +817,11 @@ impl Display for RequirementsTxtFileError { self.file.normalized_display(), ) } - RequirementsTxtParserError::UnsupportedRequirement { start, end, .. } => { + RequirementsTxtParserError::UnsupportedRequirement { start, .. } => { write!( f, - "Unsupported requirement in {} position {} to {}", + "Unsupported requirement in {} at position {start}", self.file.normalized_display(), - start, - end, ) } RequirementsTxtParserError::Pep508 { start, .. } => { @@ -865,7 +1004,7 @@ mod test { insta::with_settings!({ filters => vec![(requirements_txt.path().to_str().unwrap(), "")] }, { - insta::assert_display_snapshot!(errors, @"Unsupported URL (expected a `file://` scheme) in ``: http://localhost:8080/"); + insta::assert_display_snapshot!(errors, @"Unsupported URL (expected a `file://` scheme) in ``: `http://localhost:8080/`"); }); Ok(()) @@ -899,6 +1038,32 @@ mod test { Ok(()) } + #[test] + fn invalid_index_url() -> Result<()> { + let temp_dir = assert_fs::TempDir::new()?; + let requirements_txt = temp_dir.child("requirements.txt"); + requirements_txt.write_str(indoc! {" + --index-url 123 + "})?; + + let error = RequirementsTxt::parse(requirements_txt.path(), temp_dir.path()).unwrap_err(); + let errors = anyhow::Error::new(error) + .chain() + .map(|err| err.to_string().replace('\\', "/")) + .join("\n"); + + insta::with_settings!({ + filters => vec![(requirements_txt.path().to_str().unwrap(), "")] + }, { + insta::assert_display_snapshot!(errors, @r###" + Invalid URL in `` at position 0: `123` + relative URL without a base + "###); + }); + + Ok(()) + } + #[test] fn editable_extra() { assert_eq!( diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap index 3838a01e6..30b4fb1f6 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap @@ -145,4 +145,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap index f79457fa5..dc4a4806e 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap @@ -69,4 +69,8 @@ RequirementsTxt { }, ], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap index 34b443a57..91961fdad 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap @@ -53,4 +53,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap index fdaea6337..c6f1b96ab 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap @@ -58,4 +58,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap index eb530dc8d..077583693 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap @@ -6,4 +6,8 @@ RequirementsTxt { requirements: [], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap index 73c7bda3b..e0354af92 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap @@ -96,4 +96,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap index 8d96b6bb3..2dc8c591f 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap @@ -42,4 +42,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap index 97ff4e849..3bdafcf32 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap @@ -19,4 +19,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap index 4dc91a455..c570919d3 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap @@ -282,4 +282,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap index b856a7dff..60b773732 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap @@ -53,4 +53,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap index fdaea6337..c6f1b96ab 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap @@ -58,4 +58,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap index 3838a01e6..30b4fb1f6 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap @@ -145,4 +145,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap index f79457fa5..dc4a4806e 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap @@ -69,4 +69,8 @@ RequirementsTxt { }, ], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap index 34b443a57..91961fdad 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap @@ -53,4 +53,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap index eb530dc8d..077583693 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap @@ -6,4 +6,8 @@ RequirementsTxt { requirements: [], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap index 73c7bda3b..e0354af92 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap @@ -96,4 +96,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap index 8d96b6bb3..2dc8c591f 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap @@ -42,4 +42,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap index 97ff4e849..3bdafcf32 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap @@ -19,4 +19,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap index 4dc91a455..c570919d3 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap @@ -282,4 +282,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap index b856a7dff..60b773732 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap @@ -53,4 +53,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap index fdaea6337..c6f1b96ab 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap @@ -58,4 +58,8 @@ RequirementsTxt { ], constraints: [], editables: [], + index_url: None, + extra_index_urls: [], + find_links: [], + no_index: false, }