From 366c389385c8f90ce71072da529adb34121d716a Mon Sep 17 00:00:00 2001 From: konsti Date: Wed, 6 Dec 2023 18:21:15 +0100 Subject: [PATCH] Parse editable installs (#564) Parse `-e` for editable installs in `requirements.txt`. Unlike all the other requirements, editable installs don't have the name of the package specified. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/requirements-txt/Cargo.toml | 1 + crates/requirements-txt/src/lib.rs | 53 ++++++++++++++--- ...nts_txt__test__line-endings-basic.txt.snap | 1 + ..._test__line-endings-constraints-a.txt.snap | 1 + ..._test__line-endings-constraints-b.txt.snap | 1 + ..._txt__test__line-endings-editable.txt.snap | 58 +++++++++++++++++++ ...nts_txt__test__line-endings-empty.txt.snap | 1 + ...xt__test__line-endings-for-poetry.txt.snap | 1 + ...txt__test__line-endings-include-a.txt.snap | 1 + ...txt__test__line-endings-include-b.txt.snap | 1 + ...__line-endings-poetry-with-hashes.txt.snap | 1 + ...nts_txt__test__line-endings-small.txt.snap | 1 + ...xt__test__line-endings-whitespace.txt.snap | 1 + ...quirements_txt__test__parse-basic.txt.snap | 1 + ...ts_txt__test__parse-constraints-a.txt.snap | 1 + ...ts_txt__test__parse-constraints-b.txt.snap | 1 + ...quirements_txt__test__parse-empty.txt.snap | 1 + ...ments_txt__test__parse-for-poetry.txt.snap | 1 + ...ements_txt__test__parse-include-a.txt.snap | 1 + ...ements_txt__test__parse-include-b.txt.snap | 1 + ...t__test__parse-poetry-with-hashes.txt.snap | 1 + ...quirements_txt__test__parse-small.txt.snap | 1 + ...ments_txt__test__parse-whitespace.txt.snap | 1 + .../test-data/requirements-txt/editable.txt | 25 ++++++++ .../requirements-txt/invalid-editable.txt | 1 + 27 files changed, 153 insertions(+), 7 deletions(-) create mode 100644 crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap create mode 100644 crates/requirements-txt/test-data/requirements-txt/editable.txt create mode 100644 crates/requirements-txt/test-data/requirements-txt/invalid-editable.txt diff --git a/Cargo.lock b/Cargo.lock index 0682a9cd3..3e886a2aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2947,6 +2947,7 @@ dependencies = [ "thiserror", "tracing", "unscanny", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 004eeca58..0d5768d20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = ["crates/*"] +exclude = ["scripts"] resolver = "2" [workspace.package] diff --git a/crates/requirements-txt/Cargo.toml b/crates/requirements-txt/Cargo.toml index 11ea19f72..5a67ca78a 100644 --- a/crates/requirements-txt/Cargo.toml +++ b/crates/requirements-txt/Cargo.toml @@ -24,6 +24,7 @@ serde = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } unscanny = { workspace = true } +url = { workspace = true } [dev-dependencies] anyhow = { version = "1.0.75" } diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs index f207a4642..797a0c1d0 100644 --- a/crates/requirements-txt/src/lib.rs +++ b/crates/requirements-txt/src/lib.rs @@ -10,7 +10,6 @@ //! * `-e` //! //! Unsupported: -//! * `-e `. TBD //! * ``. TBD //! * ``. TBD //! * Options without a requirement, such as `--find-links` or `--index-url` @@ -44,6 +43,7 @@ use fs_err as fs; use serde::{Deserialize, Serialize}; use tracing::warn; use unscanny::{Pattern, Scanner}; +use url::Url; use pep508_rs::{Pep508Error, Requirement}; @@ -63,6 +63,14 @@ enum RequirementsTxtStatement { }, /// PEP 508 requirement plus metadata RequirementEntry(RequirementEntry), + /// `-e` + EditableRequirement(EditableRequirement), +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Serialize)] +pub enum EditableRequirement { + Path(PathBuf), + Url(Url), } /// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in @@ -97,6 +105,8 @@ pub struct RequirementsTxt { pub requirements: Vec, /// Constraints included with `-c` pub constraints: Vec, + /// Editables with `-e` + pub editable: Vec, } impl RequirementsTxt { @@ -182,6 +192,9 @@ impl RequirementsTxt { RequirementsTxtStatement::RequirementEntry(requirement_entry) => { data.requirements.push(requirement_entry); } + RequirementsTxtStatement::EditableRequirement(editable) => { + data.editable.push(editable); + } } } Ok(data) @@ -230,12 +243,13 @@ fn parse_entry( end, } } else if s.eat_if("-e") { - let (requirement, hashes) = parse_requirement_and_hashes(s, content)?; - RequirementsTxtStatement::RequirementEntry(RequirementEntry { - requirement, - hashes, - editable: true, - }) + let path_or_url = parse_value(s, |c: char| !['\n', '\r'].contains(&c))?; + let editable_requirement = if let Ok(url) = Url::from_str(path_or_url) { + EditableRequirement::Url(url) + } else { + EditableRequirement::Path(PathBuf::from(path_or_url)) + }; + RequirementsTxtStatement::EditableRequirement(editable_requirement) } else if s.at(char::is_ascii_alphanumeric) { let (requirement, hashes) = parse_requirement_and_hashes(s, content)?; RequirementsTxtStatement::RequirementEntry(RequirementEntry { @@ -505,6 +519,7 @@ mod test { #[test_case(Path::new("poetry-with-hashes.txt"))] #[test_case(Path::new("small.txt"))] #[test_case(Path::new("whitespace.txt"))] + #[test_case(Path::new("editable.txt"))] fn line_endings(path: &Path) { let working_dir = workspace_test_data_dir().join("requirements-txt"); let requirements_txt = working_dir.join(path); @@ -581,4 +596,28 @@ mod test { fn workspace_test_data_dir() -> PathBuf { PathBuf::from("./test-data") } + + #[test] + fn invalid_editable() { + let working_dir = workspace_test_data_dir().join("requirements-txt"); + let basic = working_dir.join("invalid-requirement"); + let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); + let errors = anyhow::Error::new(err) + .chain() + .map(ToString::to_string) + .collect::>(); + let expected = &[ + format!( + "Couldn't parse requirement in {} position 0 to 15", + basic.display() + ), + indoc! {" + Expected an alphanumeric character starting the extra name, found 'ö' + numpy[ö]==1.29 + ^" + } + .to_string(), + ]; + assert_eq!(errors, expected); + } } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap index b4523f89f..9d80cad43 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-basic.txt.snap @@ -208,4 +208,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap index e84e36b21..44472c9f6 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-a.txt.snap @@ -99,4 +99,5 @@ RequirementsTxt { marker: None, }, ], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap index c05d9bd0b..b2fa24b85 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-constraints-b.txt.snap @@ -73,4 +73,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap new file mode 100644 index 000000000..4126e4423 --- /dev/null +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-editable.txt.snap @@ -0,0 +1,58 @@ +--- +source: crates/requirements-txt/src/lib.rs +expression: actual +--- +RequirementsTxt { + requirements: [ + RequirementEntry { + requirement: Requirement { + name: PackageName( + "numpy", + ), + extras: None, + version_or_url: None, + marker: None, + }, + hashes: [], + editable: false, + }, + RequirementEntry { + requirement: Requirement { + name: PackageName( + "pandas", + ), + extras: Some( + [ + ExtraName( + "tabulate", + ), + ], + ), + version_or_url: Some( + Url( + Url { + scheme: "https", + cannot_be_a_base: false, + username: "", + password: None, + host: Some( + Domain( + "github.com", + ), + ), + port: None, + path: "/pandas-dev/pandas", + query: None, + fragment: None, + }, + ), + ), + marker: None, + }, + hashes: [], + editable: false, + }, + ], + constraints: [], + editable: [], +} diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap index b89cc12d4..e82ded18a 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-empty.txt.snap @@ -5,4 +5,5 @@ expression: actual RequirementsTxt { requirements: [], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap index b1aaa4862..2776d7d1e 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-for-poetry.txt.snap @@ -136,4 +136,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap index 493846862..970c20309 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-a.txt.snap @@ -52,4 +52,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap index fb4520ffa..efdc11611 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-include-b.txt.snap @@ -18,4 +18,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap index f5b3286e4..ee565de72 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-poetry-with-hashes.txt.snap @@ -336,4 +336,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap index 786cf37e8..e4c28b3b7 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-small.txt.snap @@ -74,4 +74,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap index 85cfe9702..4126e4423 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__line-endings-whitespace.txt.snap @@ -54,4 +54,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap index b4523f89f..9d80cad43 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-basic.txt.snap @@ -208,4 +208,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap index e84e36b21..44472c9f6 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-a.txt.snap @@ -99,4 +99,5 @@ RequirementsTxt { marker: None, }, ], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap index c05d9bd0b..b2fa24b85 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-constraints-b.txt.snap @@ -73,4 +73,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap index b89cc12d4..e82ded18a 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-empty.txt.snap @@ -5,4 +5,5 @@ expression: actual RequirementsTxt { requirements: [], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap index b1aaa4862..2776d7d1e 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-for-poetry.txt.snap @@ -136,4 +136,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap index 493846862..970c20309 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-a.txt.snap @@ -52,4 +52,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap index fb4520ffa..efdc11611 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-include-b.txt.snap @@ -18,4 +18,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap index f5b3286e4..ee565de72 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-poetry-with-hashes.txt.snap @@ -336,4 +336,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap index 786cf37e8..e4c28b3b7 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-small.txt.snap @@ -74,4 +74,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap index 85cfe9702..4126e4423 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-whitespace.txt.snap @@ -54,4 +54,5 @@ RequirementsTxt { }, ], constraints: [], + editable: [], } diff --git a/crates/requirements-txt/test-data/requirements-txt/editable.txt b/crates/requirements-txt/test-data/requirements-txt/editable.txt new file mode 100644 index 000000000..c1e99017e --- /dev/null +++ b/crates/requirements-txt/test-data/requirements-txt/editable.txt @@ -0,0 +1,25 @@ + + + # # + # # + # # + # # + # # + # # + # # + + ## + +# + + numpy # # + +\ + +pandas [tabulate] @ https://github.com/pandas-dev/pandas \ + # üh + + + # + # 안녕 + # diff --git a/crates/requirements-txt/test-data/requirements-txt/invalid-editable.txt b/crates/requirements-txt/test-data/requirements-txt/invalid-editable.txt new file mode 100644 index 000000000..69dc80962 --- /dev/null +++ b/crates/requirements-txt/test-data/requirements-txt/invalid-editable.txt @@ -0,0 +1 @@ +numpy[ö]==1.29 \ No newline at end of file