From 87adf14fdfc20df519824947b25560ba935841e5 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Tue, 2 Dec 2025 15:42:44 -0800 Subject: [PATCH] Allow reading requirements from scripts with HTTP(S) paths (#16891) ## Summary Closes https://github.com/astral-sh/uv/issues/16890. --- crates/uv-requirements/src/sources.rs | 2 - crates/uv-requirements/src/specification.rs | 60 ++++++++++++---- crates/uv/tests/it/edit.rs | 79 ++++++++++++++++++++- crates/uv/tests/it/tool_run.rs | 6 +- 4 files changed, 128 insertions(+), 19 deletions(-) diff --git a/crates/uv-requirements/src/sources.rs b/crates/uv-requirements/src/sources.rs index 5a725c24c..5bca4a22a 100644 --- a/crates/uv-requirements/src/sources.rs +++ b/crates/uv-requirements/src/sources.rs @@ -65,8 +65,6 @@ impl RequirementsSource { } else if path .extension() .is_some_and(|ext| ext.eq_ignore_ascii_case("txt") || ext.eq_ignore_ascii_case("in")) - || path.starts_with("http://") - || path.starts_with("https://") { Ok(Self::RequirementsTxt(path)) } else if path.extension().is_none() { diff --git a/crates/uv-requirements/src/specification.rs b/crates/uv-requirements/src/specification.rs index c10feeaa1..6af166088 100644 --- a/crates/uv-requirements/src/specification.rs +++ b/crates/uv-requirements/src/specification.rs @@ -33,6 +33,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use rustc_hash::FxHashSet; use tracing::instrument; +use url::Url; use uv_cache_key::CanonicalUrl; use uv_client::BaseClientBuilder; @@ -45,8 +46,9 @@ use uv_distribution_types::{ use uv_fs::{CWD, Simplified}; use uv_normalize::{ExtraName, PackageName, PipGroupName}; use uv_pypi_types::PyProjectToml; +use uv_redacted::DisplaySafeUrl; use uv_requirements_txt::{RequirementsTxt, RequirementsTxtRequirement, SourceCache}; -use uv_scripts::{Pep723Error, Pep723Metadata, Pep723Script}; +use uv_scripts::Pep723Metadata; use uv_warnings::warn_user; use crate::{RequirementsSource, SourceTree}; @@ -269,8 +271,8 @@ impl RequirementsSpecification { Self::from_requirements_txt(requirements_txt) } RequirementsSource::PyprojectToml(path) => { - let contents = match fs_err::tokio::read_to_string(&path).await { - Ok(contents) => contents, + let content = match fs_err::tokio::read_to_string(&path).await { + Ok(content) => content, Err(err) if err.kind() == std::io::ErrorKind::NotFound => { return Err(anyhow::anyhow!("File not found: `{}`", path.user_display())); } @@ -282,7 +284,7 @@ impl RequirementsSpecification { )); } }; - let pyproject_toml = toml::from_str::(&contents) + let pyproject_toml = toml::from_str::(&content) .with_context(|| format!("Failed to parse: `{}`", path.user_display()))?; Self { @@ -291,7 +293,15 @@ impl RequirementsSpecification { } } RequirementsSource::Pep723Script(path) => { - let script = match Pep723Script::read(&path).await { + let content = if let Some(content) = cache.get(path.as_path()) { + content.clone() + } else { + let content = read_file(path, client_builder).await?; + cache.insert(path.clone(), content.clone()); + content + }; + + let metadata = match Pep723Metadata::parse(content.as_bytes()) { Ok(Some(script)) => script, Ok(None) => { return Err(anyhow::anyhow!( @@ -299,16 +309,10 @@ impl RequirementsSpecification { path.user_display(), )); } - Err(Pep723Error::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => { - return Err(anyhow::anyhow!( - "Failed to read `{}` (not found)", - path.user_display(), - )); - } Err(err) => return Err(err.into()), }; - Self::from_pep723_metadata(&script.metadata) + Self::from_pep723_metadata(&metadata) } RequirementsSource::SetupPy(path) => { if !path.is_file() { @@ -347,11 +351,10 @@ impl RequirementsSpecification { )); } RequirementsSource::Extensionless(path) => { - // Read the file content. let content = if let Some(content) = cache.get(path.as_path()) { content.clone() } else { - let content = uv_fs::read_to_string_transcode(&path).await?; + let content = read_file(path, client_builder).await?; cache.insert(path.clone(), content.clone()); content }; @@ -741,3 +744,32 @@ pub struct GroupsSpecification { /// The enabled groups. pub groups: Vec, } + +/// Read the contents of a path, fetching over HTTP(S) if necessary. +async fn read_file(path: &Path, client_builder: &BaseClientBuilder<'_>) -> Result { + // If the path is a URL, fetch it over HTTP(S). + if path.starts_with("http://") || path.starts_with("https://") { + // Only continue if we are absolutely certain no local file exists. + // + // We don't do this check on Windows since the file path would + // be invalid anyway, and thus couldn't refer to a local file. + if !cfg!(unix) || matches!(path.try_exists(), Ok(false)) { + let url = DisplaySafeUrl::parse(&path.to_string_lossy())?; + + let client = client_builder.build(); + let response = client + .for_host(&url) + .get(Url::from(url.clone())) + .send() + .await?; + + response.error_for_status_ref()?; + + return Ok(response.text().await?); + } + } + + // Read the file content. + let content = uv_fs::read_to_string_transcode(path).await?; + Ok(content) +} diff --git a/crates/uv/tests/it/edit.rs b/crates/uv/tests/it/edit.rs index 3eecd82bd..001bfdbb6 100644 --- a/crates/uv/tests/it/edit.rs +++ b/crates/uv/tests/it/edit.rs @@ -15,7 +15,7 @@ use indoc::{formatdoc, indoc}; use insta::assert_snapshot; use std::path::Path; use url::Url; -use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method, matchers::path}; use uv_cache_key::{RepositoryUrl, cache_digest}; use uv_fs::Simplified; @@ -7393,6 +7393,83 @@ fn add_extensionless_script() -> Result<()> { Ok(()) } +/// Add from a remote PEP 723 script via `-r`. +#[tokio::test] +async fn add_requirements_from_remote_script() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str(indoc! {r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [] + "#})?; + + // Create a mock server that serves a PEP 723 script. + let server = MockServer::start().await; + let script_content = indoc! {r#" + # /// script + # requires-python = ">=3.12" + # dependencies = [ + # "anyio>=4", + # "rich", + # ] + # /// + import anyio + from rich.pretty import pprint + pprint("Hello, world!") + "#}; + + Mock::given(method("GET")) + .and(path("/script")) + .respond_with(ResponseTemplate::new(200).set_body_string(script_content)) + .mount(&server) + .await; + + let script_url = format!("{}/script", server.uri()); + + uv_snapshot!(context.filters(), context.add().arg("-r").arg(&script_url), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 8 packages in [TIME] + Prepared 7 packages in [TIME] + Installed 7 packages in [TIME] + + anyio==4.3.0 + + idna==3.6 + + markdown-it-py==3.0.0 + + mdurl==0.1.2 + + pygments==2.17.2 + + rich==13.7.1 + + sniffio==1.3.1 + "###); + + let pyproject_toml_content = context.read("pyproject.toml"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + pyproject_toml_content, @r###" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [ + "anyio>=4", + "rich>=13.7.1", + ] + "### + ); + }); + + Ok(()) +} + /// Remove a dependency that is present in multiple places. #[test] fn remove_repeated() -> Result<()> { diff --git a/crates/uv/tests/it/tool_run.rs b/crates/uv/tests/it/tool_run.rs index 8e08f0aa6..ce1d19326 100644 --- a/crates/uv/tests/it/tool_run.rs +++ b/crates/uv/tests/it/tool_run.rs @@ -2907,7 +2907,9 @@ fn tool_run_with_incompatible_build_constraints() -> Result<()> { #[test] fn tool_run_with_dependencies_from_script() -> Result<()> { - let context = TestContext::new("3.12").with_filtered_counts(); + let context = TestContext::new("3.12") + .with_filtered_counts() + .with_filtered_missing_file_error(); let script_contents = indoc! {r#" # /// script @@ -2992,7 +2994,7 @@ fn tool_run_with_dependencies_from_script() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: Failed to read `missing_file.py` (not found) + error: failed to read from file `missing_file.py`: [OS ERROR 2] "); Ok(())