From ed8f6e455681b83889298b24befcfae2f3d86ab0 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 25 Apr 2024 19:15:39 -0400 Subject: [PATCH] Add `--target` support to `sync` and `install` (#3257) ## Summary The approach taken here is to model `--target` as an install scheme in which all the directories are just subdirectories of the `--target`. From there, everything else... just works? Like, upgrade, uninstalls, editables, etc. all "just work". Closes #1517. --- crates/uv-interpreter/Cargo.toml | 1 + crates/uv-interpreter/src/interpreter.rs | 68 ++++++++++---- crates/uv-interpreter/src/lib.rs | 2 + .../uv-interpreter/src/python_environment.rs | 44 ++++++--- crates/uv-interpreter/src/target.rs | 40 ++++++++ crates/uv-workspace/src/settings.rs | 1 + crates/uv/src/cli.rs | 15 +++ crates/uv/src/commands/pip_install.rs | 13 ++- crates/uv/src/commands/pip_sync.rs | 12 ++- crates/uv/src/commands/pip_uninstall.rs | 13 ++- crates/uv/src/main.rs | 3 + crates/uv/src/settings.rs | 10 ++ crates/uv/tests/pip_sync.rs | 92 +++++++++++++++++++ uv.schema.json | 6 ++ 14 files changed, 284 insertions(+), 36 deletions(-) create mode 100644 crates/uv-interpreter/src/target.rs diff --git a/crates/uv-interpreter/Cargo.toml b/crates/uv-interpreter/Cargo.toml index 4076b2a39..101085871 100644 --- a/crates/uv-interpreter/Cargo.toml +++ b/crates/uv-interpreter/Cargo.toml @@ -26,6 +26,7 @@ uv-warnings = { workspace = true } configparser = { workspace = true } fs-err = { workspace = true, features = ["tokio"] } +itertools = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } rmp-serde = { workspace = true } diff --git a/crates/uv-interpreter/src/interpreter.rs b/crates/uv-interpreter/src/interpreter.rs index 56d8a3d32..1b2248a19 100644 --- a/crates/uv-interpreter/src/interpreter.rs +++ b/crates/uv-interpreter/src/interpreter.rs @@ -18,8 +18,8 @@ use uv_cache::{Cache, CacheBucket, CachedByTimestamp, Freshness, Timestamp}; use uv_fs::{write_atomic_sync, PythonExt, Simplified}; use uv_toolchain::PythonVersion; -use crate::Error; use crate::Virtualenv; +use crate::{Error, Target}; /// A Python executable and its associated platform markers. #[derive(Debug, Clone)] @@ -35,6 +35,7 @@ pub struct Interpreter { sys_executable: PathBuf, stdlib: PathBuf, tags: OnceCell, + target: Option, gil_disabled: bool, } @@ -62,6 +63,7 @@ impl Interpreter { sys_executable: info.sys_executable, stdlib: info.stdlib, tags: OnceCell::new(), + target: None, }) } @@ -91,6 +93,7 @@ impl Interpreter { sys_executable: PathBuf::from("/dev/null"), stdlib: PathBuf::from("/dev/null"), tags: OnceCell::new(), + target: None, gil_disabled: false, } } @@ -106,6 +109,17 @@ impl Interpreter { } } + /// Return a new [`Interpreter`] to install into the given `--target` directory. + /// + /// Initializes the `--target` directory with the expected layout. + #[must_use] + pub fn with_target(self, target: Target) -> Self { + Self { + target: Some(target), + ..self + } + } + /// Returns the path to the Python virtual environment. #[inline] pub fn platform(&self) -> &Platform { @@ -135,9 +149,15 @@ impl Interpreter { /// /// See: pub fn is_virtualenv(&self) -> bool { + // Maybe this should return `false` if it's a target? self.prefix != self.base_prefix } + /// Returns `true` if the environment is a `--target` environment. + pub fn is_target(&self) -> bool { + self.target.is_some() + } + /// Returns `Some` if the environment is externally managed, optionally including an error /// message from the `EXTERNALLY-MANAGED` file. /// @@ -148,6 +168,11 @@ impl Interpreter { return None; } + // If we're installing into a target directory, it's never externally managed. + if self.is_target() { + return None; + } + let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else { return None; }; @@ -303,28 +328,37 @@ impl Interpreter { self.gil_disabled } + /// Return the `--target` directory for this interpreter, if any. + pub fn target(&self) -> Option<&Target> { + self.target.as_ref() + } + /// Return the [`Layout`] environment used to install wheels into this interpreter. pub fn layout(&self) -> Layout { Layout { python_version: self.python_tuple(), sys_executable: self.sys_executable().to_path_buf(), os_name: self.markers.os_name.clone(), - scheme: Scheme { - purelib: self.purelib().to_path_buf(), - platlib: self.platlib().to_path_buf(), - scripts: self.scripts().to_path_buf(), - data: self.data().to_path_buf(), - include: if self.is_virtualenv() { - // If the interpreter is a venv, then the `include` directory has a different structure. - // See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172 - self.prefix.join("include").join("site").join(format!( - "python{}.{}", - self.python_major(), - self.python_minor() - )) - } else { - self.include().to_path_buf() - }, + scheme: if let Some(target) = self.target.as_ref() { + target.scheme() + } else { + Scheme { + purelib: self.purelib().to_path_buf(), + platlib: self.platlib().to_path_buf(), + scripts: self.scripts().to_path_buf(), + data: self.data().to_path_buf(), + include: if self.is_virtualenv() { + // If the interpreter is a venv, then the `include` directory has a different structure. + // See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172 + self.prefix.join("include").join("site").join(format!( + "python{}.{}", + self.python_major(), + self.python_minor() + )) + } else { + self.include().to_path_buf() + }, + } }, } } diff --git a/crates/uv-interpreter/src/lib.rs b/crates/uv-interpreter/src/lib.rs index 1286ff436..607be77f4 100644 --- a/crates/uv-interpreter/src/lib.rs +++ b/crates/uv-interpreter/src/lib.rs @@ -19,12 +19,14 @@ pub use crate::find_python::{find_best_python, find_default_python, find_request pub use crate::interpreter::Interpreter; use crate::interpreter::InterpreterInfoError; pub use crate::python_environment::PythonEnvironment; +pub use crate::target::Target; pub use crate::virtualenv::Virtualenv; mod cfg; mod find_python; mod interpreter; mod python_environment; +mod target; mod virtualenv; #[derive(Debug, Error)] diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 63164314a..db490fac7 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -1,3 +1,4 @@ +use itertools::Either; use std::env; use std::path::{Path, PathBuf}; @@ -8,7 +9,7 @@ use uv_cache::Cache; use uv_fs::{LockedFile, Simplified}; use crate::cfg::PyVenvConfiguration; -use crate::{find_default_python, find_requested_python, Error, Interpreter}; +use crate::{find_default_python, find_requested_python, Error, Interpreter, Target}; /// A Python environment, consisting of a Python [`Interpreter`] and its associated paths. #[derive(Debug, Clone)] @@ -68,7 +69,16 @@ impl PythonEnvironment { } } - /// Returns the location of the Python interpreter. + /// Create a [`PythonEnvironment`] from an existing [`Interpreter`] and `--target` directory. + #[must_use] + pub fn with_target(self, target: Target) -> Self { + Self { + interpreter: self.interpreter.with_target(target), + ..self + } + } + + /// Returns the root (i.e., `prefix`) of the Python interpreter. pub fn root(&self) -> &Path { &self.root } @@ -97,15 +107,19 @@ impl PythonEnvironment { /// Some distributions also create symbolic links from `purelib` to `platlib`; in such cases, we /// still deduplicate the entries, returning a single path. pub fn site_packages(&self) -> impl Iterator { - let purelib = self.interpreter.purelib(); - let platlib = self.interpreter.platlib(); - std::iter::once(purelib).chain( - if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) { - None - } else { - Some(platlib) - }, - ) + if let Some(target) = self.interpreter.target() { + Either::Left(std::iter::once(target.root())) + } else { + let purelib = self.interpreter.purelib(); + let platlib = self.interpreter.platlib(); + Either::Right(std::iter::once(purelib).chain( + if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) { + None + } else { + Some(platlib) + }, + )) + } } /// Returns the path to the `bin` directory inside a virtual environment. @@ -115,7 +129,13 @@ impl PythonEnvironment { /// Grab a file lock for the virtual environment to prevent concurrent writes across processes. pub fn lock(&self) -> Result { - if self.interpreter.is_virtualenv() { + if let Some(target) = self.interpreter.target() { + // If we're installing into a `--target`, use a target-specific lock file. + LockedFile::acquire( + target.root().join(".lock"), + target.root().simplified_display(), + ) + } else if self.interpreter.is_virtualenv() { // If the environment a virtualenv, use a virtualenv-specific lock file. LockedFile::acquire(self.root.join(".lock"), self.root.simplified_display()) } else { diff --git a/crates/uv-interpreter/src/target.rs b/crates/uv-interpreter/src/target.rs new file mode 100644 index 000000000..c20c1b36a --- /dev/null +++ b/crates/uv-interpreter/src/target.rs @@ -0,0 +1,40 @@ +use std::path::{Path, PathBuf}; + +use pypi_types::Scheme; + +/// A `--target` directory into which packages can be installed, separate from a virtual environment +/// or system Python interpreter. +#[derive(Debug, Clone)] +pub struct Target(PathBuf); + +impl Target { + /// Return the [`Scheme`] for the `--target` directory. + pub fn scheme(&self) -> Scheme { + Scheme { + purelib: self.0.clone(), + platlib: self.0.clone(), + scripts: self.0.join("bin"), + data: self.0.clone(), + include: self.0.join("include"), + } + } + + /// Initialize the `--target` directory. + pub fn init(&self) -> std::io::Result<()> { + fs_err::create_dir_all(&self.0)?; + fs_err::create_dir_all(self.0.join("bin"))?; + fs_err::create_dir_all(self.0.join("include"))?; + Ok(()) + } + + /// Return the path to the `--target` directory. + pub fn root(&self) -> &Path { + &self.0 + } +} + +impl From for Target { + fn from(path: PathBuf) -> Self { + Self(path) + } +} diff --git a/crates/uv-workspace/src/settings.rs b/crates/uv-workspace/src/settings.rs index 5534d859f..3f97126c1 100644 --- a/crates/uv-workspace/src/settings.rs +++ b/crates/uv-workspace/src/settings.rs @@ -47,6 +47,7 @@ pub struct PipOptions { pub python: Option, pub system: Option, pub break_system_packages: Option, + pub target: Option, pub offline: Option, pub index_url: Option, pub extra_index_url: Option>, diff --git a/crates/uv/src/cli.rs b/crates/uv/src/cli.rs index d5b4b5df2..35d7b089e 100644 --- a/crates/uv/src/cli.rs +++ b/crates/uv/src/cli.rs @@ -768,6 +768,11 @@ pub(crate) struct PipSyncArgs { #[arg(long, overrides_with("break_system_packages"))] pub(crate) no_break_system_packages: bool, + /// Install packages into the specified directory, rather than into the virtual environment + /// or system Python interpreter. + #[arg(long)] + pub(crate) target: Option, + /// Use legacy `setuptools` behavior when building source distributions without a /// `pyproject.toml`. #[arg(long, overrides_with("no_legacy_setup_py"))] @@ -1132,6 +1137,11 @@ pub(crate) struct PipInstallArgs { #[arg(long, overrides_with("break_system_packages"))] pub(crate) no_break_system_packages: bool, + /// Install packages into the specified directory, rather than into the virtual environment + /// or system Python interpreter. + #[arg(long)] + pub(crate) target: Option, + /// Use legacy `setuptools` behavior when building source distributions without a /// `pyproject.toml`. #[arg(long, overrides_with("no_legacy_setup_py"))] @@ -1335,6 +1345,11 @@ pub(crate) struct PipUninstallArgs { #[arg(long, overrides_with("break_system_packages"))] pub(crate) no_break_system_packages: bool, + /// Uninstall packages from the specified directory, rather than from the virtual environment + /// or system Python interpreter. + #[arg(long)] + pub(crate) target: Option, + /// Run offline, i.e., without accessing the network. #[arg(long, overrides_with("no_offline"))] pub(crate) offline: bool, diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 80f524569..85b73c0d5 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -1,11 +1,9 @@ use std::borrow::Cow; use std::fmt::Write; - use std::path::Path; use anstream::eprint; use anyhow::{anyhow, Context, Result}; - use itertools::Itertools; use owo_colors::OwoColorize; use tempfile::tempdir_in; @@ -33,7 +31,7 @@ use uv_configuration::{KeyringProviderType, TargetTriple}; use uv_dispatch::BuildDispatch; use uv_fs::Simplified; use uv_installer::{BuiltEditable, Downloader, Plan, Planner, ResolvedEditable, SitePackages}; -use uv_interpreter::{Interpreter, PythonEnvironment}; +use uv_interpreter::{Interpreter, PythonEnvironment, Target}; use uv_normalize::PackageName; use uv_requirements::{ ExtrasSpecification, LookaheadResolver, NamedRequirementsResolver, RequirementsSource, @@ -84,6 +82,7 @@ pub(crate) async fn pip_install( python: Option, system: bool, break_system_packages: bool, + target: Option, native_tls: bool, cache: Cache, dry_run: bool, @@ -134,6 +133,14 @@ pub(crate) async fn pip_install( venv.python_executable().user_display().cyan() ); + // Apply any `--target` directory. + let venv = if let Some(target) = target { + target.init()?; + venv.with_target(target) + } else { + venv + }; + // If the environment is externally managed, abort. if let Some(externally_managed) = venv.interpreter().is_externally_managed() { if break_system_packages { diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index 84240f4f0..f8482dc40 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -11,7 +11,6 @@ use distribution_types::{ IndexLocations, InstalledMetadata, LocalDist, LocalEditable, LocalEditables, Name, ResolvedDist, }; use install_wheel_rs::linker::LinkMode; - use platform_tags::Tags; use pypi_types::Yanked; use requirements_txt::EditableRequirement; @@ -27,7 +26,7 @@ use uv_configuration::{KeyringProviderType, TargetTriple}; use uv_dispatch::BuildDispatch; use uv_fs::Simplified; use uv_installer::{is_dynamic, Downloader, Plan, Planner, ResolvedEditable, SitePackages}; -use uv_interpreter::{Interpreter, PythonEnvironment}; +use uv_interpreter::{Interpreter, PythonEnvironment, Target}; use uv_requirements::{ ExtrasSpecification, NamedRequirementsResolver, RequirementsSource, RequirementsSpecification, SourceTreeResolver, @@ -64,6 +63,7 @@ pub(crate) async fn pip_sync( python: Option, system: bool, break_system_packages: bool, + target: Option, native_tls: bool, cache: Cache, printer: Printer, @@ -113,6 +113,14 @@ pub(crate) async fn pip_sync( venv.python_executable().user_display().cyan() ); + // Apply any `--target` directory. + let venv = if let Some(target) = target { + target.init()?; + venv.with_target(target) + } else { + venv + }; + // If the environment is externally managed, abort. if let Some(externally_managed) = venv.interpreter().is_externally_managed() { if break_system_packages { diff --git a/crates/uv/src/commands/pip_uninstall.rs b/crates/uv/src/commands/pip_uninstall.rs index 633237e2b..93e7f9b58 100644 --- a/crates/uv/src/commands/pip_uninstall.rs +++ b/crates/uv/src/commands/pip_uninstall.rs @@ -12,11 +12,11 @@ use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity}; use uv_configuration::KeyringProviderType; use uv_fs::Simplified; -use uv_interpreter::PythonEnvironment; +use uv_interpreter::{PythonEnvironment, Target}; +use uv_requirements::{RequirementsSource, RequirementsSpecification}; use crate::commands::{elapsed, ExitStatus}; use crate::printer::Printer; -use uv_requirements::{RequirementsSource, RequirementsSpecification}; /// Uninstall packages from the current environment. #[allow(clippy::too_many_arguments)] @@ -25,6 +25,7 @@ pub(crate) async fn pip_uninstall( python: Option, system: bool, break_system_packages: bool, + target: Option, cache: Cache, connectivity: Connectivity, native_tls: bool, @@ -54,6 +55,14 @@ pub(crate) async fn pip_uninstall( venv.python_executable().user_display().cyan(), ); + // Apply any `--target` directory. + let venv = if let Some(target) = target { + target.init()?; + venv.with_target(target) + } else { + venv + }; + // If the environment is externally managed, abort. if let Some(externally_managed) = venv.interpreter().is_externally_managed() { if break_system_packages { diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index 945b28fd0..34cccdf60 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -270,6 +270,7 @@ async fn run() -> Result { args.shared.python, args.shared.system, args.shared.break_system_packages, + args.shared.target, globals.native_tls, cache, printer, @@ -334,6 +335,7 @@ async fn run() -> Result { args.shared.python, args.shared.system, args.shared.break_system_packages, + args.shared.target, globals.native_tls, cache, args.dry_run, @@ -362,6 +364,7 @@ async fn run() -> Result { args.shared.python, args.shared.system, args.shared.break_system_packages, + args.shared.target, cache, args.shared.connectivity, globals.native_tls, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index d458ff214..745c6767d 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -9,6 +9,7 @@ use uv_configuration::{ ConfigSettings, IndexStrategy, KeyringProviderType, NoBinary, NoBuild, PreviewMode, Reinstall, SetupPyStrategy, TargetTriple, Upgrade, }; +use uv_interpreter::Target; use uv_normalize::PackageName; use uv_requirements::ExtrasSpecification; use uv_resolver::{AnnotationStyle, DependencyMode, ExcludeNewer, PreReleaseMode, ResolutionMode}; @@ -302,6 +303,7 @@ impl PipSyncSettings { no_system, break_system_packages, no_break_system_packages, + target, legacy_setup_py, no_legacy_setup_py, no_build_isolation, @@ -332,6 +334,7 @@ impl PipSyncSettings { python, system: flag(system, no_system), break_system_packages: flag(break_system_packages, no_break_system_packages), + target, offline: flag(offline, no_offline), index_url: index_url.and_then(Maybe::into_option), extra_index_url: extra_index_url.map(|extra_index_urls| { @@ -423,6 +426,7 @@ impl PipInstallSettings { no_system, break_system_packages, no_break_system_packages, + target, legacy_setup_py, no_legacy_setup_py, no_build_isolation, @@ -463,6 +467,7 @@ impl PipInstallSettings { python, system: flag(system, no_system), break_system_packages: flag(break_system_packages, no_break_system_packages), + target, offline: flag(offline, no_offline), index_url: index_url.and_then(Maybe::into_option), extra_index_url: extra_index_url.map(|extra_index_urls| { @@ -530,6 +535,7 @@ impl PipUninstallSettings { no_system, break_system_packages, no_break_system_packages, + target, offline, no_offline, } = args; @@ -545,6 +551,7 @@ impl PipUninstallSettings { python, system: flag(system, no_system), break_system_packages: flag(break_system_packages, no_break_system_packages), + target, offline: flag(offline, no_offline), keyring_provider, ..PipOptions::default() @@ -801,6 +808,7 @@ pub(crate) struct PipSharedSettings { pub(crate) system: bool, pub(crate) extras: ExtrasSpecification, pub(crate) break_system_packages: bool, + pub(crate) target: Option, pub(crate) connectivity: Connectivity, pub(crate) index_strategy: IndexStrategy, pub(crate) keyring_provider: KeyringProviderType, @@ -840,6 +848,7 @@ impl PipSharedSettings { python, system, break_system_packages, + target, offline, index_url, extra_index_url, @@ -955,6 +964,7 @@ impl PipSharedSettings { .break_system_packages .or(break_system_packages) .unwrap_or_default(), + target: args.target.or(target).map(Target::from), no_binary: NoBinary::from_args(args.no_binary.or(no_binary).unwrap_or_default()), compile_bytecode: args .compile_bytecode diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index 2ab2d634a..fa5f5b7fc 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -4808,3 +4808,95 @@ fn require_hashes_registry_invalid_hash() -> Result<()> { Ok(()) } + +/// Sync to a `--target` directory. +#[test] +fn target() -> Result<()> { + let context = TestContext::new("3.12"); + + // Install `iniconfig` to the target directory. + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("iniconfig==2.0.0")?; + + uv_snapshot!(command(&context) + .arg("requirements.in") + .arg("--target") + .arg("target"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + iniconfig==2.0.0 + "###); + + // Ensure that the package is present in the target directory. + assert!(context.temp_dir.child("target").child("iniconfig").is_dir()); + + // Ensure that we can't import the package. + context.assert_command("import iniconfig").failure(); + + // Ensure that we can import the package by augmenting the `PYTHONPATH`. + Command::new(venv_to_interpreter(&context.venv)) + .arg("-B") + .arg("-c") + .arg("import iniconfig") + .env("PYTHONPATH", context.temp_dir.child("target").path()) + .current_dir(&context.temp_dir) + .assert() + .success(); + + // Upgrade it. + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("iniconfig==1.1.1")?; + + uv_snapshot!(command(&context) + .arg("requirements.in") + .arg("--target") + .arg("target"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - iniconfig==2.0.0 + + iniconfig==1.1.1 + "###); + + // Remove it, and replace with `flask`, which includes a binary. + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("flask")?; + + uv_snapshot!(command(&context) + .arg("requirements.in") + .arg("--target") + .arg("target"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + + flask==3.0.3 + - iniconfig==1.1.1 + "###); + // Ensure that the binary is present in the target directory. + assert!(context + .temp_dir + .child("target") + .child("bin") + .child(format!("flask{EXE_SUFFIX}")) + .is_file()); + + Ok(()) +} diff --git a/uv.schema.json b/uv.schema.json index 77040fd71..143e6164f 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -489,6 +489,12 @@ "boolean", "null" ] + }, + "target": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": false