From 673bece5950b0c1f4d5537774209cf8cd69abd2e Mon Sep 17 00:00:00 2001 From: konsti Date: Fri, 5 Jan 2024 16:01:06 +0100 Subject: [PATCH] Allow `pip-compile` without a venv (#494) The semantics are a bit unintuitive because `--python-version` is a preference when looking for a python version without a venv, but if we don't find that exact version we'll take `python3` and patch the markers. This will make more sense once we start provisioning python builds. We can now resolve black with both python 3.8 and 3.12, with or without that python version being in scope. In the example below, `PATH=$HOME/.cargo/bin:/usr/bin` removes the pyenv builds and leaves only `python3`, which is python 3.11. ```console $ RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38 0.004108s DEBUG puffin::commands::pip_compile Using Python 3.8 at /home/konsti/.local/bin/python3.8 Resolved 8 packages in 44ms # This file was autogenerated by Puffin v0.0.1 via the following command: # puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38 black==23.11.0 [...] platformdirs==4.0.0 # via black tomli==2.0.1 # via black typing-extensions==4.8.0 # via black $ PATH=$HOME/.cargo/bin:/usr/bin RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38 0.004315s DEBUG puffin::commands::pip_compile Using Python 3.11 at /usr/bin/python3 Resolved 8 packages in 43ms # This file was autogenerated by Puffin v0.0.1 via the following command: # puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38 black==23.11.0 [...] platformdirs==4.0.0 # via black tomli==2.0.1 # via black typing-extensions==4.8.0 # via black ``` ```console $ RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312 0.004216s DEBUG puffin::commands::pip_compile Using Python 3.12 at /home/konsti/.local/bin/python3.12 Resolved 6 packages in 37ms # This file was autogenerated by Puffin v0.0.1 via the following command: # puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312 black==23.11.0 [...] platformdirs==4.0.0 # via black $ PATH=$HOME/.cargo/bin:/usr/bin RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312 0.004190s DEBUG puffin::commands::pip_compile Using Python 3.11 at /usr/bin/python3 Resolved 6 packages in 39ms # This file was autogenerated by Puffin v0.0.1 via the following command: # puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312 black==23.11.0 [...] platformdirs==4.0.0 # via black ``` Fixes #235. Co-authored-by: Charlie Marsh --- Cargo.lock | 1 + README.md | 14 +++- crates/puffin-cli/src/commands/pip_compile.rs | 20 +++--- crates/puffin-cli/src/main.rs | 5 +- crates/puffin-interpreter/Cargo.toml | 1 + crates/puffin-interpreter/src/interpreter.rs | 71 ++++++++++++++++++- crates/puffin-interpreter/src/lib.rs | 4 ++ .../src/python_version.rs | 15 +++- 8 files changed, 110 insertions(+), 21 deletions(-) rename crates/{puffin-cli => puffin-interpreter}/src/python_version.rs (88%) diff --git a/Cargo.lock b/Cargo.lock index 17c62e497..31f5c6e1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2624,6 +2624,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "which", ] [[package]] diff --git a/README.md b/README.md index 0dbce8222..a55d61d7f 100644 --- a/README.md +++ b/README.md @@ -103,15 +103,23 @@ Puffin's `requirements.txt` files may not be portable across platforms and Pytho Puffin itself does not depend on Python, but it does need to locate a Python environment to (1) install dependencies into the environment, and (2) build source distributions. -When running `pip-install` or `pip-sync`, Puffin will search for a Python environment in the +When running `pip-sync` or `pip-install`, Puffin will search for a virtual environment in the following order: - An activated virtual environment based on the `VIRTUAL_ENV` environment variable. - An activated Conda environment based on the `CONDA_PREFIX` environment variable. - A virtual environment at `.venv` in the current directory, or in the nearest parent directory. -If no Python environment is found, Puffin will prompt the user to create a virtual environment in -the current directory. +If no virtual environment is found, Puffin will prompt the user to create one in the current +directory via `puffin venv`. + +When running `pip-compile`, Puffin does not _require_ a virtual environment and will search for a +Python interpreter in the following order: + +- An activated virtual environment based on the `VIRTUAL_ENV` environment variable. +- An activated Conda environment based on the `CONDA_PREFIX` environment variable. +- A virtual environment at `.venv` in the current directory, or in the nearest parent directory. +- The Python interpreter available as `python3` on the system path (preferring, e.g., `python3.7` if `--python-version=3.7` is specified). ### Dependency caching diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index 93aee5318..336eb0267 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -20,7 +20,7 @@ use puffin_cache::Cache; use puffin_client::RegistryClientBuilder; use puffin_dispatch::BuildDispatch; use puffin_installer::Downloader; -use puffin_interpreter::Virtualenv; +use puffin_interpreter::{Interpreter, PythonVersion}; use puffin_normalize::ExtraName; use puffin_resolver::{Manifest, PreReleaseMode, ResolutionMode, ResolutionOptions, Resolver}; use requirements_txt::EditableRequirement; @@ -28,7 +28,6 @@ use requirements_txt::EditableRequirement; use crate::commands::reporters::{DownloadReporter, ResolverReporter}; use crate::commands::{elapsed, ExitStatus}; use crate::printer::Printer; -use crate::python_version::PythonVersion; use crate::requirements::{ExtrasSpecification, RequirementsSource, RequirementsSpecification}; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -115,20 +114,19 @@ pub(crate) async fn pip_compile( // Detect the current Python interpreter. let platform = Platform::current()?; - let venv = Virtualenv::from_env(platform, &cache)?; + let interpreter = Interpreter::find(python_version.as_ref(), platform, &cache)?; debug!( "Using Python {} at {}", - venv.interpreter().markers().python_version, - venv.python_executable().display() + interpreter.markers().python_version, + interpreter.sys_executable().display() ); // Determine the tags, markers, and interpreter to use for resolution. - let interpreter = venv.interpreter().clone(); - let tags = venv.interpreter().tags()?; + let tags = interpreter.tags()?; let markers = python_version.map_or_else( - || Cow::Borrowed(venv.interpreter().markers()), - |python_version| Cow::Owned(python_version.markers(venv.interpreter().markers())), + || Cow::Borrowed(interpreter.markers()), + |python_version| Cow::Owned(python_version.markers(interpreter.markers())), ); // Instantiate a client. @@ -142,7 +140,7 @@ pub(crate) async fn pip_compile( &cache, &interpreter, &index_urls, - venv.python_executable(), + interpreter.sys_executable().to_path_buf(), no_build, ) .with_options(options); @@ -170,7 +168,7 @@ pub(crate) async fn pip_compile( let downloader = Downloader::new(&cache, tags, &client, &build_dispatch) .with_reporter(DownloadReporter::from(printer).with_length(editables.len() as u64)); - let editable_wheel_dir = tempdir_in(venv.root())?; + let editable_wheel_dir = tempdir_in(cache.root())?; let editable_metadata: Vec<_> = downloader .build_editables(editables, editable_wheel_dir.path()) .await diff --git a/crates/puffin-cli/src/main.rs b/crates/puffin-cli/src/main.rs index 500772ded..78e40c588 100644 --- a/crates/puffin-cli/src/main.rs +++ b/crates/puffin-cli/src/main.rs @@ -10,12 +10,12 @@ use colored::Colorize; use distribution_types::{IndexUrl, IndexUrls}; use puffin_cache::{Cache, CacheArgs}; use puffin_installer::Reinstall; +use puffin_interpreter::PythonVersion; use puffin_normalize::{ExtraName, PackageName}; use puffin_resolver::{PreReleaseMode, ResolutionMode}; use requirements::ExtrasSpecification; use crate::commands::{extra_name_with_clap_error, ExitStatus}; -use crate::python_version::PythonVersion; use crate::requirements::RequirementsSource; #[cfg(target_os = "windows")] @@ -37,7 +37,6 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; mod commands; mod logging; mod printer; -mod python_version; mod requirements; #[derive(Parser)] @@ -179,7 +178,7 @@ struct PipCompileArgs { /// /// If a patch version is omitted, the most recent known patch version for that minor version /// is assumed. For example, `3.7` is mapped to `3.7.17`. - #[arg(long, short, value_enum)] + #[arg(long, short)] python_version: Option, /// Try to resolve at a past time. diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index 8317658bd..d5e1d88c4 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -29,6 +29,7 @@ serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +which = { workspace = true} [dev-dependencies] indoc = { version = "2.0.4" } diff --git a/crates/puffin-interpreter/src/interpreter.rs b/crates/puffin-interpreter/src/interpreter.rs index c6152c561..2fd9488dd 100644 --- a/crates/puffin-interpreter/src/interpreter.rs +++ b/crates/puffin-interpreter/src/interpreter.rs @@ -15,7 +15,8 @@ use puffin_cache::{Cache, CacheBucket, CachedByTimestamp}; use puffin_fs::write_atomic_sync; use crate::python_platform::PythonPlatform; -use crate::Error; +use crate::virtual_env::detect_virtual_env; +use crate::{Error, PythonVersion}; /// A Python executable and its associated platform markers. #[derive(Debug, Clone)] @@ -34,10 +35,15 @@ impl Interpreter { let info = InterpreterQueryResult::query_cached(executable, cache)?; debug_assert!( info.base_prefix == info.base_exec_prefix, - "Not a venv python: {}, prefix: {}", + "Not a virtual environment (Python: {}, prefix: {})", executable.display(), info.base_prefix.display() ); + debug_assert!( + info.sys_executable.is_absolute(), + "`sys.executable` is not an absolute python; Python installation is broken: {}", + info.sys_executable.display() + ); Ok(Self { platform: PythonPlatform(platform), @@ -76,6 +82,67 @@ impl Interpreter { } } + /// Detect the python interpreter to use. + /// + /// Note that `python_version` is a preference here, not a requirement. + /// + /// We check, in order: + /// * `VIRTUAL_ENV` and `CONDA_PREFIX` + /// * A `.venv` folder + /// * If a python version is given: `pythonx.y` (TODO(konstin): `py -x.y` on windows), + /// * `python3` (unix) or `python.exe` (windows) + pub fn find( + python_version: Option<&PythonVersion>, + platform: Platform, + cache: &Cache, + ) -> Result { + let platform = PythonPlatform::from(platform); + if let Some(venv) = detect_virtual_env(&platform)? { + let executable = platform.venv_python(venv); + let interpreter = Self::query(&executable, platform.0, cache)?; + return Ok(interpreter); + }; + + #[cfg(unix)] + { + if let Some(python_version) = python_version { + let requested = format!( + "python{}.{}", + python_version.major(), + python_version.minor() + ); + if let Ok(executable) = which::which(&requested) { + debug!("Resolved {requested} to {}", executable.display()); + let interpreter = Interpreter::query(&executable, platform.0, cache)?; + return Ok(interpreter); + } + } + + let executable = which::which("python3") + .map_err(|err| Error::WhichNotFound("python3".to_string(), err))?; + debug!("Resolved python3 to {}", executable.display()); + let interpreter = Interpreter::query(&executable, platform.0, cache)?; + Ok(interpreter) + } + + #[cfg(windows)] + { + if let Some(python_version) = python_version { + compile_error!("Implement me") + } + + let executable = which::which("python.exe") + .map_err(|err| Error::WhichNotFound("python.exe".to_string(), err))?; + let interpreter = Interpreter::query(&executable, platform.0, cache)?; + Ok(interpreter) + } + + #[cfg(not(any(unix, windows)))] + { + compile_error!("only unix (like mac and linux) and windows are supported") + } + } + /// Returns the path to the Python virtual environment. pub fn platform(&self) -> &Platform { &self.platform diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index 149d2db1c..cf060f830 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -5,11 +5,13 @@ use std::time::SystemTimeError; use thiserror::Error; pub use crate::interpreter::Interpreter; +pub use crate::python_version::PythonVersion; pub use crate::virtual_env::Virtualenv; mod cfg; mod interpreter; mod python_platform; +mod python_version; mod virtual_env; #[derive(Debug, Error)] @@ -20,6 +22,8 @@ pub enum Error { BrokenVenv(PathBuf, PathBuf), #[error("Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them.")] Conflict, + #[error("Could not find `{0}` in PATH")] + WhichNotFound(String, #[source] which::Error), #[error("Failed to locate a virtualenv or Conda environment (checked: `VIRTUAL_ENV`, `CONDA_PREFIX`, and `.venv`). Run `puffin venv` to create a virtual environment.")] NotFound, #[error(transparent)] diff --git a/crates/puffin-cli/src/python_version.rs b/crates/puffin-interpreter/src/python_version.rs similarity index 88% rename from crates/puffin-cli/src/python_version.rs rename to crates/puffin-interpreter/src/python_version.rs index a8561b1e6..754558b38 100644 --- a/crates/puffin-cli/src/python_version.rs +++ b/crates/puffin-interpreter/src/python_version.rs @@ -1,11 +1,12 @@ use std::str::FromStr; + use tracing::debug; use pep440_rs::Version; use pep508_rs::{MarkerEnvironment, StringVersion}; #[derive(Debug, Clone)] -pub(crate) struct PythonVersion(StringVersion); +pub struct PythonVersion(StringVersion); impl FromStr for PythonVersion { type Err = String; @@ -65,7 +66,7 @@ impl PythonVersion { /// /// The returned [`MarkerEnvironment`] will preserve the base environment's platform markers, /// but override its Python version markers. - pub(crate) fn markers(self, base: &MarkerEnvironment) -> MarkerEnvironment { + pub fn markers(self, base: &MarkerEnvironment) -> MarkerEnvironment { let mut markers = base.clone(); // Ex) `implementation_version == "3.12.0"` @@ -81,4 +82,14 @@ impl PythonVersion { markers } + + /// Return the major version of this Python version. + pub fn major(&self) -> u64 { + self.0.release()[0] + } + + /// Return the minor version of this Python version. + pub fn minor(&self) -> u64 { + self.0.release()[1] + } }