Allow `pip-compile` without a venv (#494)

The semantics are a bit unintuitive because `--python-version` is a
preference when looking for a python version without a venv, but if we
don't find that exact version we'll take `python3` and patch the
markers. This will make more sense once we start provisioning python
builds.

We can now resolve black with both python 3.8 and 3.12, with or without
that python version being in scope. In the example below,
`PATH=$HOME/.cargo/bin:/usr/bin` removes the pyenv builds and leaves
only `python3`, which is python 3.11.

```console
$ RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38
    0.004108s DEBUG puffin::commands::pip_compile Using Python 3.8 at /home/konsti/.local/bin/python3.8
Resolved 8 packages in 44ms
# This file was autogenerated by Puffin v0.0.1 via the following command:
#    puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38
black==23.11.0
[...]
platformdirs==4.0.0
    # via black
tomli==2.0.1
    # via black
typing-extensions==4.8.0
    # via black
$ PATH=$HOME/.cargo/bin:/usr/bin RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38
    0.004315s DEBUG puffin::commands::pip_compile Using Python 3.11 at /usr/bin/python3
Resolved 8 packages in 43ms
# This file was autogenerated by Puffin v0.0.1 via the following command:
#    puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py38
black==23.11.0
[...]
platformdirs==4.0.0
    # via black
tomli==2.0.1
    # via black
typing-extensions==4.8.0
    # via black
```

```console
$ RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312
    0.004216s DEBUG puffin::commands::pip_compile Using Python 3.12 at /home/konsti/.local/bin/python3.12
Resolved 6 packages in 37ms
# This file was autogenerated by Puffin v0.0.1 via the following command:
#    puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312
black==23.11.0
[...]
platformdirs==4.0.0
    # via black
$ PATH=$HOME/.cargo/bin:/usr/bin RUST_LOG=puffin::commands=debug cargo run --bin puffin -q -- pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312
    0.004190s DEBUG puffin::commands::pip_compile Using Python 3.11 at /usr/bin/python3
Resolved 6 packages in 39ms
# This file was autogenerated by Puffin v0.0.1 via the following command:
#    puffin pip-compile -v scripts/benchmarks/requirements/black.in --python-version py312
black==23.11.0
[...]
platformdirs==4.0.0
    # via black
```

Fixes #235.

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>

Cargo.lock

@@ -2624,6 +2624,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
+ "which",
 ]
 
 [[package]]

README.md

@@ -103,15 +103,23 @@ Puffin's `requirements.txt` files may not be portable across platforms and Pytho
 Puffin itself does not depend on Python, but it does need to locate a Python environment to (1)
 install dependencies into the environment, and (2) build source distributions.
 
-When running `pip-install` or `pip-sync`, Puffin will search for a Python environment in the
+When running `pip-sync` or `pip-install`, Puffin will search for a virtual environment in the
 following order:
 
 - An activated virtual environment based on the `VIRTUAL_ENV` environment variable.
 - An activated Conda environment based on the `CONDA_PREFIX` environment variable.
 - A virtual environment at `.venv` in the current directory, or in the nearest parent directory.
 
-If no Python environment is found, Puffin will prompt the user to create a virtual environment in
+If no virtual environment is found, Puffin will prompt the user to create one in the current
-the current directory.
+directory via `puffin venv`.
+
+When running `pip-compile`, Puffin does not _require_ a virtual environment and will search for a
+Python interpreter in the following order:
+
+- An activated virtual environment based on the `VIRTUAL_ENV` environment variable.
+- An activated Conda environment based on the `CONDA_PREFIX` environment variable.
+- A virtual environment at `.venv` in the current directory, or in the nearest parent directory.
+- The Python interpreter available as `python3` on the system path (preferring, e.g., `python3.7` if `--python-version=3.7` is specified).
 
 ### Dependency caching
 

crates/puffin-cli/src/commands/pip_compile.rs

@@ -20,7 +20,7 @@ use puffin_cache::Cache;
 use puffin_client::RegistryClientBuilder;
 use puffin_dispatch::BuildDispatch;
 use puffin_installer::Downloader;
-use puffin_interpreter::Virtualenv;
+use puffin_interpreter::{Interpreter, PythonVersion};
 use puffin_normalize::ExtraName;
 use puffin_resolver::{Manifest, PreReleaseMode, ResolutionMode, ResolutionOptions, Resolver};
 use requirements_txt::EditableRequirement;
@@ -28,7 +28,6 @@ use requirements_txt::EditableRequirement;
 use crate::commands::reporters::{DownloadReporter, ResolverReporter};
 use crate::commands::{elapsed, ExitStatus};
 use crate::printer::Printer;
-use crate::python_version::PythonVersion;
 use crate::requirements::{ExtrasSpecification, RequirementsSource, RequirementsSpecification};
 
 const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -115,20 +114,19 @@ pub(crate) async fn pip_compile(
 
     // Detect the current Python interpreter.
     let platform = Platform::current()?;
-    let venv = Virtualenv::from_env(platform, &cache)?;
+    let interpreter = Interpreter::find(python_version.as_ref(), platform, &cache)?;
 
     debug!(
         "Using Python {} at {}",
-        venv.interpreter().markers().python_version,
+        interpreter.markers().python_version,
-        venv.python_executable().display()
+        interpreter.sys_executable().display()
     );
 
     // Determine the tags, markers, and interpreter to use for resolution.
-    let interpreter = venv.interpreter().clone();
+    let tags = interpreter.tags()?;
-    let tags = venv.interpreter().tags()?;
     let markers = python_version.map_or_else(
-        || Cow::Borrowed(venv.interpreter().markers()),
+        || Cow::Borrowed(interpreter.markers()),
-        |python_version| Cow::Owned(python_version.markers(venv.interpreter().markers())),
+        |python_version| Cow::Owned(python_version.markers(interpreter.markers())),
     );
 
     // Instantiate a client.
@@ -142,7 +140,7 @@ pub(crate) async fn pip_compile(
         &cache,
         &interpreter,
         &index_urls,
-        venv.python_executable(),
+        interpreter.sys_executable().to_path_buf(),
         no_build,
     )
     .with_options(options);
@@ -170,7 +168,7 @@ pub(crate) async fn pip_compile(
         let downloader = Downloader::new(&cache, tags, &client, &build_dispatch)
             .with_reporter(DownloadReporter::from(printer).with_length(editables.len() as u64));
 
-        let editable_wheel_dir = tempdir_in(venv.root())?;
+        let editable_wheel_dir = tempdir_in(cache.root())?;
         let editable_metadata: Vec<_> = downloader
             .build_editables(editables, editable_wheel_dir.path())
             .await

crates/puffin-cli/src/main.rs

@@ -10,12 +10,12 @@ use colored::Colorize;
 use distribution_types::{IndexUrl, IndexUrls};
 use puffin_cache::{Cache, CacheArgs};
 use puffin_installer::Reinstall;
+use puffin_interpreter::PythonVersion;
 use puffin_normalize::{ExtraName, PackageName};
 use puffin_resolver::{PreReleaseMode, ResolutionMode};
 use requirements::ExtrasSpecification;
 
 use crate::commands::{extra_name_with_clap_error, ExitStatus};
-use crate::python_version::PythonVersion;
 use crate::requirements::RequirementsSource;
 
 #[cfg(target_os = "windows")]
@@ -37,7 +37,6 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 mod commands;
 mod logging;
 mod printer;
-mod python_version;
 mod requirements;
 
 #[derive(Parser)]
@@ -179,7 +178,7 @@ struct PipCompileArgs {
     ///
     /// If a patch version is omitted, the most recent known patch version for that minor version
     /// is assumed. For example, `3.7` is mapped to `3.7.17`.
-    #[arg(long, short, value_enum)]
+    #[arg(long, short)]
     python_version: Option<PythonVersion>,
 
     /// Try to resolve at a past time.

crates/puffin-interpreter/Cargo.toml

@@ -29,6 +29,7 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
+which = { workspace = true}
 
 [dev-dependencies]
 indoc = { version = "2.0.4" }

crates/puffin-interpreter/src/interpreter.rs

@@ -15,7 +15,8 @@ use puffin_cache::{Cache, CacheBucket, CachedByTimestamp};
 use puffin_fs::write_atomic_sync;
 
 use crate::python_platform::PythonPlatform;
-use crate::Error;
+use crate::virtual_env::detect_virtual_env;
+use crate::{Error, PythonVersion};
 
 /// A Python executable and its associated platform markers.
 #[derive(Debug, Clone)]
@@ -34,10 +35,15 @@ impl Interpreter {
         let info = InterpreterQueryResult::query_cached(executable, cache)?;
         debug_assert!(
             info.base_prefix == info.base_exec_prefix,
-            "Not a venv python: {}, prefix: {}",
+            "Not a virtual environment (Python: {}, prefix: {})",
             executable.display(),
             info.base_prefix.display()
         );
+        debug_assert!(
+            info.sys_executable.is_absolute(),
+            "`sys.executable` is not an absolute python; Python installation is broken: {}",
+            info.sys_executable.display()
+        );
 
         Ok(Self {
             platform: PythonPlatform(platform),
@@ -76,6 +82,67 @@ impl Interpreter {
         }
     }
 
+    /// Detect the python interpreter to use.
+    ///
+    /// Note that `python_version` is a preference here, not a requirement.
+    ///
+    /// We check, in order:
+    /// * `VIRTUAL_ENV` and `CONDA_PREFIX`
+    /// * A `.venv` folder
+    /// * If a python version is given: `pythonx.y` (TODO(konstin): `py -x.y` on windows),
+    /// * `python3` (unix) or `python.exe` (windows)
+    pub fn find(
+        python_version: Option<&PythonVersion>,
+        platform: Platform,
+        cache: &Cache,
+    ) -> Result<Self, Error> {
+        let platform = PythonPlatform::from(platform);
+        if let Some(venv) = detect_virtual_env(&platform)? {
+            let executable = platform.venv_python(venv);
+            let interpreter = Self::query(&executable, platform.0, cache)?;
+            return Ok(interpreter);
+        };
+
+        #[cfg(unix)]
+        {
+            if let Some(python_version) = python_version {
+                let requested = format!(
+                    "python{}.{}",
+                    python_version.major(),
+                    python_version.minor()
+                );
+                if let Ok(executable) = which::which(&requested) {
+                    debug!("Resolved {requested} to {}", executable.display());
+                    let interpreter = Interpreter::query(&executable, platform.0, cache)?;
+                    return Ok(interpreter);
+                }
+            }
+
+            let executable = which::which("python3")
+                .map_err(|err| Error::WhichNotFound("python3".to_string(), err))?;
+            debug!("Resolved python3 to {}", executable.display());
+            let interpreter = Interpreter::query(&executable, platform.0, cache)?;
+            Ok(interpreter)
+        }
+
+        #[cfg(windows)]
+        {
+            if let Some(python_version) = python_version {
+                compile_error!("Implement me")
+            }
+
+            let executable = which::which("python.exe")
+                .map_err(|err| Error::WhichNotFound("python.exe".to_string(), err))?;
+            let interpreter = Interpreter::query(&executable, platform.0, cache)?;
+            Ok(interpreter)
+        }
+
+        #[cfg(not(any(unix, windows)))]
+        {
+            compile_error!("only unix (like mac and linux) and windows are supported")
+        }
+    }
+
     /// Returns the path to the Python virtual environment.
     pub fn platform(&self) -> &Platform {
         &self.platform

crates/puffin-interpreter/src/lib.rs

@@ -5,11 +5,13 @@ use std::time::SystemTimeError;
 use thiserror::Error;
 
 pub use crate::interpreter::Interpreter;
+pub use crate::python_version::PythonVersion;
 pub use crate::virtual_env::Virtualenv;
 
 mod cfg;
 mod interpreter;
 mod python_platform;
+mod python_version;
 mod virtual_env;
 
 #[derive(Debug, Error)]
@@ -20,6 +22,8 @@ pub enum Error {
     BrokenVenv(PathBuf, PathBuf),
     #[error("Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them.")]
     Conflict,
+    #[error("Could not find `{0}` in PATH")]
+    WhichNotFound(String, #[source] which::Error),
     #[error("Failed to locate a virtualenv or Conda environment (checked: `VIRTUAL_ENV`, `CONDA_PREFIX`, and `.venv`). Run `puffin venv` to create a virtual environment.")]
     NotFound,
     #[error(transparent)]

crates/puffin-interpreter/src/python_version.rs

@@ -1,11 +1,12 @@
 use std::str::FromStr;
+
 use tracing::debug;
 
 use pep440_rs::Version;
 use pep508_rs::{MarkerEnvironment, StringVersion};
 
 #[derive(Debug, Clone)]
-pub(crate) struct PythonVersion(StringVersion);
+pub struct PythonVersion(StringVersion);
 
 impl FromStr for PythonVersion {
     type Err = String;
@@ -65,7 +66,7 @@ impl PythonVersion {
     ///
     /// The returned [`MarkerEnvironment`] will preserve the base environment's platform markers,
     /// but override its Python version markers.
-    pub(crate) fn markers(self, base: &MarkerEnvironment) -> MarkerEnvironment {
+    pub fn markers(self, base: &MarkerEnvironment) -> MarkerEnvironment {
         let mut markers = base.clone();
 
         // Ex) `implementation_version == "3.12.0"`
@@ -81,4 +82,14 @@ impl PythonVersion {
 
         markers
     }
+
+    /// Return the major version of this Python version.
+    pub fn major(&self) -> u64 {
+        self.0.release()[0]
+    }
+
+    /// Return the minor version of this Python version.
+    pub fn minor(&self) -> u64 {
+        self.0.release()[1]
+    }
 }