uv/crates/uv-interpreter/src/find_python.rs

801 lines
30 KiB
Rust

use std::borrow::Cow;
use std::env;
use std::ffi::{OsStr, OsString};
use std::path::PathBuf;
use tracing::{debug, instrument};
use uv_cache::Cache;
use uv_warnings::warn_user_once;
use crate::interpreter::InterpreterInfoError;
use crate::py_launcher::{py_list_paths, Error as PyLauncherError, PyListPath};
use crate::virtualenv::{detect_virtualenv, virtualenv_python_executable};
use crate::PythonVersion;
use crate::{Error, Interpreter};
/// Find a Python of a specific version, a binary with a name or a path to a binary.
///
/// Supported formats:
/// * `-p 3.10` searches for an installed Python 3.10 (`py --list-paths` on Windows, `python3.10` on
/// Linux/Mac). Specifying a patch version is not supported.
/// * `-p python3.10` or `-p python.exe` looks for a binary in `PATH`.
/// * `-p /home/ferris/.local/bin/python3.10` uses this exact Python.
///
/// When the user passes a patch version (e.g. 3.12.1), we currently search for a matching minor
/// version (e.g. `python3.12` on unix) and error when the version mismatches, as a binary with the
/// patch version (e.g. `python3.12.1`) is often not in `PATH` and we make the simplifying
/// assumption that the user has only this one patch version installed.
#[instrument(skip_all, fields(%request))]
pub fn find_requested_python(request: &str, cache: &Cache) -> Result<Option<Interpreter>, Error> {
debug!("Starting interpreter discovery for Python @ `{request}`");
let versions = request
.splitn(3, '.')
.map(str::parse::<u8>)
.collect::<Result<Vec<_>, _>>();
if let Ok(versions) = versions {
// `-p 3.10` or `-p 3.10.1`
let selector = match versions.as_slice() {
[requested_major] => PythonVersionSelector::Major(*requested_major),
[major, minor] => PythonVersionSelector::MajorMinor(*major, *minor),
[major, minor, requested_patch] => {
PythonVersionSelector::MajorMinorPatch(*major, *minor, *requested_patch)
}
// SAFETY: Guaranteed by the Ok(versions) guard
_ => unreachable!(),
};
let interpreter = find_python(selector, cache)?;
interpreter
.as_ref()
.inspect(|inner| warn_on_unsupported_python(inner));
Ok(interpreter)
} else {
match fs_err::metadata(request) {
Ok(metadata) => {
// Map from user-provided path to an executable.
let path = uv_fs::absolutize_path(request.as_ref())?;
let executable = if metadata.is_dir() {
// If the user provided a directory, assume it's a virtual environment.
// `-p /home/ferris/.venv`
if cfg!(windows) {
Cow::Owned(path.join("Scripts/python.exe"))
} else {
Cow::Owned(path.join("bin/python"))
}
} else {
// Otherwise, assume it's a Python executable.
// `-p /home/ferris/.local/bin/python3.10`
path
};
Interpreter::query(executable, cache)
.inspect(warn_on_unsupported_python)
.map(Some)
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
// `-p python3.10`; Generally not used on windows because all Python are `python.exe`.
let Some(executable) = find_executable(request)? else {
return Ok(None);
};
Interpreter::query(executable, cache)
.inspect(warn_on_unsupported_python)
.map(Some)
}
Err(err) => return Err(err.into()),
}
}
}
/// Pick a sensible default for the Python a user wants when they didn't specify a version.
///
/// We prefer the test overwrite `UV_TEST_PYTHON_PATH` if it is set, otherwise `python3`/`python` or
/// `python.exe` respectively.
#[instrument(skip_all)]
pub fn find_default_python(cache: &Cache) -> Result<Interpreter, Error> {
debug!("Starting interpreter discovery for default Python");
try_find_default_python(cache)?
.ok_or(if cfg!(windows) {
Error::NoPythonInstalledWindows
} else if cfg!(unix) {
Error::NoPythonInstalledUnix
} else {
unreachable!("Only Unix and Windows are supported")
})
.inspect(warn_on_unsupported_python)
}
/// Same as [`find_default_python`] but returns `None` if no python is found instead of returning an `Err`.
pub(crate) fn try_find_default_python(cache: &Cache) -> Result<Option<Interpreter>, Error> {
find_python(PythonVersionSelector::Default, cache)
}
/// Find a Python version matching `selector`.
///
/// It searches for an existing installation in the following order:
/// * Search for the python binary in `PATH` (or `UV_TEST_PYTHON_PATH` if set). Visits each path and for each path resolves the
/// files in the following order:
/// * Major.Minor.Patch: `pythonx.y.z`, `pythonx.y`, `python.x`, `python`
/// * Major.Minor: `pythonx.y`, `pythonx`, `python`
/// * Major: `pythonx`, `python`
/// * Default: `python3`, `python`
/// * (windows): For each of the above, test for the existence of `python.bat` shim (pyenv-windows) last.
/// * (windows): Discover installations using `py --list-paths` (PEP514). Continue if `py` is not installed.
///
/// (Windows): Filter out the Windows store shim (Enabled in Settings/Apps/Advanced app settings/App execution aliases).
fn find_python(
selector: PythonVersionSelector,
cache: &Cache,
) -> Result<Option<Interpreter>, Error> {
#[allow(non_snake_case)]
let UV_TEST_PYTHON_PATH = env::var_os("UV_TEST_PYTHON_PATH");
let use_override = UV_TEST_PYTHON_PATH.is_some();
let possible_names = selector.possible_names();
#[allow(non_snake_case)]
let PATH = UV_TEST_PYTHON_PATH
.or(env::var_os("PATH"))
.unwrap_or_default();
// We use `which` here instead of joining the paths ourselves because `which` checks for us if the python
// binary is executable and exists. It also has some extra logic that handles inconsistent casing on Windows
// and expands `~`.
for path in env::split_paths(&PATH) {
for name in possible_names.iter().flatten() {
if let Ok(paths) = which::which_in_global(&**name, Some(&path)) {
for path in paths {
#[cfg(windows)]
if windows::is_windows_store_shim(&path) {
continue;
}
let interpreter = match Interpreter::query(&path, cache) {
Ok(interpreter) => interpreter,
// If the Python version is < 3.4, the `-I` flag is not supported, so
// we can't run the script at all, and need to sniff it from the output.
Err(Error::PythonSubcommandOutput { stderr, .. })
if stderr.contains("Unknown option: -I") =>
{
// If the user _requested_ a version prior to 3.4, raise an error, as
// 3.4 is the minimum supported version for invoking the interpreter
// query script at all.
match selector {
PythonVersionSelector::Major(major) if major < 3 => {
return Err(Error::UnsupportedPython(major.to_string()));
}
PythonVersionSelector::MajorMinor(major, minor)
if (major, minor) < (3, 4) =>
{
return Err(Error::UnsupportedPython(format!(
"{major}.{minor}"
)));
}
PythonVersionSelector::MajorMinorPatch(major, minor, patch)
if (major, minor) < (3, 4) =>
{
return Err(Error::UnsupportedPython(format!(
"{major}.{minor}.{patch}"
)));
}
_ => {}
}
debug!(
"Found a Python installation that isn't supported by uv, skipping."
);
continue;
}
Err(Error::QueryScript {
err: InterpreterInfoError::UnsupportedPythonVersion { .. },
..
}) => {
// If the user _requested_ a version prior to 3.7, raise an error, as
// 3.7 is the minimum supported version for running the interpreter
// query script.
match selector {
PythonVersionSelector::Major(major) if major < 3 => {
return Err(Error::UnsupportedPython(major.to_string()));
}
PythonVersionSelector::MajorMinor(major, minor)
if (major, minor) < (3, 7) =>
{
return Err(Error::UnsupportedPython(format!(
"{major}.{minor}"
)));
}
PythonVersionSelector::MajorMinorPatch(major, minor, patch)
if (major, minor) < (3, 7) =>
{
return Err(Error::UnsupportedPython(format!(
"{major}.{minor}.{patch}"
)));
}
_ => {}
}
debug!(
"Found a Python installation that isn't supported by uv, skipping."
);
continue;
}
Err(error) => return Err(error),
};
let installation = PythonInstallation::Interpreter(interpreter);
if let Some(interpreter) = installation.select(selector, cache)? {
return Ok(Some(interpreter));
}
}
}
}
// Python's `venv` model doesn't have this case because they use the `sys.executable` by default
// which is sufficient to support pyenv-windows. Unfortunately, we can't rely on the executing Python version.
// That's why we explicitly search for a Python shim as last resort.
if cfg!(windows) {
if let Ok(shims) = which::which_in_global("python.bat", Some(&path)) {
for shim in shims {
let interpreter = match Interpreter::query(&shim, cache) {
Ok(interpreter) => interpreter,
Err(error) => {
// Don't fail when querying the shim failed. E.g it's possible that no python version is selected
// in the shim in which case pyenv prints to stdout.
tracing::warn!("Failed to query python shim: {error}");
continue;
}
};
if let Some(interpreter) =
PythonInstallation::Interpreter(interpreter).select(selector, cache)?
{
return Ok(Some(interpreter));
}
}
}
}
}
if cfg!(windows) && !use_override {
// Use `py` to find the python installation on the system.
match py_list_paths() {
Ok(paths) => {
for entry in paths {
let installation = PythonInstallation::PyListPath(entry);
if let Some(interpreter) = installation.select(selector, cache)? {
return Ok(Some(interpreter));
}
}
}
// Do not error when `py` is not available
Err(PyLauncherError::NotFound) => debug!("`py` is not installed"),
Err(error) => return Err(Error::PyLauncher(error)),
}
}
Ok(None)
}
/// Find the Python interpreter in `PATH` matching the given name (e.g., `python3`), respecting
/// `UV_PYTHON_PATH`.
///
/// Returns `Ok(None)` if not found.
fn find_executable<R: AsRef<OsStr> + Into<OsString> + Copy>(
requested: R,
) -> Result<Option<PathBuf>, Error> {
#[allow(non_snake_case)]
let UV_TEST_PYTHON_PATH = env::var_os("UV_TEST_PYTHON_PATH");
let use_override = UV_TEST_PYTHON_PATH.is_some();
#[allow(non_snake_case)]
let PATH = UV_TEST_PYTHON_PATH
.or(env::var_os("PATH"))
.unwrap_or_default();
// We use `which` here instead of joining the paths ourselves because `which` checks for us if the python
// binary is executable and exists. It also has some extra logic that handles inconsistent casing on Windows
// and expands `~`.
for path in env::split_paths(&PATH) {
let paths = match which::which_in_global(requested, Some(&path)) {
Ok(paths) => paths,
Err(which::Error::CannotFindBinaryPath) => continue,
Err(err) => return Err(Error::WhichError(requested.into(), err)),
};
#[allow(clippy::never_loop)]
for path in paths {
#[cfg(windows)]
if windows::is_windows_store_shim(&path) {
continue;
}
return Ok(Some(path));
}
}
if cfg!(windows) && !use_override {
// Use `py` to find the python installation on the system.
match py_list_paths() {
Ok(paths) => {
for entry in paths {
// Ex) `--python python3.12.exe`
if entry.executable_path.file_name() == Some(requested.as_ref()) {
return Ok(Some(entry.executable_path));
}
// Ex) `--python python3.12`
if entry
.executable_path
.file_stem()
.is_some_and(|stem| stem == requested.as_ref())
{
return Ok(Some(entry.executable_path));
}
}
}
// Do not error when `py` is not available
Err(PyLauncherError::NotFound) => debug!("`py` is not installed"),
Err(error) => return Err(Error::PyLauncher(error)),
}
}
Ok(None)
}
#[derive(Debug, Clone)]
enum PythonInstallation {
PyListPath(PyListPath),
Interpreter(Interpreter),
}
impl PythonInstallation {
fn major(&self) -> u8 {
match self {
Self::PyListPath(PyListPath { major, .. }) => *major,
Self::Interpreter(interpreter) => interpreter.python_major(),
}
}
fn minor(&self) -> u8 {
match self {
Self::PyListPath(PyListPath { minor, .. }) => *minor,
Self::Interpreter(interpreter) => interpreter.python_minor(),
}
}
/// Selects the interpreter if it matches the selector (version specification).
fn select(
self,
selector: PythonVersionSelector,
cache: &Cache,
) -> Result<Option<Interpreter>, Error> {
let selected = match selector {
PythonVersionSelector::Default => true,
PythonVersionSelector::Major(major) => self.major() == major,
PythonVersionSelector::MajorMinor(major, minor) => {
self.major() == major && self.minor() == minor
}
PythonVersionSelector::MajorMinorPatch(major, minor, requested_patch) => {
let interpreter = self.into_interpreter(cache)?;
return Ok(
if major == interpreter.python_major()
&& minor == interpreter.python_minor()
&& requested_patch == interpreter.python_patch()
{
Some(interpreter)
} else {
None
},
);
}
};
if selected {
self.into_interpreter(cache).map(Some)
} else {
Ok(None)
}
}
pub(super) fn into_interpreter(self, cache: &Cache) -> Result<Interpreter, Error> {
match self {
Self::PyListPath(PyListPath {
executable_path, ..
}) => Interpreter::query(executable_path, cache),
Self::Interpreter(interpreter) => Ok(interpreter),
}
}
}
#[derive(Copy, Clone, Debug)]
enum PythonVersionSelector {
Default,
Major(u8),
MajorMinor(u8, u8),
MajorMinorPatch(u8, u8, u8),
}
impl PythonVersionSelector {
fn possible_names(self) -> [Option<Cow<'static, str>>; 4] {
let (python, python3, extension) = if cfg!(windows) {
(
Cow::Borrowed("python.exe"),
Cow::Borrowed("python3.exe"),
".exe",
)
} else {
(Cow::Borrowed("python"), Cow::Borrowed("python3"), "")
};
match self {
Self::Default => [Some(python3), Some(python), None, None],
Self::Major(major) => [
Some(Cow::Owned(format!("python{major}{extension}"))),
Some(python),
None,
None,
],
Self::MajorMinor(major, minor) => [
Some(Cow::Owned(format!("python{major}.{minor}{extension}"))),
Some(Cow::Owned(format!("python{major}{extension}"))),
Some(python),
None,
],
Self::MajorMinorPatch(major, minor, patch) => [
Some(Cow::Owned(format!(
"python{major}.{minor}.{patch}{extension}",
))),
Some(Cow::Owned(format!("python{major}.{minor}{extension}"))),
Some(Cow::Owned(format!("python{major}{extension}"))),
Some(python),
],
}
}
}
fn warn_on_unsupported_python(interpreter: &Interpreter) {
// Warn on usage with an unsupported Python version
if interpreter.python_tuple() < (3, 8) {
warn_user_once!(
"uv is only compatible with Python 3.8+, found Python {}.",
interpreter.python_version()
);
}
}
/// Find a matching Python or any fallback Python.
///
/// If no Python version is provided, we will use the first available interpreter.
///
/// If a Python version is provided, we will first try to find an exact match. If
/// that cannot be found and a patch version was requested, we will look for a match
/// without comparing the patch version number. If that cannot be found, we fall back to
/// the first available version.
///
/// See [`Self::find_version`] for details on the precedence of Python lookup locations.
#[instrument(skip_all, fields(?python_version))]
pub fn find_best_python(
python_version: Option<&PythonVersion>,
system: bool,
cache: &Cache,
) -> Result<Interpreter, Error> {
if let Some(python_version) = python_version {
debug!(
"Starting interpreter discovery for Python {}",
python_version
);
} else {
debug!("Starting interpreter discovery for active Python");
}
// First, check for an exact match (or the first available version if no Python version was provided)
if let Some(interpreter) = find_version(python_version, system, cache)? {
warn_on_unsupported_python(&interpreter);
return Ok(interpreter);
}
if let Some(python_version) = python_version {
// If that fails, and a specific patch version was requested try again allowing a
// different patch version
if python_version.patch().is_some() {
if let Some(interpreter) =
find_version(Some(&python_version.without_patch()), system, cache)?
{
warn_on_unsupported_python(&interpreter);
return Ok(interpreter);
}
}
}
// If a Python version was requested but cannot be fulfilled, just take any version
if let Some(interpreter) = find_version(None, system, cache)? {
return Ok(interpreter);
}
Err(Error::PythonNotFound)
}
/// Find a Python interpreter.
///
/// We check, in order, the following locations:
///
/// - `UV_DEFAULT_PYTHON`, which is set to the python interpreter when using `python -m uv`.
/// - `VIRTUAL_ENV` and `CONDA_PREFIX`
/// - A `.venv` folder
/// - If a python version is given: Search `PATH` and `py --list-paths`, see `find_python`
/// - `python3` (unix) or `python.exe` (windows)
///
/// If `UV_TEST_PYTHON_PATH` is set, we will not check for Python versions in the
/// global PATH, instead we will search using the provided path. Virtual environments
/// will still be respected.
///
/// If a version is provided and an interpreter cannot be found with the given version,
/// we will return [`None`].
fn find_version(
python_version: Option<&PythonVersion>,
system: bool,
cache: &Cache,
) -> Result<Option<Interpreter>, Error> {
let version_matches = |interpreter: &Interpreter| -> bool {
if let Some(python_version) = python_version {
// If a patch version was provided, check for an exact match
interpreter.satisfies(python_version)
} else {
// The version always matches if one was not provided
true
}
};
// Check if the venv Python matches.
if !system {
if let Some(venv) = detect_virtualenv()? {
let executable = virtualenv_python_executable(venv);
let interpreter = Interpreter::query(executable, cache)?;
if version_matches(&interpreter) {
return Ok(Some(interpreter));
}
};
}
// Look for the requested version with by search for `python{major}.{minor}` in `PATH` on
// Unix and `py --list-paths` on Windows.
let interpreter = if let Some(python_version) = python_version {
find_requested_python(&python_version.string, cache)?
} else {
try_find_default_python(cache)?
};
if let Some(interpreter) = interpreter {
debug_assert!(version_matches(&interpreter));
Ok(Some(interpreter))
} else {
Ok(None)
}
}
mod windows {
/// On Windows we might encounter the Windows Store proxy shim (enabled in:
/// Settings/Apps/Advanced app settings/App execution aliases). When Python is _not_ installed
/// via the Windows Store, but the proxy shim is enabled, then executing `python.exe` or
/// `python3.exe` will redirect to the Windows Store installer.
///
/// We need to detect that these `python.exe` and `python3.exe` files are _not_ Python
/// executables.
///
/// This method is taken from Rye:
///
/// > This is a pretty dumb way. We know how to parse this reparse point, but Microsoft
/// > does not want us to do this as the format is unstable. So this is a best effort way.
/// > we just hope that the reparse point has the python redirector in it, when it's not
/// > pointing to a valid Python.
///
/// See: <https://github.com/astral-sh/rye/blob/b0e9eccf05fe4ff0ae7b0250a248c54f2d780b4d/rye/src/cli/shim.rs#L108>
#[cfg(windows)]
pub(super) fn is_windows_store_shim(path: &std::path::Path) -> bool {
use std::os::windows::fs::MetadataExt;
use std::os::windows::prelude::OsStrExt;
use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING};
use winapi::um::handleapi::{CloseHandle, INVALID_HANDLE_VALUE};
use winapi::um::ioapiset::DeviceIoControl;
use winapi::um::winbase::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT};
use winapi::um::winioctl::FSCTL_GET_REPARSE_POINT;
use winapi::um::winnt::{FILE_ATTRIBUTE_REPARSE_POINT, MAXIMUM_REPARSE_DATA_BUFFER_SIZE};
// The path must be absolute.
if !path.is_absolute() {
return false;
}
// The path must point to something like:
// `C:\Users\crmar\AppData\Local\Microsoft\WindowsApps\python3.exe`
let mut components = path.components().rev();
// Ex) `python.exe` or `python3.exe`
if !components
.next()
.and_then(|component| component.as_os_str().to_str())
.is_some_and(|component| component == "python.exe" || component == "python3.exe")
{
return false;
}
// Ex) `WindowsApps`
if !components
.next()
.is_some_and(|component| component.as_os_str() == "WindowsApps")
{
return false;
}
// Ex) `Microsoft`
if !components
.next()
.is_some_and(|component| component.as_os_str() == "Microsoft")
{
return false;
}
// The file is only relevant if it's a reparse point.
let Ok(md) = fs_err::symlink_metadata(path) else {
return false;
};
if md.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT == 0 {
return false;
}
let mut path_encoded = path
.as_os_str()
.encode_wide()
.chain(std::iter::once(0))
.collect::<Vec<_>>();
// SAFETY: The path is null-terminated.
#[allow(unsafe_code)]
let reparse_handle = unsafe {
CreateFileW(
path_encoded.as_mut_ptr(),
0,
0,
std::ptr::null_mut(),
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT,
std::ptr::null_mut(),
)
};
if reparse_handle == INVALID_HANDLE_VALUE {
return false;
}
let mut buf = [0u16; MAXIMUM_REPARSE_DATA_BUFFER_SIZE as usize];
let mut bytes_returned = 0;
// SAFETY: The buffer is large enough to hold the reparse point.
#[allow(unsafe_code, clippy::cast_possible_truncation)]
let success = unsafe {
DeviceIoControl(
reparse_handle,
FSCTL_GET_REPARSE_POINT,
std::ptr::null_mut(),
0,
buf.as_mut_ptr().cast(),
buf.len() as u32 * 2,
&mut bytes_returned,
std::ptr::null_mut(),
) != 0
};
// SAFETY: The handle is valid.
#[allow(unsafe_code)]
unsafe {
CloseHandle(reparse_handle);
}
// If the operation failed, assume it's not a reparse point.
if !success {
return false;
}
let reparse_point = String::from_utf16_lossy(&buf[..bytes_returned as usize]);
reparse_point.contains("\\AppInstallerPythonRedirector.exe")
}
#[cfg(test)]
mod tests {
use std::fmt::Debug;
use insta::assert_snapshot;
use itertools::Itertools;
use uv_cache::Cache;
use crate::{find_requested_python, Error};
fn format_err<T: Debug>(err: Result<T, Error>) -> String {
anyhow::Error::new(err.unwrap_err())
.chain()
.join("\n Caused by: ")
}
#[test]
#[cfg_attr(not(windows), ignore)]
fn no_such_python_path() {
let cache = Cache::temp().unwrap().init().unwrap();
let result = find_requested_python(r"C:\does\not\exists\python3.12", &cache)
.unwrap()
.ok_or(Error::RequestedPythonNotFound(
r"C:\does\not\exists\python3.12".to_string(),
));
assert_snapshot!(
format_err(result),
@"Failed to locate Python interpreter at: `C:\\does\\not\\exists\\python3.12`"
);
}
}
}
#[cfg(test)]
mod tests {
use insta::assert_snapshot;
use itertools::Itertools;
use uv_cache::Cache;
use crate::find_python::find_requested_python;
use crate::Error;
fn format_err<T: std::fmt::Debug>(err: Result<T, Error>) -> String {
anyhow::Error::new(err.unwrap_err())
.chain()
.join("\n Caused by: ")
}
#[test]
#[cfg_attr(not(unix), ignore)]
fn no_such_python_version() {
let cache = Cache::temp().unwrap().init().unwrap();
let request = "3.1000";
let result = find_requested_python(request, &cache)
.unwrap()
.ok_or(Error::NoSuchPython(request.to_string()));
assert_snapshot!(
format_err(result),
@"No Python 3.1000 in `PATH`. Is Python 3.1000 installed?"
);
}
#[test]
#[cfg_attr(not(unix), ignore)]
fn no_such_python_binary() {
let cache = Cache::temp().unwrap().init().unwrap();
let request = "python3.1000";
let result = find_requested_python(request, &cache)
.unwrap()
.ok_or(Error::NoSuchPython(request.to_string()));
assert_snapshot!(
format_err(result),
@"No Python python3.1000 in `PATH`. Is Python python3.1000 installed?"
);
}
#[test]
#[cfg_attr(not(unix), ignore)]
fn no_such_python_path() {
let cache = Cache::temp().unwrap().init().unwrap();
let result = find_requested_python("/does/not/exists/python3.12", &cache)
.unwrap()
.ok_or(Error::RequestedPythonNotFound(
"/does/not/exists/python3.12".to_string(),
));
assert_snapshot!(
format_err(result), @"Failed to locate Python interpreter at: `/does/not/exists/python3.12`");
}
}