//! Build wheels from source distributions.
//!
//!
mod error;
use std::ffi::OsString;
use std::fmt::Formatter;
use std::fmt::Write;
use std::io;
use std::path::{Path, PathBuf};
use std::process::ExitStatus;
use std::rc::Rc;
use std::str::FromStr;
use std::sync::LazyLock;
use std::{env, iter};
use fs_err as fs;
use indoc::formatdoc;
use itertools::Itertools;
use rustc_hash::FxHashMap;
use serde::de::{value, IntoDeserializer, SeqAccess, Visitor};
use serde::{de, Deserialize, Deserializer};
use tempfile::TempDir;
use tokio::io::AsyncBufReadExt;
use tokio::process::Command;
use tokio::sync::{Mutex, Semaphore};
use tracing::{debug, info_span, instrument, Instrument};
use uv_configuration::{BuildKind, BuildOutput, ConfigSettings, SourceStrategy};
use uv_distribution::BuildRequires;
use uv_distribution_types::{IndexLocations, Requirement, Resolution};
use uv_fs::{PythonExt, Simplified};
use uv_pep440::Version;
use uv_pep508::PackageName;
use uv_pypi_types::VerbatimParsedUrl;
use uv_python::{Interpreter, PythonEnvironment};
use uv_static::EnvVars;
use uv_types::{AnyErrorBuild, BuildContext, BuildIsolation, BuildStack, SourceBuildTrait};
use uv_warnings::warn_user_once;
use uv_workspace::WorkspaceCache;
pub use crate::error::{Error, MissingHeaderCause};
/// The default backend to use when PEP 517 is used without a `build-system` section.
static DEFAULT_BACKEND: LazyLock = LazyLock::new(|| Pep517Backend {
backend: "setuptools.build_meta:__legacy__".to_string(),
backend_path: None,
requirements: vec![Requirement::from(
uv_pep508::Requirement::from_str("setuptools >= 40.8.0").unwrap(),
)],
});
/// A `pyproject.toml` as specified in PEP 517.
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
struct PyProjectToml {
/// Build-related data
build_system: Option,
/// Project metadata
project: Option,
/// Tool configuration
tool: Option,
}
/// The `[project]` section of a pyproject.toml as specified in PEP 621.
///
/// This representation only includes a subset of the fields defined in PEP 621 necessary for
/// informing wheel builds.
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
struct Project {
/// The name of the project
name: PackageName,
/// The version of the project as supported by PEP 440
version: Option,
/// Specifies which fields listed by PEP 621 were intentionally unspecified so another tool
/// can/will provide such metadata dynamically.
dynamic: Option>,
}
/// The `[build-system]` section of a pyproject.toml as specified in PEP 517.
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
struct BuildSystem {
/// PEP 508 dependencies required to execute the build system.
requires: Vec>,
/// A string naming a Python object that will be used to perform the build.
build_backend: Option,
/// Specify that their backend code is hosted in-tree, this key contains a list of directories.
backend_path: Option,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
struct Tool {
uv: Option,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
struct ToolUv {
workspace: Option,
}
impl BackendPath {
/// Return an iterator over the paths in the backend path.
fn iter(&self) -> impl Iterator {
self.0.iter().map(String::as_str)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct BackendPath(Vec);
impl<'de> Deserialize<'de> for BackendPath {
fn deserialize(deserializer: D) -> Result
where
D: Deserializer<'de>,
{
struct StringOrVec;
impl<'de> Visitor<'de> for StringOrVec {
type Value = Vec;
fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
formatter.write_str("list of strings")
}
fn visit_str(self, s: &str) -> Result
where
E: de::Error,
{
// Allow exactly `backend-path = "."`, as used in `flit_core==2.3.0`.
if s == "." {
Ok(vec![".".to_string()])
} else {
Err(de::Error::invalid_value(de::Unexpected::Str(s), &self))
}
}
fn visit_seq(self, seq: S) -> Result
where
S: SeqAccess<'de>,
{
Deserialize::deserialize(value::SeqAccessDeserializer::new(seq))
}
}
deserializer.deserialize_any(StringOrVec).map(BackendPath)
}
}
/// `[build-backend]` from pyproject.toml
#[derive(Debug, Clone, PartialEq, Eq)]
struct Pep517Backend {
/// The build backend string such as `setuptools.build_meta:__legacy__` or `maturin` from
/// `build-backend.backend` in pyproject.toml
///
///
backend: String,
/// `build-backend.requirements` in pyproject.toml
requirements: Vec,
///
backend_path: Option,
}
impl Pep517Backend {
fn backend_import(&self) -> String {
let import = if let Some((path, object)) = self.backend.split_once(':') {
format!("from {path} import {object} as backend")
} else {
format!("import {} as backend", self.backend)
};
let backend_path_encoded = self
.backend_path
.iter()
.flat_map(BackendPath::iter)
.map(|path| {
// Turn into properly escaped python string
'"'.to_string()
+ &path.replace('\\', "\\\\").replace('"', "\\\"")
+ &'"'.to_string()
})
.join(", ");
// > Projects can specify that their backend code is hosted in-tree by including the
// > backend-path key in pyproject.toml. This key contains a list of directories, which the
// > frontend will add to the start of sys.path when loading the backend, and running the
// > backend hooks.
formatdoc! {r#"
import sys
if sys.path[0] == "":
sys.path.pop(0)
sys.path = [{backend_path}] + sys.path
{import}
"#, backend_path = backend_path_encoded}
}
}
/// Uses an [`Rc`] internally, clone freely.
#[derive(Debug, Default, Clone)]
pub struct SourceBuildContext {
/// An in-memory resolution of the default backend's requirements for PEP 517 builds.
default_resolution: Rc>>,
}
/// Holds the state through a series of PEP 517 frontend to backend calls or a single `setup.py`
/// invocation.
///
/// This keeps both the temp dir and the result of a potential `prepare_metadata_for_build_wheel`
/// call which changes how we call `build_wheel`.
pub struct SourceBuild {
temp_dir: TempDir,
source_tree: PathBuf,
config_settings: ConfigSettings,
/// If performing a PEP 517 build, the backend to use.
pep517_backend: Pep517Backend,
/// The PEP 621 project metadata, if any.
project: Option,
/// The virtual environment in which to build the source distribution.
venv: PythonEnvironment,
/// Populated if `prepare_metadata_for_build_wheel` was called.
///
/// > If the build frontend has previously called `prepare_metadata_for_build_wheel` and depends
/// > on the wheel resulting from this call to have metadata matching this earlier call, then
/// > it should provide the path to the created .dist-info directory as the `metadata_directory`
/// > argument. If this argument is provided, then `build_wheel` MUST produce a wheel with
/// > identical metadata. The directory passed in by the build frontend MUST be identical to the
/// > directory created by `prepare_metadata_for_build_wheel`, including any unrecognized files
/// > it created.
metadata_directory: Option,
/// The name of the package, if known.
package_name: Option,
/// The version of the package, if known.
package_version: Option,
/// Distribution identifier, e.g., `foo-1.2.3`. Used for error reporting if the name and
/// version are unknown.
version_id: Option,
/// Whether we do a regular PEP 517 build or an PEP 660 editable build
build_kind: BuildKind,
/// Whether to send build output to `stderr` or `tracing`, etc.
level: BuildOutput,
/// Modified PATH that contains the `venv_bin`, `user_path` and `system_path` variables in that order
modified_path: OsString,
/// Environment variables to be passed in during metadata or wheel building
environment_variables: FxHashMap,
/// Runner for Python scripts.
runner: PythonRunner,
}
impl SourceBuild {
/// Create a virtual environment in which to build a source distribution, extracting the
/// contents from an archive if necessary.
///
/// `source_dist` is for error reporting only.
pub async fn setup(
source: &Path,
subdirectory: Option<&Path>,
install_path: &Path,
fallback_package_name: Option<&PackageName>,
fallback_package_version: Option<&Version>,
interpreter: &Interpreter,
build_context: &impl BuildContext,
source_build_context: SourceBuildContext,
version_id: Option<&str>,
locations: &IndexLocations,
source_strategy: SourceStrategy,
workspace_cache: &WorkspaceCache,
config_settings: ConfigSettings,
build_isolation: BuildIsolation<'_>,
build_stack: &BuildStack,
build_kind: BuildKind,
mut environment_variables: FxHashMap,
level: BuildOutput,
concurrent_builds: usize,
) -> Result {
let temp_dir = build_context.cache().venv_dir()?;
let source_tree = if let Some(subdir) = subdirectory {
source.join(subdir)
} else {
source.to_path_buf()
};
let default_backend: Pep517Backend = DEFAULT_BACKEND.clone();
// Check if we have a PEP 517 build backend.
let (pep517_backend, project) = Self::extract_pep517_backend(
&source_tree,
install_path,
fallback_package_name,
locations,
source_strategy,
workspace_cache,
&default_backend,
)
.await
.map_err(|err| *err)?;
let package_name = project
.as_ref()
.map(|project| &project.name)
.or(fallback_package_name)
.cloned();
let package_version = project
.as_ref()
.and_then(|project| project.version.as_ref())
.or(fallback_package_version)
.cloned();
// Create a virtual environment, or install into the shared environment if requested.
let venv = if let Some(venv) = build_isolation.shared_environment(package_name.as_ref()) {
venv.clone()
} else {
uv_virtualenv::create_venv(
temp_dir.path(),
interpreter.clone(),
uv_virtualenv::Prompt::None,
false,
false,
false,
false,
)?
};
// Set up the build environment. If build isolation is disabled, we assume the build
// environment is already setup.
if build_isolation.is_isolated(package_name.as_ref()) {
debug!("Resolving build requirements");
let resolved_requirements = Self::get_resolved_requirements(
build_context,
source_build_context,
&default_backend,
&pep517_backend,
build_stack,
)
.await?;
build_context
.install(&resolved_requirements, &venv, build_stack)
.await
.map_err(|err| Error::RequirementsInstall("`build-system.requires`", err.into()))?;
} else {
debug!("Proceeding without build isolation");
}
// Figure out what the modified path should be, and remove the PATH variable from the
// environment variables if it's there.
let user_path = environment_variables.remove(&OsString::from(EnvVars::PATH));
// See if there is an OS PATH variable.
let os_path = env::var_os(EnvVars::PATH);
// Prepend the user supplied PATH to the existing OS PATH
let modified_path = if let Some(user_path) = user_path {
match os_path {
// Prepend the user supplied PATH to the existing PATH
Some(env_path) => {
let user_path = PathBuf::from(user_path);
let new_path = env::split_paths(&user_path).chain(env::split_paths(&env_path));
Some(env::join_paths(new_path).map_err(Error::BuildScriptPath)?)
}
// Use the user supplied PATH
None => Some(user_path),
}
} else {
os_path
};
// Prepend the venv bin directory to the modified path
let modified_path = if let Some(path) = modified_path {
let venv_path = iter::once(venv.scripts().to_path_buf()).chain(env::split_paths(&path));
env::join_paths(venv_path).map_err(Error::BuildScriptPath)?
} else {
OsString::from(venv.scripts())
};
// Create the PEP 517 build environment. If build isolation is disabled, we assume the build
// environment is already setup.
let runner = PythonRunner::new(concurrent_builds, level);
if build_isolation.is_isolated(package_name.as_ref()) {
debug!("Creating PEP 517 build environment");
create_pep517_build_environment(
&runner,
&source_tree,
install_path,
&venv,
&pep517_backend,
build_context,
package_name.as_ref(),
package_version.as_ref(),
version_id,
locations,
source_strategy,
workspace_cache,
build_stack,
build_kind,
level,
&config_settings,
&environment_variables,
&modified_path,
&temp_dir,
)
.await?;
}
Ok(Self {
temp_dir,
source_tree,
pep517_backend,
project,
venv,
build_kind,
level,
config_settings,
metadata_directory: None,
package_name,
package_version,
version_id: version_id.map(ToString::to_string),
environment_variables,
modified_path,
runner,
})
}
async fn get_resolved_requirements(
build_context: &impl BuildContext,
source_build_context: SourceBuildContext,
default_backend: &Pep517Backend,
pep517_backend: &Pep517Backend,
build_stack: &BuildStack,
) -> Result {
Ok(
if pep517_backend.requirements == default_backend.requirements {
let mut resolution = source_build_context.default_resolution.lock().await;
if let Some(resolved_requirements) = &*resolution {
resolved_requirements.clone()
} else {
let resolved_requirements = build_context
.resolve(&default_backend.requirements, build_stack)
.await
.map_err(|err| {
Error::RequirementsResolve("`setup.py` build", err.into())
})?;
*resolution = Some(resolved_requirements.clone());
resolved_requirements
}
} else {
build_context
.resolve(&pep517_backend.requirements, build_stack)
.await
.map_err(|err| {
Error::RequirementsResolve("`build-system.requires`", err.into())
})?
},
)
}
/// Extract the PEP 517 backend from the `pyproject.toml` or `setup.py` file.
async fn extract_pep517_backend(
source_tree: &Path,
install_path: &Path,
package_name: Option<&PackageName>,
locations: &IndexLocations,
source_strategy: SourceStrategy,
workspace_cache: &WorkspaceCache,
default_backend: &Pep517Backend,
) -> Result<(Pep517Backend, Option), Box> {
match fs::read_to_string(source_tree.join("pyproject.toml")) {
Ok(toml) => {
let pyproject_toml: toml_edit::ImDocument<_> =
toml_edit::ImDocument::from_str(&toml)
.map_err(Error::InvalidPyprojectTomlSyntax)?;
let pyproject_toml: PyProjectToml =
PyProjectToml::deserialize(pyproject_toml.into_deserializer())
.map_err(Error::InvalidPyprojectTomlSchema)?;
let backend = if let Some(build_system) = pyproject_toml.build_system {
// If necessary, lower the requirements.
let requirements = match source_strategy {
SourceStrategy::Enabled => {
if let Some(name) = pyproject_toml
.project
.as_ref()
.map(|project| &project.name)
.or(package_name)
{
let build_requires = uv_pypi_types::BuildRequires {
name: Some(name.clone()),
requires_dist: build_system.requires,
};
let build_requires = BuildRequires::from_project_maybe_workspace(
build_requires,
install_path,
locations,
source_strategy,
workspace_cache,
)
.await
.map_err(Error::Lowering)?;
build_requires.requires_dist
} else {
build_system
.requires
.into_iter()
.map(Requirement::from)
.collect()
}
}
SourceStrategy::Disabled => build_system
.requires
.into_iter()
.map(Requirement::from)
.collect(),
};
Pep517Backend {
// If `build-backend` is missing, inject the legacy setuptools backend, but
// retain the `requires`, to match `pip` and `build`. Note that while PEP 517
// says that in this case we "should revert to the legacy behaviour of running
// `setup.py` (either directly, or by implicitly invoking the
// `setuptools.build_meta:__legacy__` backend)", we found that in practice, only
// the legacy setuptools backend is allowed. See also:
// https://github.com/pypa/build/blob/de5b44b0c28c598524832dff685a98d5a5148c44/src/build/__init__.py#L114-L118
backend: build_system
.build_backend
.unwrap_or_else(|| "setuptools.build_meta:__legacy__".to_string()),
backend_path: build_system.backend_path,
requirements,
}
} else {
// If a `pyproject.toml` is present, but `[build-system]` is missing, proceed
// with a PEP 517 build using the default backend (`setuptools`), to match `pip`
// and `build`.
//
// If there is no build system defined and there is no metadata source for
// `setuptools`, warn. The build will succeed, but the metadata will be
// incomplete (for example, the package name will be `UNKNOWN`).
if pyproject_toml.project.is_none()
&& !source_tree.join("setup.py").is_file()
&& !source_tree.join("setup.cfg").is_file()
{
// Give a specific hint for `uv pip install .` in a workspace root.
let looks_like_workspace_root = pyproject_toml
.tool
.as_ref()
.and_then(|tool| tool.uv.as_ref())
.and_then(|tool| tool.workspace.as_ref())
.is_some();
if looks_like_workspace_root {
warn_user_once!(
"`{}` appears to be a workspace root without a Python project; \
consider using `uv sync` to install the workspace, or add a \
`[build-system]` table to `pyproject.toml`",
source_tree.simplified_display().cyan(),
);
} else {
warn_user_once!(
"`{}` does not appear to be a Python project, as the `pyproject.toml` \
does not include a `[build-system]` table, and neither `setup.py` \
nor `setup.cfg` are present in the directory",
source_tree.simplified_display().cyan(),
);
}
}
default_backend.clone()
};
Ok((backend, pyproject_toml.project))
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {
// We require either a `pyproject.toml` or a `setup.py` file at the top level.
if !source_tree.join("setup.py").is_file() {
return Err(Box::new(Error::InvalidSourceDist(
source_tree.to_path_buf(),
)));
}
// If no `pyproject.toml` is present, by default, proceed with a PEP 517 build using
// the default backend, to match `build`. `pip` uses `setup.py` directly in this
// case, but plans to make PEP 517 builds the default in the future.
// See: https://github.com/pypa/pip/issues/9175.
Ok((default_backend.clone(), None))
}
Err(err) => Err(Box::new(err.into())),
}
}
/// Try calling `prepare_metadata_for_build_wheel` to get the metadata without executing the
/// actual build.
pub async fn get_metadata_without_build(&mut self) -> Result