diff --git a/Cargo.lock b/Cargo.lock index ffd4b5274..77a8e23a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,9 +518,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.10" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -1045,9 +1045,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" [[package]] name = "heck" @@ -1183,7 +1183,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1270,7 +1270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.1", "serde", ] @@ -1350,7 +1350,6 @@ dependencies = [ "reflink-copy", "regex", "serde", - "serde_json", "sha2", "target-lexicon", "tempfile", @@ -2015,22 +2014,14 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", - "colored", - "directories", "flate2", "fs-err", "gourgeist", "indoc 2.0.4", - "itertools", "pep508_rs", "platform-host", "platform-tags", - "puffin-client", - "puffin-installer", - "puffin-interpreter", - "puffin-package", - "puffin-resolver", - "puffin-workspace", + "puffin-traits", "pyproject-toml", "serde", "serde_json", @@ -2040,11 +2031,37 @@ dependencies = [ "tokio", "toml 0.8.2", "tracing", - "tracing-subscriber", "which", "zip", ] +[[package]] +name = "puffin-build-cli" +version = "0.0.1" +dependencies = [ + "anyhow", + "clap", + "colored", + "directories", + "fs-err", + "futures", + "gourgeist", + "itertools", + "pep508_rs", + "platform-host", + "platform-tags", + "puffin-build", + "puffin-client", + "puffin-dispatch", + "puffin-interpreter", + "puffin-package", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "which", +] + [[package]] name = "puffin-cli" version = "0.0.1" @@ -2073,6 +2090,7 @@ dependencies = [ "predicates", "pubgrub", "puffin-client", + "puffin-dispatch", "puffin-installer", "puffin-interpreter", "puffin-package", @@ -2103,10 +2121,32 @@ dependencies = [ "serde", "serde_json", "thiserror", + "tokio", "tracing", "url", ] +[[package]] +name = "puffin-dispatch" +version = "0.1.0" +dependencies = [ + "anyhow", + "gourgeist", + "itertools", + "pep508_rs", + "platform-host", + "platform-tags", + "puffin-build", + "puffin-client", + "puffin-installer", + "puffin-interpreter", + "puffin-package", + "puffin-resolver", + "puffin-traits", + "tempfile", + "tracing", +] + [[package]] name = "puffin-installer" version = "0.0.1" @@ -2117,6 +2157,7 @@ dependencies = [ "fs-err", "install-wheel-rs", "pep440_rs 0.3.12", + "pep508_rs", "puffin-client", "puffin-interpreter", "puffin-package", @@ -2139,7 +2180,6 @@ dependencies = [ "pep440_rs 0.3.12", "pep508_rs", "platform-host", - "puffin-package", "serde_json", "tokio", "tracing", @@ -2178,9 +2218,12 @@ dependencies = [ "clap", "colored", "distribution-filename", + "fs-err", "futures", "fxhash", + "gourgeist", "insta", + "install-wheel-rs", "itertools", "once_cell", "pep440_rs 0.3.12", @@ -2190,11 +2233,28 @@ dependencies = [ "platform-tags", "pubgrub", "puffin-client", + "puffin-interpreter", "puffin-package", + "puffin-traits", + "tempfile", "thiserror", "tokio", + "tokio-util", "tracing", + "url", "waitmap", + "which", + "zip", +] + +[[package]] +name = "puffin-traits" +version = "0.1.0" +dependencies = [ + "anyhow", + "gourgeist", + "pep508_rs", + "puffin-interpreter", ] [[package]] @@ -2581,9 +2641,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.20" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ "bitflags 2.4.1", "errno", @@ -2800,9 +2860,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2810,9 +2870,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2938,9 +2998,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "task-local-extensions" @@ -3037,18 +3097,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", @@ -3130,7 +3190,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] @@ -3249,9 +3309,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" dependencies = [ "log", "pin-project-lite", diff --git a/crates/gourgeist/src/lib.rs b/crates/gourgeist/src/lib.rs index a5f83b471..737cc6265 100644 --- a/crates/gourgeist/src/lib.rs +++ b/crates/gourgeist/src/lib.rs @@ -56,6 +56,7 @@ pub enum Error { } /// Provides the paths inside a venv +#[derive(Debug, Clone)] pub struct Venv(Utf8PathBuf); impl Deref for Venv { diff --git a/crates/install-wheel-rs/Cargo.toml b/crates/install-wheel-rs/Cargo.toml index 0325d4019..22406cb11 100644 --- a/crates/install-wheel-rs/Cargo.toml +++ b/crates/install-wheel-rs/Cargo.toml @@ -38,7 +38,6 @@ rayon = { version = "1.8.0", optional = true } reflink-copy = { workspace = true } regex = { workspace = true } serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } sha2 = { workspace = true } target-lexicon = { workspace = true } tempfile = { workspace = true } diff --git a/crates/install-wheel-rs/src/lib.rs b/crates/install-wheel-rs/src/lib.rs index 832ff8a58..dfef4256b 100644 --- a/crates/install-wheel-rs/src/lib.rs +++ b/crates/install-wheel-rs/src/lib.rs @@ -12,8 +12,8 @@ pub use record::RecordEntry; pub use script::Script; pub use uninstall::{uninstall_wheel, Uninstall}; pub use wheel::{ - get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to, - SHEBANG_PYTHON, + find_dist_info, get_script_launcher, install_wheel, parse_key_value_file, read_record_file, + relative_to, SHEBANG_PYTHON, }; mod install_location; diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index d5c4482ab..3f41ed5cb 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -1026,7 +1026,7 @@ pub fn install_wheel( /// Either way, we just search the wheel for the name /// /// -fn find_dist_info( +pub fn find_dist_info( filename: &WheelFilename, archive: &mut ZipArchive, ) -> Result { diff --git a/crates/pep440-rs/src/version_specifier.rs b/crates/pep440-rs/src/version_specifier.rs index 298a89c17..13fdf0395 100644 --- a/crates/pep440-rs/src/version_specifier.rs +++ b/crates/pep440-rs/src/version_specifier.rs @@ -86,6 +86,12 @@ impl FromStr for VersionSpecifiers { } } +impl From for VersionSpecifiers { + fn from(specifier: VersionSpecifier) -> Self { + Self(vec![specifier]) + } +} + impl Display for VersionSpecifiers { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { for (idx, version_specifier) in self.0.iter().enumerate() { @@ -341,6 +347,14 @@ impl VersionSpecifier { Ok(Self { operator, version }) } + /// `==` + pub fn equals_version(version: Version) -> Self { + Self { + operator: Operator::Equal, + version, + } + } + /// Get the operator, e.g. `>=` in `>= 2.0.0` pub fn operator(&self) -> &Operator { &self.operator diff --git a/crates/pep508-rs/src/marker.rs b/crates/pep508-rs/src/marker.rs index 2e73a34cc..25cf56db7 100644 --- a/crates/pep508-rs/src/marker.rs +++ b/crates/pep508-rs/src/marker.rs @@ -312,6 +312,12 @@ impl FromStr for StringVersion { } } +impl Display for StringVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.version.fmt(f) + } +} + #[cfg(feature = "serde")] impl Serialize for StringVersion { fn serialize(&self, serializer: S) -> Result diff --git a/crates/puffin-build-cli/.gitignore b/crates/puffin-build-cli/.gitignore new file mode 100644 index 000000000..73e1f5d21 --- /dev/null +++ b/crates/puffin-build-cli/.gitignore @@ -0,0 +1 @@ +sdist_building_test_data diff --git a/crates/puffin-build-cli/Cargo.toml b/crates/puffin-build-cli/Cargo.toml new file mode 100644 index 000000000..4ab525777 --- /dev/null +++ b/crates/puffin-build-cli/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "puffin-build-cli" +version = "0.0.1" +description = "Build wheels from source distributions" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +platform-host = { path = "../platform-host" } +platform-tags = { path = "../platform-tags" } +puffin-build = { path = "../puffin-build" } +puffin-client = { path = "../puffin-client" } +puffin-dispatch = { path = "../puffin-dispatch" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-package = { path = "../puffin-package" } + +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +colored = { workspace = true } +directories = { workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +itertools = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +which = { workspace = true } diff --git a/crates/puffin-build/src/main.rs b/crates/puffin-build-cli/src/main.rs similarity index 70% rename from crates/puffin-build/src/main.rs rename to crates/puffin-build-cli/src/main.rs index 248de6a98..53208a233 100644 --- a/crates/puffin-build/src/main.rs +++ b/crates/puffin-build-cli/src/main.rs @@ -1,21 +1,27 @@ #![allow(clippy::print_stdout, clippy::print_stderr)] -use anyhow::Context; +use std::env; +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Instant; + +use anyhow::{Context, Result}; use clap::Parser; use colored::Colorize; use directories::ProjectDirs; use fs_err as fs; -use puffin_build::{Error, SourceDistributionBuilder}; -use std::path::PathBuf; -use std::process::ExitCode; -use std::time::Instant; -use std::{env, io}; use tracing::debug; use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; +use platform_host::Platform; +use puffin_build::SourceDistributionBuilder; +use puffin_client::RegistryClientBuilder; +use puffin_dispatch::BuildDispatch; +use puffin_interpreter::PythonExecutable; + #[derive(Parser)] struct Args { /// Base python in a way that can be found with `which` @@ -28,7 +34,7 @@ struct Args { sdist: PathBuf, } -async fn run() -> anyhow::Result<()> { +async fn run() -> Result<()> { let args = Args::parse(); let wheel_dir = if let Some(wheel_dir) = args.wheels { fs::create_dir_all(&wheel_dir).context("Invalid wheel directory")?; @@ -40,20 +46,17 @@ async fn run() -> anyhow::Result<()> { let dirs = ProjectDirs::from("", "", "puffin"); let cache = dirs.as_ref().map(ProjectDirs::cache_dir); - // TODO: That's no way to deal with paths in PATH - let base_python = which::which(args.python.unwrap_or("python3".into())).map_err(|err| { - Error::IO(io::Error::new( - io::ErrorKind::NotFound, - format!("Can't find `python3` ({err})"), - )) - })?; - let interpreter_info = gourgeist::get_interpreter_info(&base_python)?; + let platform = Platform::current()?; + let python = PythonExecutable::from_env(platform, cache)?; + let interpreter_info = gourgeist::get_interpreter_info(python.executable())?; + + let build_dispatch = + BuildDispatch::new(RegistryClientBuilder::default().build(), python, cache); let builder = - SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info, cache) - .await?; + SourceDistributionBuilder::setup(&args.sdist, &interpreter_info, &build_dispatch).await?; let wheel = builder.build(&wheel_dir)?; - println!("Wheel built to {}", wheel.display()); + println!("Wheel built to {}", wheel_dir.join(wheel).display()); Ok(()) } diff --git a/crates/puffin-build-cli/test_sdist_building.sh b/crates/puffin-build-cli/test_sdist_building.sh new file mode 100644 index 000000000..c521fdaa1 --- /dev/null +++ b/crates/puffin-build-cli/test_sdist_building.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Simple source distribution building integration test using the tqdm (PEP 517) and geoextract (setup.py) sdists. + +set -e + +mkdir -p sdist_building_test_data/sdist +if [ ! -f sdist_building_test_data/sdist/tqdm-4.66.1.tar.gz ]; then + wget https://files.pythonhosted.org/packages/62/06/d5604a70d160f6a6ca5fd2ba25597c24abd5c5ca5f437263d177ac242308/tqdm-4.66.1.tar.gz -O sdist_building_test_data/sdist/tqdm-4.66.1.tar.gz +fi +if [ ! -f sdist_building_test_data/sdist/geoextract-0.3.1.tar.gz ]; then + wget https://files.pythonhosted.org/packages/c4/00/9d9826a6e1c9139cc7183647f47f6b7acb290fa4c572140aa84a12728e60/geoextract-0.3.1.tar.gz -O sdist_building_test_data/sdist/geoextract-0.3.1.tar.gz +fi +rm -rf sdist_building_test_data/wheels +RUST_LOG=puffin_build=debug cargo run -p puffin-build-cli --bin puffin-build-cli -- --wheels sdist_building_test_data/wheels sdist_building_test_data/sdist/tqdm-4.66.1.tar.gz +RUST_LOG=puffin_build=debug cargo run -p puffin-build-cli --bin puffin-build-cli -- --wheels sdist_building_test_data/wheels sdist_building_test_data/sdist/geoextract-0.3.1.tar.gz + +# Check that pip accepts the wheels. It would be better to do functional checks +virtualenv -p 3.8 -q --clear sdist_building_test_data/.venv +sdist_building_test_data/.venv/bin/pip install -q --no-deps sdist_building_test_data/wheels/geoextract-0.3.1-py3-none-any.whl +sdist_building_test_data/.venv/bin/pip install -q --no-deps sdist_building_test_data/wheels/tqdm-4.66.1-py3-none-any.whl diff --git a/crates/puffin-build/Cargo.toml b/crates/puffin-build/Cargo.toml index 2648d9651..f8ebae19d 100644 --- a/crates/puffin-build/Cargo.toml +++ b/crates/puffin-build/Cargo.toml @@ -15,21 +15,13 @@ gourgeist = { path = "../gourgeist" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } -puffin-client = { path = "../puffin-client" } -puffin-installer = { path = "../puffin-installer" } -puffin-interpreter = { path = "../puffin-interpreter" } -puffin-package = { path = "../puffin-package" } -puffin-resolver = { path = "../puffin-resolver" } -puffin-workspace = { path = "../puffin-workspace" } +puffin-traits = { path = "../puffin-traits" } anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } -colored = { workspace = true } -directories = { workspace = true } flate2 = { workspace = true } fs-err = { workspace = true } indoc = { workspace = true } -itertools = { workspace = true } pyproject-toml = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -39,6 +31,5 @@ thiserror = { workspace = true } tokio = { workspace = true } toml = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } which = { workspace = true} zip = { workspace = true } diff --git a/crates/puffin-build/src/lib.rs b/crates/puffin-build/src/lib.rs index c2d070b90..b21ccf033 100644 --- a/crates/puffin-build/src/lib.rs +++ b/crates/puffin-build/src/lib.rs @@ -2,34 +2,27 @@ //! //! -use anyhow::Context; -use flate2::read::GzDecoder; -use fs_err as fs; -use fs_err::{DirEntry, File}; -use gourgeist::{InterpreterInfo, Venv}; -use indoc::formatdoc; -use itertools::{Either, Itertools}; -use pep508_rs::Requirement; -use platform_host::Platform; -use platform_tags::Tags; -use puffin_client::RegistryClientBuilder; -use puffin_installer::{CachedDistribution, Downloader, LocalIndex, RemoteDistribution, Unzipper}; -use puffin_interpreter::PythonExecutable; -use puffin_package::package_name::PackageName; -use puffin_resolver::WheelFinder; -use pyproject_toml::PyProjectToml; use std::io; use std::io::BufRead; -use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::str::FromStr; + +use flate2::read::GzDecoder; +use fs_err as fs; +use fs_err::{DirEntry, File}; +use indoc::formatdoc; +use pyproject_toml::PyProjectToml; use tar::Archive; use tempfile::{tempdir, TempDir}; use thiserror::Error; use tracing::{debug, instrument}; use zip::ZipArchive; +use gourgeist::{InterpreterInfo, Venv}; +use pep508_rs::Requirement; +use puffin_traits::BuildContext; + #[derive(Error, Debug)] pub enum Error { #[error(transparent)] @@ -42,8 +35,8 @@ pub enum Error { InvalidSourceDistribution(String), #[error("Invalid pyproject.toml")] InvalidPyprojectToml(#[from] toml::de::Error), - #[error("Failed to install requirements")] - RequirementsInstall(#[source] anyhow::Error), + #[error("Failed to install requirements from {0}")] + RequirementsInstall(&'static str, #[source] anyhow::Error), #[error("Failed to create temporary virtual environment")] Gourgeist(#[from] gourgeist::Error), #[error("Failed to run {0}")] @@ -112,9 +105,8 @@ impl SourceDistributionBuilder { /// Extract the source distribution and create a venv with the required packages pub async fn setup( sdist: &Path, - base_python: &Path, interpreter_info: &InterpreterInfo, - cache: Option<&Path>, + build_context: &impl BuildContext, ) -> Result { let temp_dir = tempdir()?; @@ -148,10 +140,9 @@ impl SourceDistributionBuilder { create_pep517_build_environment( temp_dir.path(), &source_tree, - base_python, interpreter_info, pep517_backend, - cache, + build_context, ) .await? } else { @@ -163,19 +154,24 @@ impl SourceDistributionBuilder { } let venv = gourgeist::create_venv( temp_dir.path().join("venv"), - base_python, + build_context.python().executable(), interpreter_info, true, )?; - // TODO: Resolve those once globally and cache per puffin invocation + // TODO(konstin): Resolve those once globally and cache per puffin invocation let requirements = [ Requirement::from_str("wheel").unwrap(), Requirement::from_str("setuptools").unwrap(), Requirement::from_str("pip").unwrap(), ]; - resolve_and_install(venv.as_std_path(), &requirements, cache) + let resolved_requirements = build_context + .resolve(&requirements) .await - .map_err(Error::RequirementsInstall)?; + .map_err(|err| Error::RequirementsInstall("setup.py build", err))?; + build_context + .install(&resolved_requirements, &venv) + .await + .map_err(|err| Error::RequirementsInstall("setup.py build", err))?; venv }; @@ -209,8 +205,8 @@ impl SourceDistributionBuilder { r#"{} as backend import json - if get_requires_for_build_wheel := getattr(backend, "prepare_metadata_for_build_wheel", None): - print(get_requires_for_build_wheel("{}")) + if prepare_metadata_for_build_wheel := getattr(backend, "prepare_metadata_for_build_wheel", None): + print(prepare_metadata_for_build_wheel("{}")) else: print() "#, pep517_backend.backend_import(), escape_path_for_python(&metadata_directory) @@ -255,7 +251,7 @@ impl SourceDistributionBuilder { /// /// #[instrument(skip(self))] - pub fn build(&self, wheel_dir: &Path) -> Result { + pub fn build(&self, wheel_dir: &Path) -> Result { // The build scripts run with the extracted root as cwd, so they need the absolute path let wheel_dir = fs::canonicalize(wheel_dir)?; @@ -287,9 +283,9 @@ impl SourceDistributionBuilder { }; // TODO(konstin): Faster copy such as reflink? Or maybe don't really let the user pick the target dir let wheel = wheel_dir.join(dist_wheel.file_name()); - fs::copy(dist_wheel.path(), &wheel)?; + fs::copy(dist_wheel.path(), wheel)?; // TODO(konstin): Check wheel filename - Ok(wheel) + Ok(dist_wheel.file_name().to_string_lossy().to_string()) } } @@ -297,7 +293,7 @@ impl SourceDistributionBuilder { &self, wheel_dir: &Path, pep517_backend: &Pep517Backend, - ) -> Result { + ) -> Result { let metadata_directory = self .metadata_directory .as_deref() @@ -323,18 +319,17 @@ impl SourceDistributionBuilder { )); } let stdout = String::from_utf8_lossy(&output.stdout); - let wheel = stdout - .lines() - .last() - .map(|distribution_filename| wheel_dir.join(distribution_filename)); - let Some(wheel) = wheel.filter(|wheel| wheel.is_file()) else { + let distribution_filename = stdout.lines().last(); + let Some(distribution_filename) = + distribution_filename.filter(|wheel| wheel_dir.join(wheel).is_file()) + else { return Err(Error::from_command_output( "Build backend did not return the wheel filename through `build_wheel()`" .to_string(), &output, )); }; - Ok(wheel) + Ok(distribution_filename.to_string()) } } @@ -348,19 +343,24 @@ fn escape_path_for_python(path: &Path) -> String { async fn create_pep517_build_environment( root: &Path, source_tree: &Path, - base_python: &Path, data: &InterpreterInfo, pep517_backend: &Pep517Backend, - cache: Option<&Path>, + build_context: &impl BuildContext, ) -> Result { - let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, true)?; - resolve_and_install( - venv.deref().as_std_path(), - &pep517_backend.requirements, - cache, - ) - .await - .map_err(Error::RequirementsInstall)?; + let venv = gourgeist::create_venv( + root.join(".venv"), + build_context.python().executable(), + data, + true, + )?; + let resolved_requirements = build_context + .resolve(&pep517_backend.requirements) + .await + .map_err(|err| Error::RequirementsInstall("get_requires_for_build_wheel", err))?; + build_context + .install(&resolved_requirements, &venv) + .await + .map_err(|err| Error::RequirementsInstall("get_requires_for_build_wheel", err))?; debug!( "Calling `{}.get_requires_for_build_wheel()`", @@ -375,7 +375,7 @@ async fn create_pep517_build_environment( else: requires = [] print(json.dumps(requires)) - "#, pep517_backend.backend_import() + "#, pep517_backend.backend_import() }; let output = run_python_script(&venv.python_interpreter(), &script, source_tree)?; if !output.status.success() { @@ -420,63 +420,18 @@ async fn create_pep517_build_environment( .cloned() .chain(extra_requires) .collect(); - resolve_and_install(&*venv, &requirements, cache) + let resolved_requirements = build_context + .resolve(&requirements) .await - .map_err(Error::RequirementsInstall)?; + .map_err(|err| Error::RequirementsInstall("build-system.requires", err))?; + + build_context + .install(&resolved_requirements, &venv) + .await + .map_err(|err| Error::RequirementsInstall("build-system.requires", err))?; } Ok(venv) } - -#[instrument(skip_all)] -async fn resolve_and_install( - venv: impl AsRef, - requirements: &[Requirement], - cache: Option<&Path>, -) -> anyhow::Result<()> { - debug!("Installing {} build requirements", requirements.len()); - - let local_index = if let Some(cache) = cache { - LocalIndex::try_from_directory(cache)? - } else { - LocalIndex::default() - }; - let (cached, uncached): (Vec, Vec) = - requirements.iter().partition_map(|requirement| { - let package = PackageName::normalize(&requirement.name); - if let Some(distribution) = local_index - .get(&package) - .filter(|dist| requirement.is_satisfied_by(dist.version())) - { - Either::Left(distribution.clone()) - } else { - Either::Right(requirement.clone()) - } - }); - - let client = RegistryClientBuilder::default().cache(cache).build(); - - let platform = Platform::current()?; - let python = PythonExecutable::from_venv(platform, venv.as_ref(), cache)?; - let tags = Tags::from_env(python.platform(), python.simple_version())?; - let resolution = WheelFinder::new(&tags, &client).resolve(&uncached).await?; - let uncached = resolution - .into_files() - .map(RemoteDistribution::from_file) - .collect::>>()?; - let staging = tempdir()?; - let downloads = Downloader::new(&client, cache) - .download(&uncached, cache.unwrap_or(staging.path())) - .await?; - let unzips = Unzipper::default() - .download(downloads, cache.unwrap_or(staging.path())) - .await - .context("Failed to download and unpack wheels")?; - let wheels = unzips.into_iter().chain(cached).collect::>(); - puffin_installer::Installer::new(&python).install(&wheels)?; - - Ok(()) -} - /// Returns the directory with the `pyproject.toml`/`setup.py` #[instrument(skip_all, fields(path))] fn extract_archive(sdist: &Path, extracted: &PathBuf) -> Result { diff --git a/crates/puffin-build/test.sh b/crates/puffin-build/test.sh deleted file mode 100644 index aa96e0971..000000000 --- a/crates/puffin-build/test.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e - -mkdir -p downloads -if [ ! -f downloads/tqdm-4.66.1.tar.gz ]; then - wget https://files.pythonhosted.org/packages/62/06/d5604a70d160f6a6ca5fd2ba25597c24abd5c5ca5f437263d177ac242308/tqdm-4.66.1.tar.gz -O downloads/tqdm-4.66.1.tar.gz -fi -if [ ! -f downloads/geoextract-0.3.1.tar.gz ]; then - wget https://files.pythonhosted.org/packages/c4/00/9d9826a6e1c9139cc7183647f47f6b7acb290fa4c572140aa84a12728e60/geoextract-0.3.1.tar.gz -O downloads/geoextract-0.3.1.tar.gz -fi -rm -rf wheels -RUST_LOG=puffin_build=debug cargo run -p puffin-build --bin puffin-build -- --wheels wheels downloads/tqdm-4.66.1.tar.gz -RUST_LOG=puffin_build=debug cargo run -p puffin-build --bin puffin-build -- --wheels wheels downloads/geoextract-0.3.1.tar.gz - -# Check that pip accepts the wheels. It would be better to do functional checks -virtualenv -p 3.8 -q --clear wheels/.venv -wheels/.venv/bin/pip install -q --no-deps wheels/geoextract-0.3.1-py3-none-any.whl -wheels/.venv/bin/pip install -q --no-deps wheels/tqdm-4.66.1-py3-none-any.whl \ No newline at end of file diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml index 6a9609078..e10b25691 100644 --- a/crates/puffin-cli/Cargo.toml +++ b/crates/puffin-cli/Cargo.toml @@ -16,6 +16,7 @@ platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } pubgrub = { path = "../../vendor/pubgrub" } puffin-client = { path = "../puffin-client" } +puffin-dispatch = { path = "../puffin-dispatch" } puffin-installer = { path = "../puffin-installer" } puffin-interpreter = { path = "../puffin-interpreter" } puffin-package = { path = "../puffin-package" } diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index 73be1829a..44b74966f 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -7,15 +7,15 @@ use anyhow::Result; use colored::Colorize; use fs_err::File; use itertools::Itertools; -use pubgrub::report::Reporter; -use tracing::debug; - use pep508_rs::Requirement; use platform_host::Platform; use platform_tags::Tags; +use pubgrub::report::Reporter; use puffin_client::RegistryClientBuilder; +use puffin_dispatch::BuildDispatch; use puffin_interpreter::PythonExecutable; use puffin_resolver::ResolutionMode; +use tracing::debug; use crate::commands::{elapsed, ExitStatus}; use crate::index_urls::IndexUrls; @@ -51,14 +51,13 @@ pub(crate) async fn pip_compile( // Detect the current Python interpreter. let platform = Platform::current()?; let python = PythonExecutable::from_env(platform, cache)?; + debug!( - "Using Python interpreter: {}", + "Using Python {} at {}", + python.markers().python_version, python.executable().display() ); - // Determine the current environment markers. - let markers = python.markers(); - // Determine the compatible platform tags. let tags = Tags::from_env(python.platform(), python.simple_version())?; @@ -77,9 +76,22 @@ pub(crate) async fn pip_compile( builder.build() }; + let build_dispatch = BuildDispatch::new( + RegistryClientBuilder::default().build(), + python.clone(), + cache, + ); + // Resolve the dependencies. - let resolver = - puffin_resolver::Resolver::new(requirements, constraints, mode, markers, &tags, &client); + let resolver = puffin_resolver::Resolver::new( + requirements, + constraints, + mode, + python.markers(), + &tags, + &client, + &build_dispatch, + ); let resolution = match resolver.resolve().await { Err(puffin_resolver::ResolveError::PubGrub(pubgrub::error::PubGrubError::NoSolution( mut derivation_tree, diff --git a/crates/puffin-cli/src/commands/pip_sync.rs b/crates/puffin-cli/src/commands/pip_sync.rs index 4166609a8..b04cb82bf 100644 --- a/crates/puffin-cli/src/commands/pip_sync.rs +++ b/crates/puffin-cli/src/commands/pip_sync.rs @@ -3,20 +3,16 @@ use std::path::Path; use anyhow::{Context, Result}; use colored::Colorize; -use install_wheel_rs::linker::LinkMode; use itertools::Itertools; use tracing::debug; +use install_wheel_rs::linker::LinkMode; use pep508_rs::Requirement; use platform_host::Platform; use platform_tags::Tags; use puffin_client::RegistryClientBuilder; -use puffin_installer::{ - CachedDistribution, Distribution, InstalledDistribution, LocalIndex, RemoteDistribution, - SitePackages, -}; +use puffin_installer::{Distribution, PartitionedRequirements, RemoteDistribution}; use puffin_interpreter::PythonExecutable; -use puffin_package::package_name::PackageName; use crate::commands::reporters::{ DownloadReporter, InstallReporter, UnzipReporter, WheelFinderReporter, @@ -285,90 +281,6 @@ pub(crate) async fn sync_requirements( Ok(ExitStatus::Success) } -#[derive(Debug, Default)] -struct PartitionedRequirements { - /// The distributions that are not already installed in the current environment, but are - /// available in the local cache. - local: Vec, - - /// The distributions that are not already installed in the current environment, and are - /// not available in the local cache. - remote: Vec, - - /// The distributions that are already installed in the current environment, and are - /// _not_ necessary to satisfy the requirements. - extraneous: Vec, -} - -impl PartitionedRequirements { - /// Partition a set of requirements into those that should be linked from the cache, those that - /// need to be downloaded, and those that should be removed. - pub(crate) fn try_from_requirements( - requirements: &[Requirement], - cache: Option<&Path>, - python: &PythonExecutable, - ) -> Result { - // Index all the already-installed packages in site-packages. - let mut site_packages = SitePackages::try_from_executable(python)?; - - // Index all the already-downloaded wheels in the cache. - let local_index = if let Some(cache) = cache { - LocalIndex::try_from_directory(cache)? - } else { - LocalIndex::default() - }; - - let mut local = vec![]; - let mut remote = vec![]; - let mut extraneous = vec![]; - - for requirement in requirements { - let package = PackageName::normalize(&requirement.name); - - // Filter out already-installed packages. - if let Some(dist) = site_packages.remove(&package) { - if requirement.is_satisfied_by(dist.version()) { - debug!( - "Requirement already satisfied: {} ({})", - package, - dist.version() - ); - continue; - } - extraneous.push(dist); - } - - // Identify any locally-available distributions that satisfy the requirement. - if let Some(distribution) = local_index - .get(&package) - .filter(|dist| requirement.is_satisfied_by(dist.version())) - { - debug!( - "Requirement already cached: {} ({})", - distribution.name(), - distribution.version() - ); - local.push(distribution.clone()); - } else { - debug!("Identified uncached requirement: {}", requirement); - remote.push(requirement.clone()); - } - } - - // Remove any unnecessary packages. - for (package, dist_info) in site_packages { - debug!("Unnecessary package: {} ({})", package, dist_info.version()); - extraneous.push(dist_info); - } - - Ok(PartitionedRequirements { - local, - remote, - extraneous, - }) - } -} - #[derive(Debug)] enum ChangeEventKind { /// The package was added to the environment. diff --git a/crates/puffin-client/Cargo.toml b/crates/puffin-client/Cargo.toml index b7028cfaf..5cdc3e6e1 100644 --- a/crates/puffin-client/Cargo.toml +++ b/crates/puffin-client/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] puffin-package = { path = "../puffin-package" } +futures = { workspace = true } http-cache-reqwest = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } @@ -13,6 +14,6 @@ reqwest-retry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -url = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } -futures = { workspace = true } +url = { workspace = true } diff --git a/crates/puffin-dispatch/Cargo.toml b/crates/puffin-dispatch/Cargo.toml new file mode 100644 index 000000000..6b47fa29d --- /dev/null +++ b/crates/puffin-dispatch/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "puffin-dispatch" +version = "0.1.0" +description = "Avoid cyclic crate dependencies between resolver, installer and builder" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +platform-host = { path = "../platform-host" } +platform-tags = { path = "../platform-tags" } +puffin-build = { path = "../puffin-build" } +puffin-client = { path = "../puffin-client" } +puffin-installer = { path = "../puffin-installer" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-package = { path = "../puffin-package" } +puffin-resolver = { path = "../puffin-resolver" } +puffin-traits = { path = "../puffin-traits" } + +anyhow = { workspace = true } +itertools = { workspace = true } +tempfile = { workspace = true } +tracing = { workspace = true } diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs new file mode 100644 index 000000000..0a4400bf8 --- /dev/null +++ b/crates/puffin-dispatch/src/lib.rs @@ -0,0 +1,163 @@ +//! Avoid cyclic crate dependencies between [resolver][`puffin_resolver`], +//! [installer][`puffin_installer`] and [build][`puffin_build`] through [`BuildDispatch`] +//! implementing [`BuildContext`]. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use anyhow::Context; +use itertools::Itertools; +use tempfile::tempdir; + +use gourgeist::Venv; +use pep508_rs::Requirement; +use platform_tags::Tags; +use puffin_build::SourceDistributionBuilder; +use puffin_client::RegistryClient; +use puffin_installer::{ + uninstall, Downloader, Installer, PartitionedRequirements, RemoteDistribution, Unzipper, +}; +use puffin_interpreter::PythonExecutable; +use puffin_resolver::{ResolutionMode, Resolver, WheelFinder}; +use puffin_traits::BuildContext; +use tracing::debug; + +/// The main implementation of [`BuildContext`], used by the CLI, see [`BuildContext`] +/// documentation. +pub struct BuildDispatch { + client: RegistryClient, + python: PythonExecutable, + cache: Option, +} + +impl BuildDispatch { + pub fn new(client: RegistryClient, python: PythonExecutable, cache: Option) -> Self + where + T: Into, + { + Self { + client, + python, + cache: cache.map(Into::into), + } + } +} + +impl BuildContext for BuildDispatch { + fn cache(&self) -> Option<&Path> { + self.cache.as_deref() + } + + fn python(&self) -> &PythonExecutable { + &self.python + } + + fn resolve<'a>( + &'a self, + requirements: &'a [Requirement], + ) -> Pin>> + 'a>> { + Box::pin(async { + let tags = Tags::from_env(self.python.platform(), self.python.simple_version())?; + let resolver = Resolver::new( + requirements.to_vec(), + Vec::default(), + ResolutionMode::Highest, + self.python.markers(), + &tags, + &self.client, + self, + ); + let resolution_graph = resolver.resolve().await.context( + "No solution found when resolving build dependencies for source distribution build", + )?; + Ok(resolution_graph.requirements()) + }) + } + + fn install<'a>( + &'a self, + requirements: &'a [Requirement], + venv: &'a Venv, + ) -> Pin> + 'a>> { + Box::pin(async move { + debug!( + "Install in {} requirements {}", + venv.as_str(), + requirements.iter().map(ToString::to_string).join(", ") + ); + let python = self.python().with_venv(venv.as_std_path()); + + let PartitionedRequirements { + local, + remote, + extraneous, + } = PartitionedRequirements::try_from_requirements( + requirements, + self.cache(), + &python, + )?; + + if !extraneous.is_empty() { + debug!( + "Removing {:?}", + extraneous + .iter() + .map(puffin_installer::InstalledDistribution::id) + .join(", ") + ); + + for dist_info in extraneous { + uninstall(&dist_info).await?; + } + } + + debug!( + "Fetching {}", + remote.iter().map(ToString::to_string).join(", ") + ); + + let tags = Tags::from_env(python.platform(), python.simple_version())?; + let resolution = WheelFinder::new(&tags, &self.client) + .resolve(&remote) + .await?; + + let uncached = resolution + .into_files() + .map(RemoteDistribution::from_file) + .collect::>>()?; + let staging = tempdir()?; + let downloads = Downloader::new(&self.client, self.cache.as_deref()) + .download(&uncached, self.cache.as_deref().unwrap_or(staging.path())) + .await?; + let unzips = Unzipper::default() + .download(downloads, self.cache.as_deref().unwrap_or(staging.path())) + .await + .context("Failed to download and unpack wheels")?; + + debug!( + "Fetching {}", + unzips + .iter() + .chain(&local) + .map(puffin_installer::CachedDistribution::id) + .join(", ") + ); + let wheels = unzips.into_iter().chain(local).collect::>(); + Installer::new(&python).install(&wheels)?; + Ok(()) + }) + } + + fn build_source_distribution<'a>( + &'a self, + sdist: &'a Path, + wheel_dir: &'a Path, + ) -> Pin> + 'a>> { + Box::pin(async move { + let interpreter_info = gourgeist::get_interpreter_info(self.python.executable())?; + let builder = SourceDistributionBuilder::setup(sdist, &interpreter_info, self).await?; + Ok(builder.build(wheel_dir)?) + }) + } +} diff --git a/crates/puffin-installer/Cargo.toml b/crates/puffin-installer/Cargo.toml index 55243c5f5..9064dfe7e 100644 --- a/crates/puffin-installer/Cargo.toml +++ b/crates/puffin-installer/Cargo.toml @@ -12,6 +12,7 @@ license = { workspace = true } [dependencies] install-wheel-rs = { path = "../install-wheel-rs", default-features = false } pep440_rs = { path = "../pep440-rs" } +pep508_rs = { path = "../pep508-rs" } puffin-client = { path = "../puffin-client" } puffin-interpreter = { path = "../puffin-interpreter" } puffin-package = { path = "../puffin-package" } diff --git a/crates/puffin-installer/src/lib.rs b/crates/puffin-installer/src/lib.rs index 65396a2b3..1c81c799d 100644 --- a/crates/puffin-installer/src/lib.rs +++ b/crates/puffin-installer/src/lib.rs @@ -4,6 +4,7 @@ pub use distribution::{ pub use downloader::{Downloader, Reporter as DownloadReporter}; pub use installer::{Installer, Reporter as InstallReporter}; pub use local_index::LocalIndex; +pub use plan::PartitionedRequirements; pub use site_packages::SitePackages; pub use uninstall::uninstall; pub use unzipper::{Reporter as UnzipReporter, Unzipper}; @@ -13,6 +14,7 @@ mod distribution; mod downloader; mod installer; mod local_index; +mod plan; mod site_packages; mod uninstall; mod unzipper; diff --git a/crates/puffin-installer/src/plan.rs b/crates/puffin-installer/src/plan.rs new file mode 100644 index 000000000..2a022b94d --- /dev/null +++ b/crates/puffin-installer/src/plan.rs @@ -0,0 +1,94 @@ +use std::path::Path; + +use anyhow::Result; +use tracing::debug; + +use pep508_rs::Requirement; +use puffin_interpreter::PythonExecutable; +use puffin_package::package_name::PackageName; + +use crate::{CachedDistribution, InstalledDistribution, LocalIndex, SitePackages}; + +#[derive(Debug, Default)] +pub struct PartitionedRequirements { + /// The distributions that are not already installed in the current environment, but are + /// available in the local cache. + pub local: Vec, + + /// The distributions that are not already installed in the current environment, and are + /// not available in the local cache. + pub remote: Vec, + + /// The distributions that are already installed in the current environment, and are + /// _not_ necessary to satisfy the requirements. + pub extraneous: Vec, +} + +impl PartitionedRequirements { + /// Partition a set of requirements into those that should be linked from the cache, those that + /// need to be downloaded, and those that should be removed. + pub fn try_from_requirements( + requirements: &[Requirement], + cache: Option<&Path>, + python: &PythonExecutable, + ) -> Result { + // Index all the already-installed packages in site-packages. + let mut site_packages = SitePackages::try_from_executable(python)?; + + // Index all the already-downloaded wheels in the cache. + let local_index = if let Some(cache) = cache { + LocalIndex::try_from_directory(cache)? + } else { + LocalIndex::default() + }; + + let mut local = vec![]; + let mut remote = vec![]; + let mut extraneous = vec![]; + + for requirement in requirements { + let package = PackageName::normalize(&requirement.name); + + // Filter out already-installed packages. + if let Some(dist) = site_packages.remove(&package) { + if requirement.is_satisfied_by(dist.version()) { + debug!( + "Requirement already satisfied: {} ({})", + package, + dist.version() + ); + continue; + } + extraneous.push(dist); + } + + // Identify any locally-available distributions that satisfy the requirement. + if let Some(distribution) = local_index + .get(&package) + .filter(|dist| requirement.is_satisfied_by(dist.version())) + { + debug!( + "Requirement already cached: {} ({})", + distribution.name(), + distribution.version() + ); + local.push(distribution.clone()); + } else { + debug!("Identified uncached requirement: {}", requirement); + remote.push(requirement.clone()); + } + } + + // Remove any unnecessary packages. + for (package, dist_info) in site_packages { + debug!("Unnecessary package: {} ({})", package, dist_info.version()); + extraneous.push(dist_info); + } + + Ok(PartitionedRequirements { + local, + remote, + extraneous, + }) + } +} diff --git a/crates/puffin-installer/src/unzipper.rs b/crates/puffin-installer/src/unzipper.rs index d44e12f0f..746ed575b 100644 --- a/crates/puffin-installer/src/unzipper.rs +++ b/crates/puffin-installer/src/unzipper.rs @@ -55,11 +55,19 @@ impl Unzipper { .await??; // Write the unzipped wheel to the target directory. - fs_err::tokio::rename( + let result = fs_err::tokio::rename( staging.path().join(remote.id()), wheel_cache.entry(&remote.id()), ) - .await?; + .await; + + if let Err(err) = result { + // If the renaming failed because another instance was faster, that's fine + // (`DirectoryNotEmpty` is not stable so we can't match on it) + if !wheel_cache.entry(&remote.id()).is_dir() { + return Err(err.into()); + } + } wheels.push(CachedDistribution::new( remote.name().clone(), diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index 279f45e96..c9708a971 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -13,7 +13,6 @@ license = { workspace = true } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } -puffin-package = { path = "../puffin-package" } anyhow = { workspace = true } cacache = { workspace = true } diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index 19fa890be..e26b23062 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -13,7 +13,7 @@ mod python_platform; mod virtual_env; /// A Python executable and its associated platform markers. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PythonExecutable { platform: PythonPlatform, venv: PathBuf, @@ -50,6 +50,18 @@ impl PythonExecutable { }) } + /// Create a [`PythonExecutable`] for a venv with a known base [`PythonExecutable`]. + #[must_use] + pub fn with_venv(&self, venv: &Path) -> Self { + let executable = self.platform.venv_python(venv); + + Self { + venv: venv.to_path_buf(), + executable, + ..self.clone() + } + } + /// Returns the path to the Python virtual environment. pub fn platform(&self) -> &Platform { &self.platform diff --git a/crates/puffin-package/Cargo.toml b/crates/puffin-package/Cargo.toml index 4b796bb59..a2c7db8e3 100644 --- a/crates/puffin-package/Cargo.toml +++ b/crates/puffin-package/Cargo.toml @@ -7,7 +7,6 @@ edition = "2021" pep440_rs = { path = "../pep440-rs", features = ["serde"] } pep508_rs = { path = "../pep508-rs", features = ["serde"] } -anyhow = { workspace = true } fs-err = { workspace = true } mailparse = { workspace = true } memchr = { workspace = true } @@ -20,6 +19,7 @@ tracing.workspace = true unscanny = { workspace = true } [dev-dependencies] +anyhow = { version = "1.0.75" } indoc = { version = "2.0.4" } insta = { version = "1.33.0" } serde_json = { version = "1.0.107" } diff --git a/crates/puffin-resolver/Cargo.toml b/crates/puffin-resolver/Cargo.toml index c9bbc936d..5f11c37c7 100644 --- a/crates/puffin-resolver/Cargo.toml +++ b/crates/puffin-resolver/Cargo.toml @@ -10,6 +10,8 @@ authors = { workspace = true } license = { workspace = true } [dependencies] +gourgeist = { path = "../gourgeist" } +install-wheel-rs = { path = "../install-wheel-rs" } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } @@ -17,23 +19,32 @@ platform-tags = { path = "../platform-tags" } pubgrub = { path = "../../vendor/pubgrub" } puffin-client = { path = "../puffin-client" } puffin-package = { path = "../puffin-package" } +puffin-traits = { path = "../puffin-traits" } distribution-filename = { path = "../distribution-filename" } anyhow = { workspace = true } bitflags = { workspace = true } clap = { workspace = true, features = ["derive"], optional = true } colored = { workspace = true } +fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } fxhash = { workspace = true } itertools = { workspace = true } once_cell = { workspace = true } petgraph = { workspace = true } +tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } +url = { workspace = true } waitmap = { workspace = true } +which = { workspace = true } +zip = { workspace = true } [dev-dependencies] +puffin-interpreter = { path = "../puffin-interpreter" } + once_cell = { version = "1.18.0" } insta = { version = "1.34.0" } diff --git a/crates/puffin-resolver/src/error.rs b/crates/puffin-resolver/src/error.rs index 24572d6ac..2b1ee5d97 100644 --- a/crates/puffin-resolver/src/error.rs +++ b/crates/puffin-resolver/src/error.rs @@ -25,6 +25,14 @@ pub enum ResolveError { #[error(transparent)] PubGrub(#[from] pubgrub::error::PubGrubError>), + + #[error("Failed to build source distribution {filename}")] + SourceDistribution { + filename: String, + // TODO(konstin): Gives this a proper error type + #[source] + err: anyhow::Error, + }, } impl From> for ResolveError { diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index df1c19a46..54ff22c84 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -2,6 +2,7 @@ pub use error::ResolveError; pub use mode::ResolutionMode; pub use resolution::PinnedPackage; pub use resolver::Resolver; +pub use source_distribution::BuiltSourceDistributionCache; pub use wheel_finder::{Reporter, WheelFinder}; mod error; @@ -9,4 +10,5 @@ mod mode; mod pubgrub; mod resolution; mod resolver; +mod source_distribution; mod wheel_finder; diff --git a/crates/puffin-resolver/src/resolution.rs b/crates/puffin-resolver/src/resolution.rs index 1fb44745a..dfd94d8f0 100644 --- a/crates/puffin-resolver/src/resolution.rs +++ b/crates/puffin-resolver/src/resolution.rs @@ -1,13 +1,14 @@ -use std::hash::BuildHasherDefault; - use colored::Colorize; use fxhash::FxHashMap; use petgraph::visit::EdgeRef; +use std::hash::BuildHasherDefault; + use pubgrub::range::Range; use pubgrub::solver::{Kind, State}; use pubgrub::type_aliases::SelectedDependencies; -use pep440_rs::Version; +use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; +use pep508_rs::{Requirement, VersionOrUrl}; use puffin_client::File; use puffin_package::package_name::PackageName; @@ -148,6 +149,27 @@ impl Graph { pub fn is_empty(&self) -> bool { self.0.node_count() == 0 } + + pub fn requirements(&self) -> Vec { + // Collect and sort all packages. + let mut nodes = self + .0 + .node_indices() + .map(|node| (node, &self.0[node])) + .collect::>(); + nodes.sort_unstable_by_key(|(_, package)| package.name()); + self.0 + .node_indices() + .map(|node| Requirement { + name: self.0[node].name.to_string(), + extras: None, + version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from( + VersionSpecifier::equals_version(self.0[node].version.clone()), + ))), + marker: None, + }) + .collect() + } } /// Write the graph in the `{name}=={version}` format of requirements.txt that pip uses. diff --git a/crates/puffin-resolver/src/resolver.rs b/crates/puffin-resolver/src/resolver.rs index 1b1c5291f..bab985404 100644 --- a/crates/puffin-resolver/src/resolver.rs +++ b/crates/puffin-resolver/src/resolver.rs @@ -2,12 +2,13 @@ use std::borrow::Borrow; use std::collections::hash_map::Entry; +use std::future::Future; +use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; use anyhow::Result; use futures::channel::mpsc::UnboundedReceiver; -use futures::future::Either; use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt}; use fxhash::{FxHashMap, FxHashSet}; use pubgrub::error::PubGrubError; @@ -18,13 +19,14 @@ use tokio::select; use tracing::{debug, trace}; use waitmap::WaitMap; -use distribution_filename::WheelFilename; +use distribution_filename::{SourceDistributionFilename, WheelFilename}; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; use puffin_client::{File, RegistryClient, SimpleJson}; use puffin_package::dist_info_name::DistInfoName; use puffin_package::metadata::Metadata21; use puffin_package::package_name::PackageName; +use puffin_traits::BuildContext; use crate::error::ResolveError; use crate::mode::{CandidateSelector, ResolutionMode}; @@ -32,8 +34,10 @@ use crate::pubgrub::package::PubGrubPackage; use crate::pubgrub::version::{PubGrubVersion, MIN_VERSION}; use crate::pubgrub::{iter_requirements, version_range}; use crate::resolution::Graph; +use crate::source_distribution::{download_and_build_sdist, read_dist_info}; +use crate::BuiltSourceDistributionCache; -pub struct Resolver<'a> { +pub struct Resolver<'a, Ctx: BuildContext> { requirements: Vec, constraints: Vec, markers: &'a MarkerEnvironment, @@ -41,9 +45,10 @@ pub struct Resolver<'a> { client: &'a RegistryClient, selector: CandidateSelector, cache: Arc, + build_context: &'a Ctx, } -impl<'a> Resolver<'a> { +impl<'a, Ctx: BuildContext> Resolver<'a, Ctx> { /// Initialize a new resolver. pub fn new( requirements: Vec, @@ -52,6 +57,7 @@ impl<'a> Resolver<'a> { markers: &'a MarkerEnvironment, tags: &'a Tags, client: &'a RegistryClient, + build_context: &'a Ctx, ) -> Self { Self { selector: CandidateSelector::from_mode(mode, &requirements), @@ -61,6 +67,7 @@ impl<'a> Resolver<'a> { markers, tags, client, + build_context, } } @@ -266,27 +273,43 @@ impl<'a> Resolver<'a> { }; let simple_json = entry.value(); - // Select the latest compatible version. - let Some(file) = self + // Try to find a wheel. If there isn't any, to a find a source distribution. If there + // isn't any either, short circuit and fail the resolution. + let Some((file, request)) = self .selector .iter_candidates(package_name, &simple_json.files) - .find(|file| { - let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else { - return false; - }; - - if !name.is_compatible(self.tags) { - return false; + .find_map(|file| { + let wheel_filename = WheelFilename::from_str(file.filename.as_str()).ok()?; + if !wheel_filename.is_compatible(self.tags) { + return None; } - if !range + if range .borrow() - .contains(&PubGrubVersion::from(name.version.clone())) + .contains(&PubGrubVersion::from(wheel_filename.version.clone())) { - return false; - }; + Some((file, Request::Wheel(file.clone()))) + } else { + None + } + }) + .or_else(|| { + self.selector + .iter_candidates(package_name, &simple_json.files) + .find_map(|file| { + let sdist_filename = + SourceDistributionFilename::parse(&file.filename, package_name) + .ok()?; - true + if range + .borrow() + .contains(&PubGrubVersion::from(sdist_filename.version.clone())) + { + Some((file, Request::Sdist((file.clone(), sdist_filename)))) + } else { + None + } + }) }) else { // Short circuit: we couldn't find _any_ compatible versions for a package. @@ -296,7 +319,7 @@ impl<'a> Resolver<'a> { // Emit a request to fetch the metadata for this version. if in_flight.insert(file.hashes.sha256.clone()) { - request_sink.unbounded_send(Request::Version(file.clone()))?; + request_sink.unbounded_send(request)?; } selection = index; @@ -321,7 +344,7 @@ impl<'a> Resolver<'a> { ); // Find a compatible version. - let Some(wheel) = self + let mut wheel = self .selector .iter_candidates(package_name, &simple_json.files) .find_map(|file| { @@ -345,29 +368,67 @@ impl<'a> Resolver<'a> { name: package_name.clone(), version: name.version.clone(), }) - }) - else { - // Short circuit: we couldn't find _any_ compatible versions for a package. - return Ok((package, None)); - }; + }); - debug!( - "Selecting: {}=={} ({})", - wheel.name, wheel.version, wheel.file.filename - ); + if wheel.is_none() { + if let Some((sdist_file, parsed_filename)) = + self.selector + .iter_candidates(package_name, &simple_json.files) + .filter_map(|file| { + let Ok(parsed_filename) = + SourceDistributionFilename::parse(&file.filename, package_name) + else { + return None; + }; - // We want to return a package pinned to a specific version; but we _also_ want to - // store the exact file that we selected to satisfy that version. - pins.entry(wheel.name) - .or_default() - .insert(wheel.version.clone(), wheel.file.clone()); + if !range.borrow().contains(&PubGrubVersion::from( + parsed_filename.version.clone(), + )) { + return None; + }; - // Emit a request to fetch the metadata for this version. - if in_flight.insert(wheel.file.hashes.sha256.clone()) { - request_sink.unbounded_send(Request::Version(wheel.file.clone()))?; + Some((file, parsed_filename)) + }) + .max_by(|left, right| left.1.version.cmp(&right.1.version)) + { + // Emit a request to fetch the metadata for this version. + if in_flight.insert(sdist_file.hashes.sha256.clone()) { + request_sink.unbounded_send(Request::Sdist(( + sdist_file.clone(), + parsed_filename.clone(), + )))?; + } + // TODO(konstin): That's not a wheel + wheel = Some(Wheel { + file: sdist_file.clone(), + name: package_name.clone(), + version: parsed_filename.version.clone(), + }); + } } - Ok((package, Some(PubGrubVersion::from(wheel.version)))) + if let Some(wheel) = wheel { + debug!( + "Selecting: {}=={} ({})", + wheel.name, wheel.version, wheel.file.filename + ); + + // We want to return a package pinned to a specific version; but we _also_ want to + // store the exact file that we selected to satisfy that version. + pins.entry(wheel.name) + .or_default() + .insert(wheel.version.clone(), wheel.file.clone()); + + // Emit a request to fetch the metadata for this version. + if in_flight.insert(wheel.file.hashes.sha256.clone()) { + request_sink.unbounded_send(Request::Wheel(wheel.file.clone()))?; + } + + Ok((package, Some(PubGrubVersion::from(wheel.version)))) + } else { + // Short circuit: we couldn't find _any_ compatible versions for a package. + Ok((package, None)) + } } }; } @@ -485,20 +546,7 @@ impl<'a> Resolver<'a> { /// Fetch the metadata for a stream of packages and versions. async fn fetch(&self, request_stream: UnboundedReceiver) -> Result<(), ResolveError> { let mut response_stream = request_stream - .map({ - |request: Request| match request { - Request::Package(package_name) => Either::Left( - self.client - .simple(package_name.clone()) - .map_ok(move |metadata| Response::Package(package_name, metadata)), - ), - Request::Version(file) => Either::Right( - self.client - .file(file.clone()) - .map_ok(move |metadata| Response::Version(file, metadata)), - ), - } - }) + .map(|request| self.process_request(request)) .buffer_unordered(32) .ready_chunks(32); @@ -509,18 +557,61 @@ impl<'a> Resolver<'a> { trace!("Received package metadata for {}", package_name); self.cache.packages.insert(package_name.clone(), metadata); } - Response::Version(file, metadata) => { + Response::Wheel(file, metadata) => { trace!("Received file metadata for {}", file.filename); self.cache .versions .insert(file.hashes.sha256.clone(), metadata); } + Response::Sdist(file, metadata) => { + trace!("Received sdist build metadata for {}", file.filename); + self.cache + .versions + .insert(file.hashes.sha256.clone(), metadata); + } } } } Ok::<(), ResolveError>(()) } + + fn process_request( + &'a self, + request: Request, + ) -> Pin> + 'a>> { + match request { + Request::Package(package_name) => Box::pin( + self.client + .simple(package_name.clone()) + .map_ok(move |metadata| Response::Package(package_name, metadata)) + .map_err(ResolveError::Client), + ), + Request::Wheel(file) => Box::pin( + self.client + .file(file.clone()) + .map_ok(move |metadata| Response::Wheel(file, metadata)) + .map_err(ResolveError::Client), + ), + Request::Sdist((file, filename)) => Box::pin(async move { + let cached_wheel = self.build_context.cache().and_then(|cache| { + BuiltSourceDistributionCache::new(cache).find_wheel(&filename, self.tags) + }); + let metadata21 = if let Some(cached_wheel) = cached_wheel { + read_dist_info(cached_wheel).await + } else { + download_and_build_sdist(&file, self.client, self.build_context, &filename) + .await + } + .map_err(|err| ResolveError::SourceDistribution { + filename: file.filename.clone(), + err, + })?; + + Ok(Response::Sdist(file, metadata21)) + }), + } + } } #[derive(Debug, Clone)] @@ -533,12 +624,15 @@ struct Wheel { version: pep440_rs::Version, } +/// Fetch the metadata for an item #[derive(Debug)] enum Request { /// A request to fetch the metadata for a package. Package(PackageName), + /// A request to fetch and build the source distribution for a specific package version + Sdist((File, SourceDistributionFilename)), /// A request to fetch the metadata for a specific version of a package. - Version(File), + Wheel(File), } #[derive(Debug)] @@ -546,7 +640,9 @@ enum Response { /// The returned metadata for a package. Package(PackageName, SimpleJson), /// The returned metadata for a specific version of a package. - Version(File, Metadata21), + Wheel(File, Metadata21), + /// The returned metadata for an sdist build. + Sdist(File, Metadata21), } struct SolverCache { diff --git a/crates/puffin-resolver/src/source_distribution.rs b/crates/puffin-resolver/src/source_distribution.rs new file mode 100644 index 000000000..69d37e246 --- /dev/null +++ b/crates/puffin-resolver/src/source_distribution.rs @@ -0,0 +1,116 @@ +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use anyhow::Result; +use fs_err::tokio as fs; +use tempfile::tempdir; +use tokio::task::spawn_blocking; +use tokio_util::compat::FuturesAsyncReadCompatExt; +use tracing::debug; +use url::Url; +use zip::ZipArchive; + +use distribution_filename::{SourceDistributionFilename, WheelFilename}; +use pep440_rs::Version; +use platform_tags::Tags; +use puffin_client::{File, RegistryClient}; +use puffin_package::metadata::Metadata21; +use puffin_package::package_name::PackageName; +use puffin_traits::BuildContext; + +const BUILT_WHEELS_CACHE: &str = "built-wheels-v0"; + +/// TODO(konstin): Find a better home for me? +/// +/// Stores wheels built from source distributions. We need to keep those separate from the regular +/// wheel cache since a wheel with the same name may be uploaded after we made our build and in that +/// case the hashes would clash. +pub struct BuiltSourceDistributionCache(PathBuf); + +impl BuiltSourceDistributionCache { + pub fn new(path: impl AsRef) -> Self { + Self(path.as_ref().join(BUILT_WHEELS_CACHE)) + } + + pub fn version(&self, name: &PackageName, version: &Version) -> PathBuf { + self.0.join(name.to_string()).join(version.to_string()) + } + + /// Search for a wheel matching the tags that was built from the given source distribution. + pub fn find_wheel( + &self, + filename: &SourceDistributionFilename, + tags: &Tags, + ) -> Option { + let Ok(read_dir) = fs_err::read_dir(self.version(&filename.name, &filename.version)) else { + return None; + }; + + for entry in read_dir { + let Ok(entry) = entry else { continue }; + let Ok(wheel) = WheelFilename::from_str(entry.file_name().to_string_lossy().as_ref()) + else { + continue; + }; + + if wheel.is_compatible(tags) { + return Some(entry.path().clone()); + } + } + None + } +} + +pub(crate) async fn download_and_build_sdist( + file: &File, + client: &RegistryClient, + build_context: &impl BuildContext, + sdist_filename: &SourceDistributionFilename, +) -> Result { + debug!("Building {}", &file.filename); + let url = Url::parse(&file.url)?; + let reader = client.stream_external(&url).await?; + let mut reader = tokio::io::BufReader::new(reader.compat()); + let temp_dir = tempdir()?; + + let sdist_dir = temp_dir.path().join("sdist"); + tokio::fs::create_dir(&sdist_dir).await?; + let sdist_file = sdist_dir.join(&file.filename); + let mut writer = tokio::fs::File::create(&sdist_file).await?; + tokio::io::copy(&mut reader, &mut writer).await?; + + let wheel_dir = if let Some(cache) = &build_context.cache() { + BuiltSourceDistributionCache::new(cache) + .version(&sdist_filename.name, &sdist_filename.version) + } else { + temp_dir.path().join("wheels") + }; + + fs::create_dir_all(&wheel_dir).await?; + + let disk_filename = build_context + .build_source_distribution(&sdist_file, &wheel_dir) + .await?; + + let metadata21 = read_dist_info(wheel_dir.join(disk_filename)).await?; + + debug!("Finished Building {}", &file.filename); + Ok(metadata21) +} + +pub(crate) async fn read_dist_info(wheel: PathBuf) -> Result { + let dist_info = spawn_blocking(move || -> Result { + let mut archive = ZipArchive::new(std::fs::File::open(&wheel)?)?; + let dist_info_prefix = install_wheel_rs::find_dist_info( + &WheelFilename::from_str(wheel.file_name().unwrap().to_string_lossy().as_ref())?, + &mut archive, + )?; + let dist_info = std::io::read_to_string( + archive.by_name(&format!("{dist_info_prefix}.dist-info/METADATA"))?, + )?; + Ok(dist_info) + }) + .await + .unwrap()?; + Ok(Metadata21::parse(dist_info.as_bytes())?) +} diff --git a/crates/puffin-resolver/tests/resolver.rs b/crates/puffin-resolver/tests/resolver.rs index 50878d868..a5aea98df 100644 --- a/crates/puffin-resolver/tests/resolver.rs +++ b/crates/puffin-resolver/tests/resolver.rs @@ -3,16 +3,57 @@ //! Integration tests for the resolver. These tests rely on a live network connection, and hit //! `PyPI` directly. +use std::future::Future; +use std::path::Path; +use std::pin::Pin; use std::str::FromStr; use anyhow::Result; +use gourgeist::Venv; use once_cell::sync::Lazy; use pep508_rs::{MarkerEnvironment, Requirement, StringVersion}; use platform_host::{Arch, Os, Platform}; use platform_tags::Tags; use puffin_client::RegistryClientBuilder; +use puffin_interpreter::PythonExecutable; use puffin_resolver::{ResolutionMode, Resolver}; +use puffin_traits::BuildContext; + +struct DummyContext; + +impl BuildContext for DummyContext { + fn cache(&self) -> Option<&Path> { + panic!("The test should not need to build source distributions") + } + + fn python(&self) -> &PythonExecutable { + panic!("The test should not need to build source distributions") + } + + fn resolve<'a>( + &'a self, + _requirements: &'a [Requirement], + ) -> Pin>> + 'a>> { + panic!("The test should not need to build source distributions") + } + + fn install<'a>( + &'a self, + _requirements: &'a [Requirement], + _venv: &'a Venv, + ) -> Pin> + 'a>> { + panic!("The test should not need to build source distributions") + } + + fn build_source_distribution<'a>( + &'a self, + _sdist: &'a Path, + _wheel_dir: &'a Path, + ) -> Pin> + 'a>> { + panic!("The test should not need to build source distributions") + } +} #[tokio::test] async fn pylint() -> Result<()> { @@ -29,6 +70,7 @@ async fn pylint() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -52,6 +94,7 @@ async fn black() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -75,6 +118,7 @@ async fn black_colorama() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -98,6 +142,7 @@ async fn black_python_310() -> Result<()> { &MARKERS_310, &TAGS_310, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -123,6 +168,7 @@ async fn black_mypy_extensions() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -148,6 +194,7 @@ async fn black_mypy_extensions_extra() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -173,6 +220,7 @@ async fn black_flake8() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -196,6 +244,7 @@ async fn black_lowest() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; @@ -219,6 +268,7 @@ async fn black_lowest_direct() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + &DummyContext, ); let resolution = resolver.resolve().await?; diff --git a/crates/puffin-traits/Cargo.toml b/crates/puffin-traits/Cargo.toml new file mode 100644 index 000000000..fd5331fd2 --- /dev/null +++ b/crates/puffin-traits/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "puffin-traits" +version = "0.1.0" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +puffin-interpreter = { path = "../puffin-interpreter" } + +anyhow = { workspace = true } diff --git a/crates/puffin-traits/src/lib.rs b/crates/puffin-traits/src/lib.rs new file mode 100644 index 000000000..898a1e8ce --- /dev/null +++ b/crates/puffin-traits/src/lib.rs @@ -0,0 +1,78 @@ +//! Avoid cyclic crate dependencies between resolver, installer and builder. + +use std::future::Future; +use std::path::Path; +use std::pin::Pin; + +use gourgeist::Venv; +use pep508_rs::Requirement; +use puffin_interpreter::PythonExecutable; + +/// Avoid cyclic crate dependencies between resolver, installer and builder. +/// +/// To resolve the dependencies of a packages, we may need to build one or more source +/// distributions. To building a source distribution, we need to create a virtual environment from +/// the same base python as we use for the root resolution, resolve the build requirements +/// (potentially which nested source distributions, recursing a level deeper), installing +/// them and then build. The installer, the resolver and the source distribution builder are each in +/// their own crate. To avoid circular crate dependencies, this type dispatches between the three +/// crates with its three main methods ([`BuildContext::resolve`], [`BuildContext::install`] and +/// [`BuildContext::build_source_distribution`]). +/// +/// The overall main crate structure looks like this: +/// +/// ```text +/// ┌────────────────┐ +/// │puffin-cli │ +/// └───────▲────────┘ +/// │ +/// │ +/// ┌───────┴────────┐ +/// ┌─────────►│puffin-dispatch │◄─────────┐ +/// │ └───────▲────────┘ │ +/// │ │ │ +/// │ │ │ +/// ┌───────┴────────┐ ┌───────┴────────┐ ┌────────┴───────┐ +/// │puffin-resolver │ │puffin-installer│ │puffin-build │ +/// └───────▲────────┘ └───────▲────────┘ └────────▲───────┘ +/// │ │ │ +/// └─────────────┐ │ ┌──────────────┘ +/// ┌──┴────┴────┴───┐ +/// │puffin-traits │ +/// └────────────────┘ +/// ``` +/// +/// Put in a different way, this trait allows `puffin-resolver` to depend on `puffin-build` and +/// `puffin-build` to depend on `puffin-resolver` which having actual crate dependencies between +/// them. + +// TODO(konstin): Proper error types +pub trait BuildContext { + // TODO(konstin): Add a cache abstraction + fn cache(&self) -> Option<&Path>; + + /// All (potentially nested) source distribution builds use the same base python and can reuse + /// it's metadata (e.g. wheel compatibility tags). + fn python(&self) -> &PythonExecutable; + + /// Resolve the given requirements into a ready-to-install set of package versions. + fn resolve<'a>( + &'a self, + requirements: &'a [Requirement], + ) -> Pin>> + 'a>>; + /// Install the given set of package versions into the virtual environment. The environment must + /// use the same base python as [`Self::python`] + fn install<'a>( + &'a self, + requirements: &'a [Requirement], + venv: &'a Venv, + ) -> Pin> + 'a>>; + /// Build a source distribution into a wheel from an archive. + /// + /// Returns the filename of the built wheel inside the given `wheel_dir`. + fn build_source_distribution<'a>( + &'a self, + sdist: &'a Path, + wheel_dir: &'a Path, + ) -> Pin> + 'a>>; +}