diff --git a/crates/distribution-types/src/editable.rs b/crates/distribution-types/src/editable.rs index f4d6c32df..ade8b283b 100644 --- a/crates/distribution-types/src/editable.rs +++ b/crates/distribution-types/src/editable.rs @@ -8,7 +8,7 @@ use requirements_txt::EditableRequirement; #[derive(Debug, Clone)] pub struct LocalEditable { pub requirement: EditableRequirement, - /// Either the path to the editable or its checkout + /// Either the path to the editable or its checkout. pub path: PathBuf, } diff --git a/crates/distribution-types/src/lib.rs b/crates/distribution-types/src/lib.rs index 2c0168cda..76104faa8 100644 --- a/crates/distribution-types/src/lib.rs +++ b/crates/distribution-types/src/lib.rs @@ -47,6 +47,7 @@ use pep440_rs::Version; use pep508_rs::VerbatimUrl; use puffin_normalize::PackageName; use pypi_types::{File, IndexUrl}; +use requirements_txt::EditableRequirement; pub use crate::any::*; pub use crate::cached::*; @@ -243,6 +244,28 @@ impl Dist { } } + /// Create a [`Dist`] for a local editable distribution. + pub fn from_editable(name: PackageName, editable: LocalEditable) -> Result { + match editable.requirement { + EditableRequirement::Path { url, path } => { + Ok(Self::Source(SourceDist::Path(PathSourceDist { + name, + url, + path, + editable: true, + }))) + } + EditableRequirement::Url(url) => Ok(Self::Source(SourceDist::Path(PathSourceDist { + name, + path: url + .to_file_path() + .map_err(|()| Error::UrlFilename(url.to_url()))?, + url, + editable: true, + }))), + } + } + /// Returns the [`File`] instance, if this dist is from a registry with simple json api support pub fn file(&self) -> Option<&File> { match self { diff --git a/crates/distribution-types/src/resolution.rs b/crates/distribution-types/src/resolution.rs index 47ae4480e..65e69f5df 100644 --- a/crates/distribution-types/src/resolution.rs +++ b/crates/distribution-types/src/resolution.rs @@ -2,8 +2,9 @@ use rustc_hash::FxHashMap; use pep508_rs::Requirement; use puffin_normalize::PackageName; +use requirements_txt::EditableRequirement; -use crate::{BuiltDist, Dist, SourceDist}; +use crate::{BuiltDist, Dist, PathSourceDist, SourceDist}; /// A set of packages pinned at specific versions. #[derive(Debug, Default, Clone)] @@ -45,17 +46,45 @@ impl Resolution { self.0.is_empty() } - /// Return the set of [`Requirement`]s that this resolution represents. + /// Return the set of [`Requirement`]s that this resolution represents, exclusive of any + /// editable requirements. pub fn requirements(&self) -> Vec { let mut requirements = self .0 .values() - .cloned() - .map(Requirement::from) + .filter_map(|dist| match dist { + Dist::Source(SourceDist::Path(PathSourceDist { editable: true, .. })) => None, + dist => Some(Requirement::from(dist.clone())), + }) .collect::>(); requirements.sort_unstable_by(|a, b| a.name.cmp(&b.name)); requirements } + + /// Return the set of [`EditableRequirement`]s that this resolution represents. + pub fn editable_requirements(&self) -> Vec { + let mut requirements = self + .0 + .values() + .filter_map(|dist| { + let Dist::Source(SourceDist::Path(PathSourceDist { + url, + path, + editable: true, + .. + })) = dist + else { + return None; + }; + Some(EditableRequirement::Path { + path: path.clone(), + url: url.clone(), + }) + }) + .collect::>(); + requirements.sort_unstable_by(|a, b| a.url().cmp(b.url())); + requirements + } } impl From for Requirement { diff --git a/crates/puffin-cli/src/commands/pip_install.rs b/crates/puffin-cli/src/commands/pip_install.rs index be8dab558..84849b684 100644 --- a/crates/puffin-cli/src/commands/pip_install.rs +++ b/crates/puffin-cli/src/commands/pip_install.rs @@ -1,21 +1,25 @@ use std::fmt::Write; +use std::path::Path; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, Utc}; use colored::Colorize; use fs_err as fs; use itertools::Itertools; +use tempfile::tempdir_in; use tracing::debug; -use distribution_types::{AnyDist, Metadata, Resolution}; +use distribution_types::{AnyDist, LocalEditable, Metadata, Resolution}; use install_wheel_rs::linker::LinkMode; -use pep508_rs::Requirement; +use pep508_rs::{MarkerEnvironment, Requirement}; use platform_host::Platform; use platform_tags::Tags; use puffin_cache::Cache; -use puffin_client::RegistryClientBuilder; +use puffin_client::{RegistryClient, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; -use puffin_installer::{Downloader, InstallPlan, Reinstall, SitePackages}; +use puffin_installer::{ + BuiltEditable, Downloader, EditableMode, InstallPlan, Reinstall, SitePackages, +}; use puffin_interpreter::Virtualenv; use puffin_resolver::{ Manifest, PreReleaseMode, ResolutionGraph, ResolutionMode, ResolutionOptions, Resolver, @@ -73,17 +77,14 @@ pub(crate) async fn pip_install( // enough to let us remove this check. But right now, for large environments, it's an order of // magnitude faster to validate the environment than to resolve the requirements. if reinstall.is_none() && satisfied(&spec, &venv)? { - let s = if spec.requirements.len() == 1 { - "" - } else { - "s" - }; + let num_requirements = spec.requirements.len() + spec.editables.len(); + let s = if num_requirements == 1 { "" } else { "s" }; writeln!( printer, "{}", format!( "Audited {} in {}", - format!("{} package{}", spec.requirements.len(), s).bold(), + format!("{num_requirements} package{s}").bold(), elapsed(start.elapsed()) ) .dimmed() @@ -91,18 +92,61 @@ pub(crate) async fn pip_install( return Ok(ExitStatus::Success); } - let editable_requirements = spec.editables.clone(); + // Determine the compatible platform tags. + let tags = Tags::from_interpreter(venv.interpreter())?; + + // Determine the interpreter to use for resolution. + let interpreter = venv.interpreter().clone(); + + // Determine the markers to use for resolution. + let markers = venv.interpreter().markers(); + + // Instantiate a client. + let client = RegistryClientBuilder::new(cache.clone()) + .index_urls(index_urls.clone()) + .build(); + + let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer); + let build_dispatch = BuildDispatch::new( + client.clone(), + cache.clone(), + interpreter, + fs::canonicalize(venv.python_executable())?, + no_build, + index_urls.clone(), + ) + .with_options(options); + + // Build all editable distributions. The editables are shared between resolution and + // installation, and should live for the duration of the command. If an editable is already + // installed in the environment, we'll still re-build it here. + let editable_wheel_dir; + let editables = if spec.editables.is_empty() { + vec![] + } else { + editable_wheel_dir = tempdir_in(venv.root())?; + build_editables( + &spec.editables, + editable_wheel_dir.path(), + &cache, + &tags, + &client, + &build_dispatch, + printer, + ) + .await? + }; // Resolve the requirements. let resolution = match resolve( spec, + &editables, reinstall, - resolution_mode, - prerelease_mode, - &index_urls, - no_build, - exclude_newer, - &cache, + &tags, + markers, + &client, + &build_dispatch, + options, &venv, printer, ) @@ -124,11 +168,13 @@ pub(crate) async fn pip_install( // Sync the environment. install( &resolution, + &editables, reinstall, - &editable_requirements, link_mode, index_urls, - no_build, + &tags, + &client, + &build_dispatch, &cache, &venv, printer, @@ -184,20 +230,74 @@ fn specification( /// Returns `true` if the requirements are already satisfied. fn satisfied(spec: &RequirementsSpecification, venv: &Virtualenv) -> Result { - Ok(SitePackages::from_executable(venv)?.satisfies(&spec.requirements, &spec.constraints)?) + Ok(SitePackages::from_executable(venv)?.satisfies( + &spec.requirements, + &spec.editables, + &spec.constraints, + )?) +} + +/// Build a set of editable distributions. +async fn build_editables( + editables: &[EditableRequirement], + editable_wheel_dir: &Path, + cache: &Cache, + tags: &Tags, + client: &RegistryClient, + build_dispatch: &BuildDispatch, + mut printer: Printer, +) -> Result, Error> { + let start = std::time::Instant::now(); + + let downloader = Downloader::new(cache, tags, client, build_dispatch) + .with_reporter(DownloadReporter::from(printer).with_length(editables.len() as u64)); + + let editables: Vec = editables + .iter() + .map(|editable| match editable { + EditableRequirement::Path { path, .. } => Ok(LocalEditable { + requirement: editable.clone(), + path: path.clone(), + }), + EditableRequirement::Url(_) => { + bail!("Editable installs for URLs are not yet supported"); + } + }) + .collect::>()?; + + let editables: Vec<_> = downloader + .build_editables(editables, editable_wheel_dir) + .await + .context("Failed to build editables")? + .into_iter() + .collect(); + + let s = if editables.len() == 1 { "" } else { "s" }; + writeln!( + printer, + "{}", + format!( + "Built {} in {}", + format!("{} editable{}", editables.len(), s).bold(), + elapsed(start.elapsed()) + ) + .dimmed() + )?; + + Ok(editables) } /// Resolve a set of requirements, similar to running `pip-compile`. #[allow(clippy::too_many_arguments)] async fn resolve( spec: RequirementsSpecification, + editables: &[BuiltEditable], reinstall: &Reinstall, - resolution_mode: ResolutionMode, - prerelease_mode: PreReleaseMode, - index_urls: &IndexUrls, - no_build: bool, - exclude_newer: Option>, - cache: &Cache, + tags: &Tags, + markers: &MarkerEnvironment, + client: &RegistryClient, + build_dispatch: &BuildDispatch, + options: ResolutionOptions, venv: &Virtualenv, mut printer: Printer, ) -> Result { @@ -225,43 +325,28 @@ async fn resolve( .collect(), }; - // TODO(charlie): Support editable installs. + // Map the editables to their metadata. + let editables = editables + .iter() + .map(|built_editable| { + ( + built_editable.editable.clone(), + built_editable.metadata.clone(), + ) + }) + .collect(); + let manifest = Manifest::new( requirements, constraints, overrides, preferences, project, - Vec::new(), + editables, ); - let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer); - - // Determine the compatible platform tags. - let tags = Tags::from_interpreter(venv.interpreter())?; - - // Determine the interpreter to use for resolution. - let interpreter = venv.interpreter().clone(); - - // Determine the markers to use for resolution. - let markers = venv.interpreter().markers(); - - // Instantiate a client. - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_urls.clone()) - .build(); - - let build_dispatch = BuildDispatch::new( - client.clone(), - cache.clone(), - interpreter, - fs::canonicalize(venv.python_executable())?, - no_build, - index_urls.clone(), - ) - .with_options(options); // Resolve the dependencies. - let resolver = Resolver::new(manifest, options, markers, &tags, &client, &build_dispatch) + let resolver = Resolver::new(manifest, options, markers, tags, client, build_dispatch) .with_reporter(ResolverReporter::from(printer)); let resolution = resolver.resolve().await?; @@ -284,41 +369,41 @@ async fn resolve( #[allow(clippy::too_many_arguments)] async fn install( resolution: &Resolution, + built_editables: &[BuiltEditable], reinstall: &Reinstall, - editables: &[EditableRequirement], link_mode: LinkMode, index_urls: IndexUrls, - no_build: bool, + tags: &Tags, + client: &RegistryClient, + build_dispatch: &BuildDispatch, cache: &Cache, venv: &Virtualenv, mut printer: Printer, ) -> Result<(), Error> { let start = std::time::Instant::now(); - // Determine the current environment markers. - let tags = Tags::from_interpreter(venv.interpreter())?; - // Partition into those that should be linked from the cache (`local`), those that need to be // downloaded (`remote`), and those that should be removed (`extraneous`). let InstallPlan { local, remote, reinstalls, - editables: _, + editables, extraneous: _, } = InstallPlan::from_requirements( &resolution.requirements(), - editables, + &resolution.editable_requirements(), reinstall, &index_urls, cache, venv, - &tags, + tags, + EditableMode::Mutable, ) .context("Failed to determine installation plan")?; // Nothing to do. - if remote.is_empty() && local.is_empty() && reinstalls.is_empty() { + if remote.is_empty() && local.is_empty() && reinstalls.is_empty() && editables.is_empty() { let s = if resolution.len() == 1 { "" } else { "s" }; writeln!( printer, @@ -334,12 +419,7 @@ async fn install( return Ok(()); } - // Instantiate a client. - let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_urls.clone()) - .build(); - - // Resolve any registry-based requirements. + // Map any registry-based requirements back to those returned by the resolver. let remote = remote .iter() .map(|dist| { @@ -350,22 +430,25 @@ async fn install( }) .collect::>(); + // Map any local editable requirements back to those that were built ahead of time. + let built_editables = editables + .iter() + .map(|editable| { + let built_editable = built_editables + .iter() + .find(|built_editable| built_editable.editable.requirement == *editable) + .expect("Editable should be built"); + built_editable.wheel.clone() + }) + .collect::>(); + // Download, build, and unzip any missing distributions. let wheels = if remote.is_empty() { vec![] } else { let start = std::time::Instant::now(); - let build_dispatch = BuildDispatch::new( - client.clone(), - cache.clone(), - venv.interpreter().clone(), - fs::canonicalize(venv.python_executable())?, - no_build, - index_urls.clone(), - ); - - let downloader = Downloader::new(cache, &tags, &client, &build_dispatch) + let downloader = Downloader::new(cache, tags, client, build_dispatch) .with_reporter(DownloadReporter::from(printer).with_length(remote.len() as u64)); let wheels = downloader @@ -404,7 +487,11 @@ async fn install( } // Install the resolved distributions. - let wheels = wheels.into_iter().chain(local).collect::>(); + let wheels = wheels + .into_iter() + .chain(local) + .chain(built_editables) + .collect::>(); if !wheels.is_empty() { let start = std::time::Instant::now(); puffin_installer::Installer::new(venv) diff --git a/crates/puffin-cli/src/commands/pip_sync.rs b/crates/puffin-cli/src/commands/pip_sync.rs index 5444089ee..ce39fda49 100644 --- a/crates/puffin-cli/src/commands/pip_sync.rs +++ b/crates/puffin-cli/src/commands/pip_sync.rs @@ -15,7 +15,7 @@ use platform_tags::Tags; use puffin_cache::Cache; use puffin_client::RegistryClientBuilder; use puffin_dispatch::BuildDispatch; -use puffin_installer::{Downloader, InstallPlan, Reinstall, SitePackages}; +use puffin_installer::{Downloader, EditableMode, InstallPlan, Reinstall, SitePackages}; use puffin_interpreter::Virtualenv; use puffin_traits::OnceMap; use pypi_types::{IndexUrls, Yanked}; @@ -98,6 +98,7 @@ pub(crate) async fn sync_requirements( cache, &venv, &tags, + EditableMode::Immutable, ) .context("Failed to determine installation plan")?; @@ -193,7 +194,7 @@ pub(crate) async fn sync_requirements( DownloadReporter::from(printer).with_length((editables.len() + remote.len()) as u64), ); - // We must not cache editable wheels, so we put them in a temp dir. + // Build any editable requirements. let editable_wheel_dir = tempdir_in(venv.root())?; let built_editables = if editables.is_empty() { Vec::new() diff --git a/crates/puffin-cli/src/main.rs b/crates/puffin-cli/src/main.rs index f6c633b9c..5ab6314e9 100644 --- a/crates/puffin-cli/src/main.rs +++ b/crates/puffin-cli/src/main.rs @@ -246,6 +246,10 @@ struct PipInstallArgs { #[clap(short, long, group = "sources")] requirement: Vec, + /// Install the editable package based on the provided local file path. + #[clap(short, long, group = "sources")] + editable: Vec, + /// Constrain versions using the given requirements files. /// /// Constraints files are `requirements.txt`-like files that only control the _version_ of a @@ -480,6 +484,7 @@ async fn inner() -> Result { .package .into_iter() .map(RequirementsSource::Package) + .chain(args.editable.into_iter().map(RequirementsSource::Editable)) .chain(args.requirement.into_iter().map(RequirementsSource::from)) .collect::>(); let constraints = args diff --git a/crates/puffin-cli/tests/pip_compile.rs b/crates/puffin-cli/tests/pip_compile.rs index eab1cf139..24ddf5871 100644 --- a/crates/puffin-cli/tests/pip_compile.rs +++ b/crates/puffin-cli/tests/pip_compile.rs @@ -2560,7 +2560,7 @@ fn compile_editable() -> Result<()> { requirements_in.write_str(indoc! {r" -e ../../scripts/editable-installs/poetry_editable -e ../../scripts/editable-installs/maturin_editable - boltons # normal depedency for comparison + boltons # normal dependency for comparison " })?; diff --git a/crates/puffin-cli/tests/pip_install.rs b/crates/puffin-cli/tests/pip_install.rs index d4e646e53..1b1b7722d 100644 --- a/crates/puffin-cli/tests/pip_install.rs +++ b/crates/puffin-cli/tests/pip_install.rs @@ -425,3 +425,128 @@ fn allow_incompatibilities() -> Result<()> { Ok(()) } + +#[test] +fn install_editable() -> Result<()> { + let temp_dir = assert_fs::TempDir::new()?; + let cache_dir = assert_fs::TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + // Install the editable package. + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip-install") + .arg("-e") + .arg("../../scripts/editable-installs/poetry_editable") + .arg("--cache-dir") + .arg(cache_dir.path()) + .env("VIRTUAL_ENV", venv.as_os_str()) + .env("CARGO_TARGET_DIR", "../../../target/target_install_editable"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Built 1 editable in [TIME] + Resolved 2 packages in [TIME] + Downloaded 1 package in [TIME] + Installed 2 packages in [TIME] + + numpy==1.26.2 + + poetry-editable @ ../../scripts/editable-installs/poetry_editable + "###); + }); + + // Install it again (no-op). + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip-install") + .arg("-e") + .arg("../../scripts/editable-installs/poetry_editable") + .arg("--cache-dir") + .arg(cache_dir.path()) + .env("VIRTUAL_ENV", venv.as_os_str()) + .env("CARGO_TARGET_DIR", "../../../target/target_install_editable"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Audited 1 package in [TIME] + "###); + }); + + // Add another, non-editable dependency. + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip-install") + .arg("-e") + .arg("../../scripts/editable-installs/poetry_editable") + .arg("black") + .arg("--cache-dir") + .arg(cache_dir.path()) + .env("VIRTUAL_ENV", venv.as_os_str()) + .env("CARGO_TARGET_DIR", "../../../target/target_install_editable"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Built 1 editable in [TIME] + Resolved 15 packages in [TIME] + Downloaded 13 packages in [TIME] + Installed 14 packages in [TIME] + + aiohttp==3.9.1 + + aiosignal==1.3.1 + + attrs==23.1.0 + + black==23.12.0 + + click==8.1.7 + + frozenlist==1.4.1 + + idna==3.6 + + multidict==6.0.4 + + mypy-extensions==1.0.0 + + packaging==23.2 + + pathspec==0.12.1 + + platformdirs==4.1.0 + - poetry-editable==0.1.0 + + poetry-editable @ ../../scripts/editable-installs/poetry_editable + + yarl==1.9.4 + "###); + }); + + // Add another, editable dependency. + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip-install") + .arg("-e") + .arg("../../scripts/editable-installs/poetry_editable") + .arg("black") + .arg("-e") + .arg("../../scripts/editable-installs/maturin_editable") + .arg("--cache-dir") + .arg(cache_dir.path()) + .env("VIRTUAL_ENV", venv.as_os_str()) + .env("CARGO_TARGET_DIR", "../../../target/target_install_editable"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Built 2 editables in [TIME] + Resolved 16 packages in [TIME] + Installed 2 packages in [TIME] + + maturin-editable @ ../../scripts/editable-installs/maturin_editable + - poetry-editable==0.1.0 + + poetry-editable @ ../../scripts/editable-installs/poetry_editable + "###); + }); + + Ok(()) +} diff --git a/crates/puffin-cli/tests/pip_sync.rs b/crates/puffin-cli/tests/pip_sync.rs index 2a43ec625..5c2f2d5fe 100644 --- a/crates/puffin-cli/tests/pip_sync.rs +++ b/crates/puffin-cli/tests/pip_sync.rs @@ -2090,7 +2090,7 @@ fn reinstall_package() -> Result<()> { } #[test] -fn install_editable() -> Result<()> { +fn sync_editable() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; let cache_dir = assert_fs::TempDir::new()?; let venv = create_venv_py312(&temp_dir, &cache_dir); diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs index d38fce2f4..07f340145 100644 --- a/crates/puffin-dispatch/src/lib.rs +++ b/crates/puffin-dispatch/src/lib.rs @@ -16,7 +16,7 @@ use platform_tags::Tags; use puffin_build::{SourceBuild, SourceBuildContext}; use puffin_cache::Cache; use puffin_client::RegistryClient; -use puffin_installer::{Downloader, InstallPlan, Installer, Reinstall}; +use puffin_installer::{Downloader, EditableMode, InstallPlan, Installer, Reinstall}; use puffin_interpreter::{Interpreter, Virtualenv}; use puffin_resolver::{Manifest, ResolutionOptions, Resolver}; use puffin_traits::{BuildContext, BuildKind, OnceMap}; @@ -147,6 +147,7 @@ impl BuildContext for BuildDispatch { self.cache(), venv, &tags, + EditableMode::default(), )?; // Resolve any registry-based requirements. diff --git a/crates/puffin-installer/src/downloader.rs b/crates/puffin-installer/src/downloader.rs index b4aa24598..b56c11cfa 100644 --- a/crates/puffin-installer/src/downloader.rs +++ b/crates/puffin-installer/src/downloader.rs @@ -13,14 +13,8 @@ use puffin_cache::Cache; use puffin_client::RegistryClient; use puffin_distribution::{DistributionDatabase, DistributionDatabaseError, LocalWheel, Unzip}; use puffin_traits::{BuildContext, OnceMap}; -use pypi_types::Metadata21; -#[derive(Debug, Clone)] -pub struct BuiltEditable { - pub editable: LocalEditable, - pub wheel: CachedDist, - pub metadata: Metadata21, -} +use crate::editable::BuiltEditable; #[derive(thiserror::Error, Debug)] pub enum Error { diff --git a/crates/puffin-installer/src/editable.rs b/crates/puffin-installer/src/editable.rs new file mode 100644 index 000000000..194611d31 --- /dev/null +++ b/crates/puffin-installer/src/editable.rs @@ -0,0 +1,9 @@ +use distribution_types::{CachedDist, LocalEditable}; +use pypi_types::Metadata21; + +#[derive(Debug, Clone)] +pub struct BuiltEditable { + pub editable: LocalEditable, + pub wheel: CachedDist, + pub metadata: Metadata21, +} diff --git a/crates/puffin-installer/src/lib.rs b/crates/puffin-installer/src/lib.rs index 17c090bb0..54a75da20 100644 --- a/crates/puffin-installer/src/lib.rs +++ b/crates/puffin-installer/src/lib.rs @@ -1,10 +1,12 @@ pub use downloader::{Downloader, Reporter as DownloadReporter}; +pub use editable::BuiltEditable; pub use installer::{Installer, Reporter as InstallReporter}; -pub use plan::{InstallPlan, Reinstall}; +pub use plan::{EditableMode, InstallPlan, Reinstall}; pub use site_packages::SitePackages; pub use uninstall::uninstall; mod downloader; +mod editable; mod installer; mod plan; mod site_packages; diff --git a/crates/puffin-installer/src/plan.rs b/crates/puffin-installer/src/plan.rs index b2dd617f7..cf4304b89 100644 --- a/crates/puffin-installer/src/plan.rs +++ b/crates/puffin-installer/src/plan.rs @@ -46,6 +46,7 @@ pub struct InstallPlan { impl InstallPlan { /// Partition a set of requirements into those that should be linked from the cache, those that /// need to be downloaded, and those that should be removed. + #[allow(clippy::too_many_arguments)] pub fn from_requirements( requirements: &[Requirement], editable_requirements: &[EditableRequirement], @@ -54,6 +55,7 @@ impl InstallPlan { cache: &Cache, venv: &Virtualenv, tags: &Tags, + editable_mode: EditableMode, ) -> Result { // Index all the already-installed packages in site-packages. let mut site_packages = @@ -90,8 +92,17 @@ impl InstallPlan { } editables.push(editable.clone()); } else { - if site_packages.remove_editable(editable.raw()).is_some() { - debug!("Treating editable requirement as immutable: {editable}"); + if let Some(dist) = site_packages.remove_editable(editable.raw()) { + match editable_mode { + EditableMode::Immutable => { + debug!("Treating editable requirement as immutable: {editable}"); + } + EditableMode::Mutable => { + debug!("Treating editable requirement as mutable: {editable}"); + reinstalls.push(dist); + editables.push(editable.clone()); + } + } } else { editables.push(editable.clone()); } @@ -351,3 +362,13 @@ impl Reinstall { matches!(self, Self::None) } } + +#[derive(Debug, Default, Copy, Clone)] +pub enum EditableMode { + /// Assume that editables are immutable, such that they're left untouched if already present + /// in the environment. + #[default] + Immutable, + /// Assume that editables are mutable, such that they're always reinstalled. + Mutable, +} diff --git a/crates/puffin-installer/src/site_packages.rs b/crates/puffin-installer/src/site_packages.rs index a3dfa5175..0a6543e44 100644 --- a/crates/puffin-installer/src/site_packages.rs +++ b/crates/puffin-installer/src/site_packages.rs @@ -10,6 +10,7 @@ use pep440_rs::{Version, VersionSpecifiers}; use pep508_rs::Requirement; use puffin_interpreter::Virtualenv; use puffin_normalize::PackageName; +use requirements_txt::EditableRequirement; /// An index over the packages installed in an environment. /// @@ -221,12 +222,29 @@ impl<'a> SitePackages<'a> { pub fn satisfies( &self, requirements: &[Requirement], + editables: &[EditableRequirement], constraints: &[Requirement], ) -> Result { let mut requirements = requirements.to_vec(); let mut seen = FxHashSet::with_capacity_and_hasher(requirements.len(), BuildHasherDefault::default()); + // Verify that all editable requirements are met. + for requirement in editables { + let Some(distribution) = self + .by_url + .get(requirement.raw()) + .map(|idx| &self.distributions[*idx]) + else { + // The package isn't installed. + return Ok(false); + }; + + // Recurse into the dependencies. + requirements.extend(distribution.metadata()?.requires_dist); + } + + // Verify that all non-editable requirements are met. while let Some(requirement) = requirements.pop() { if !requirement.evaluate_markers(self.venv.interpreter().markers(), &[]) { continue; diff --git a/crates/puffin-resolver/src/resolution.rs b/crates/puffin-resolver/src/resolution.rs index 951035af9..faf9ddbc3 100644 --- a/crates/puffin-resolver/src/resolution.rs +++ b/crates/puffin-resolver/src/resolution.rs @@ -10,7 +10,7 @@ use pubgrub::type_aliases::SelectedDependencies; use rustc_hash::FxHashMap; use url::Url; -use distribution_types::{Dist, DistributionId, Identifier, LocalEditable, Metadata, PackageId}; +use distribution_types::{Dist, LocalEditable, Metadata, PackageId}; use pep440_rs::Version; use pep508_rs::{Requirement, VerbatimUrl}; use puffin_normalize::{ExtraName, PackageName}; @@ -28,7 +28,7 @@ pub struct ResolutionGraph { /// The underlying graph. petgraph: petgraph::graph::Graph, petgraph::Directed>, /// The set of editable requirements in this resolution. - editables: FxHashMap, + editables: FxHashMap, /// Any diagnostics that were encountered while building the graph. diagnostics: Vec, } @@ -41,7 +41,7 @@ impl ResolutionGraph { distributions: &OnceMap, redirects: &OnceMap, state: &State, PubGrubPriority>, - editables: FxHashMap, + editables: FxHashMap, ) -> Result { // TODO(charlie): petgraph is a really heavy and unnecessary dependency here. We should // write our own graph, given that our requirements are so simple. @@ -66,11 +66,15 @@ impl ResolutionGraph { inverse.insert(package_name, index); } PubGrubPackage::Package(package_name, None, Some(url)) => { - let url = redirects.get(url).map_or_else( - || url.clone(), - |url| VerbatimUrl::unknown(url.value().clone()), - ); - let pinned_package = Dist::from_url(package_name.clone(), url)?; + let pinned_package = if let Some((editable, _)) = editables.get(package_name) { + Dist::from_editable(package_name.clone(), editable.clone())? + } else { + let url = redirects.get(url).map_or_else( + || url.clone(), + |url| VerbatimUrl::unknown(url.value().clone()), + ); + Dist::from_url(package_name.clone(), url)? + }; let index = petgraph.add_node(pinned_package); inverse.insert(package_name, index); @@ -173,13 +177,6 @@ impl ResolutionGraph { /// Return the set of [`Requirement`]s that this graph represents. pub fn requirements(&self) -> Vec { - // Collect and sort all packages. - let mut nodes = self - .petgraph - .node_indices() - .map(|node| (node, &self.petgraph[node])) - .collect::>(); - nodes.sort_unstable_by_key(|(_, package)| package.name()); self.petgraph .node_indices() .map(|node| &self.petgraph[node]) @@ -199,14 +196,6 @@ impl ResolutionGraph { ) -> &petgraph::graph::Graph, petgraph::Directed> { &self.petgraph } - - /// Return the set of editable requirements in this resolution. - /// - /// The editable requirements themselves are unchanged, but their dependencies were added to the general - /// list of dependencies. - pub fn editables(&self) -> &FxHashMap { - &self.editables - } } /// Write the graph in the `{name}=={version}` format of requirements.txt that pip uses. @@ -222,9 +211,8 @@ impl std::fmt::Display for ResolutionGraph { // Print out the dependency graph. for (index, package) in nodes { - if let Some((editable_requirement, _)) = self.editables.get(&package.distribution_id()) - { - writeln!(f, "-e {editable_requirement}")?; + if let Some((editable, _)) = self.editables.get(package.name()) { + writeln!(f, "-e {editable}")?; } else { writeln!(f, "{package}")?; } diff --git a/crates/puffin-resolver/src/resolver.rs b/crates/puffin-resolver/src/resolver.rs index dc0ba65d1..7d95408d1 100644 --- a/crates/puffin-resolver/src/resolver.rs +++ b/crates/puffin-resolver/src/resolver.rs @@ -19,8 +19,7 @@ use url::Url; use distribution_filename::WheelFilename; use distribution_types::{ - BuiltDist, Dist, DistributionId, Identifier, LocalEditable, Metadata, PackageId, SourceDist, - VersionOrUrl, + BuiltDist, Dist, LocalEditable, Metadata, PackageId, SourceDist, VersionOrUrl, }; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; @@ -157,7 +156,7 @@ pub struct Resolver<'a, Provider: ResolverProvider> { markers: &'a MarkerEnvironment, selector: CandidateSelector, index: Arc, - editables: FxHashMap, + editables: FxHashMap, reporter: Option>, provider: Provider, } @@ -204,15 +203,17 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { // Determine all the editable requirements. let mut editables = FxHashMap::default(); for (editable_requirement, metadata) in &manifest.editables { - let dist = Dist::from_url(metadata.name.clone(), editable_requirement.url().clone()) + // Convert the editable requirement into a distribution. + let dist = Dist::from_editable(metadata.name.clone(), editable_requirement.clone()) .expect("This is a valid distribution"); + // Mock editable responses. index.distributions.register(&dist.package_id()); index .distributions .done(dist.package_id(), metadata.clone()); editables.insert( - dist.distribution_id(), + dist.name().clone(), (editable_requirement.clone(), metadata.clone()), ); }