diff --git a/Cargo.lock b/Cargo.lock
index b20b5f7b6..4b602d5df 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2330,8 +2330,11 @@ dependencies = [
 name = "puffin-cache"
 version = "0.0.1"
 dependencies = [
+ "clap",
+ "directories",
  "hex",
  "seahash",
+ "tempfile",
  "url",
 ]
 
@@ -2348,7 +2351,6 @@ dependencies = [
  "chrono",
  "clap",
  "colored",
- "directories",
  "fs-err",
  "futures",
  "gourgeist",
@@ -2365,6 +2367,7 @@ dependencies = [
  "platform-tags",
  "predicates",
  "pubgrub",
+ "puffin-cache",
  "puffin-client",
  "puffin-dispatch",
  "puffin-distribution",
@@ -2425,7 +2428,6 @@ dependencies = [
  "anyhow",
  "clap",
  "colored",
- "directories",
  "distribution-filename",
  "fs-err",
  "futures",
@@ -2437,6 +2439,7 @@ dependencies = [
  "platform-host",
  "platform-tags",
  "puffin-build",
+ "puffin-cache",
  "puffin-client",
  "puffin-dispatch",
  "puffin-interpreter",
diff --git a/crates/puffin-cache/Cargo.toml b/crates/puffin-cache/Cargo.toml
index 8def0fe6f..2fa37268e 100644
--- a/crates/puffin-cache/Cargo.toml
+++ b/crates/puffin-cache/Cargo.toml
@@ -11,6 +11,9 @@ authors = { workspace = true }
 license = { workspace = true }
 
 [dependencies]
+clap = { workspace = true, features = ["derive"], optional = true }
+directories = { workspace = true }
 hex = { workspace = true }
 seahash = { workspace = true }
+tempfile = { workspace = true }
 url = { workspace = true }
diff --git a/crates/puffin-cache/src/cli.rs b/crates/puffin-cache/src/cli.rs
new file mode 100644
index 000000000..9f6f5d86d
--- /dev/null
+++ b/crates/puffin-cache/src/cli.rs
@@ -0,0 +1,72 @@
+#![cfg(feature = "clap")]
+
+use std::io;
+use std::path::PathBuf;
+
+use clap::Parser;
+use directories::ProjectDirs;
+use tempfile::{tempdir, TempDir};
+
+#[derive(Parser, Debug, Clone)]
+pub struct CacheArgs {
+    /// Avoid reading from or writing to the cache.
+    #[arg(global = true, long, short)]
+    no_cache: bool,
+
+    /// Path to the cache directory.
+    #[arg(global = true, long, env = "PUFFIN_CACHE_DIR")]
+    cache_dir: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct CacheDir {
+    /// The cache directory.
+    cache_dir: PathBuf,
+    /// A temporary cache directory, if the user requested `--no-cache`. Included to ensure that
+    /// the temporary directory exists for the length of the operation, but is dropped at the end
+    /// as appropriate.
+    #[allow(dead_code)]
+    tempdir: Option<TempDir>,
+}
+
+impl TryFrom<CacheArgs> for CacheDir {
+    type Error = io::Error;
+
+    /// Prefer, in order:
+    /// 1. A temporary cache directory, if the user requested `--no-cache`.
+    /// 2. The specific cache directory specified by the user via `--cache-dir` or `PUFFIN_CACHE_DIR`.
+    /// 3. The system-appropriate cache directory.
+    /// 4. A `.puffin_cache` directory in the current working directory.
+    fn try_from(value: CacheArgs) -> Result<Self, Self::Error> {
+        let project_dirs = ProjectDirs::from("", "", "puffin");
+        if value.no_cache {
+            let tempdir = tempdir()?;
+            let cache_dir = tempdir.path().to_path_buf();
+            Ok(Self {
+                cache_dir,
+                tempdir: Some(tempdir),
+            })
+        } else if let Some(cache_dir) = value.cache_dir {
+            Ok(Self {
+                cache_dir,
+                tempdir: None,
+            })
+        } else if let Some(project_dirs) = project_dirs {
+            Ok(Self {
+                cache_dir: project_dirs.cache_dir().to_path_buf(),
+                tempdir: None,
+            })
+        } else {
+            Ok(Self {
+                cache_dir: PathBuf::from(".puffin_cache"),
+                tempdir: None,
+            })
+        }
+    }
+}
+
+impl CacheDir {
+    pub fn path(&self) -> &PathBuf {
+        &self.cache_dir
+    }
+}
diff --git a/crates/puffin-cache/src/lib.rs b/crates/puffin-cache/src/lib.rs
index e266b5261..000f87eec 100644
--- a/crates/puffin-cache/src/lib.rs
+++ b/crates/puffin-cache/src/lib.rs
@@ -3,10 +3,13 @@ use std::hash::Hasher;
 use seahash::SeaHasher;
 
 pub use canonical_url::{CanonicalUrl, RepositoryUrl};
+#[cfg(feature = "clap")]
+pub use cli::{CacheArgs, CacheDir};
 pub use digest::digest;
 
 mod cache_key;
 mod canonical_url;
+mod cli;
 mod digest;
 
 /// A trait for types that can be hashed in a stable way across versions and platforms.
diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml
index bb838bef9..5a4f2ea9e 100644
--- a/crates/puffin-cli/Cargo.toml
+++ b/crates/puffin-cli/Cargo.toml
@@ -20,16 +20,17 @@ pep440_rs = { path = "../pep440-rs" }
 pep508_rs = { path = "../pep508-rs" }
 platform-host = { path = "../platform-host" }
 platform-tags = { path = "../platform-tags" }
+puffin-cache = { path = "../puffin-cache" }
 puffin-client = { path = "../puffin-client" }
 puffin-dispatch = { path = "../puffin-dispatch" }
 puffin-distribution = { path = "../puffin-distribution" }
 puffin-installer = { path = "../puffin-installer" }
 puffin-interpreter = { path = "../puffin-interpreter" }
 puffin-normalize = { path = "../puffin-normalize" }
-pypi-types = { path = "../pypi-types" }
-requirements-txt = { path = "../requirements-txt" }
 puffin-resolver = { path = "../puffin-resolver", features = ["clap"] }
 puffin-workspace = { path = "../puffin-workspace" }
+pypi-types = { path = "../pypi-types" }
+requirements-txt = { path = "../requirements-txt" }
 
 anstream = { workspace = true }
 anyhow = { workspace = true }
@@ -38,7 +39,6 @@ cacache = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 colored = { workspace = true }
-directories = { workspace = true }
 fs-err = { workspace = true, features = ["tokio"] }
 futures = { workspace = true }
 indicatif = { workspace = true }
diff --git a/crates/puffin-cli/src/main.rs b/crates/puffin-cli/src/main.rs
index d02728af0..c13d94ced 100644
--- a/crates/puffin-cli/src/main.rs
+++ b/crates/puffin-cli/src/main.rs
@@ -1,5 +1,4 @@
-use std::borrow::Cow;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::ExitCode;
 use std::str::FromStr;
 
@@ -7,10 +6,9 @@ use anyhow::Result;
 use chrono::{DateTime, Days, NaiveDate, NaiveTime, Utc};
 use clap::{Args, Parser, Subcommand};
 use colored::Colorize;
-use directories::ProjectDirs;
-use tempfile::tempdir;
 use url::Url;
 
+use puffin_cache::{CacheArgs, CacheDir};
 use puffin_normalize::{ExtraName, PackageName};
 use puffin_resolver::{PreReleaseMode, ResolutionMode};
 use requirements::ExtrasSpecification;
@@ -58,13 +56,8 @@ struct Cli {
     #[arg(global = true, long, short, conflicts_with = "quiet")]
     verbose: bool,
 
-    /// Avoid reading from or writing to the cache.
-    #[arg(global = true, long, short)]
-    no_cache: bool,
-
-    /// Path to the cache directory.
-    #[arg(global = true, long, env = "PUFFIN_CACHE_DIR")]
-    cache_dir: Option<PathBuf>,
+    #[command(flatten)]
+    cache_args: CacheArgs,
 }
 
 #[derive(Subcommand)]
@@ -256,21 +249,7 @@ async fn inner() -> Result<ExitStatus> {
         printer::Printer::Default
     };
 
-    // Prefer, in order:
-    // 1. A temporary cache directory, if the user requested `--no-cache`.
-    // 2. The specific cache directory specified by the user via `--cache-dir` or `PUFFIN_CACHE_DIR`.
-    // 3. The system-appropriate cache directory.
-    // 4. A `.puffin_cache` directory in the current working directory.
-    let project_dirs = ProjectDirs::from("", "", "puffin");
-    let cache_dir = if cli.no_cache {
-        Cow::Owned(tempdir()?.into_path())
-    } else if let Some(cache_dir) = cli.cache_dir {
-        Cow::Owned(cache_dir)
-    } else if let Some(project_dirs) = project_dirs.as_ref() {
-        Cow::Borrowed(project_dirs.cache_dir())
-    } else {
-        Cow::Borrowed(Path::new(".puffin_cache"))
-    };
+    let cache_dir = CacheDir::try_from(cli.cache_args)?;
 
     match cli.command {
         Commands::PipCompile(args) => {
@@ -307,7 +286,7 @@ async fn inner() -> Result<ExitStatus> {
                 args.no_build,
                 args.python_version,
                 args.exclude_newer,
-                &cache_dir,
+                cache_dir.path(),
                 printer,
             )
             .await
@@ -325,7 +304,7 @@ async fn inner() -> Result<ExitStatus> {
                 args.link_mode.unwrap_or_default(),
                 index_urls,
                 args.no_build,
-                &cache_dir,
+                cache_dir.path(),
                 printer,
             )
             .await
@@ -337,10 +316,10 @@ async fn inner() -> Result<ExitStatus> {
                 .map(RequirementsSource::from)
                 .chain(args.requirement.into_iter().map(RequirementsSource::from))
                 .collect::<Vec<_>>();
-            commands::pip_uninstall(&sources, &cache_dir, printer).await
+            commands::pip_uninstall(&sources, cache_dir.path(), printer).await
         }
-        Commands::Clean => commands::clean(&cache_dir, printer),
-        Commands::Freeze => commands::freeze(&cache_dir, printer),
+        Commands::Clean => commands::clean(cache_dir.path(), printer),
+        Commands::Freeze => commands::freeze(cache_dir.path(), printer),
         Commands::Venv(args) => commands::venv(&args.name, args.python.as_deref(), printer),
         Commands::Add(args) => commands::add(&args.name, printer),
         Commands::Remove(args) => commands::remove(&args.name, printer),
diff --git a/crates/puffin-dev/Cargo.toml b/crates/puffin-dev/Cargo.toml
index b15932b28..6ded24d44 100644
--- a/crates/puffin-dev/Cargo.toml
+++ b/crates/puffin-dev/Cargo.toml
@@ -17,6 +17,7 @@ pep508_rs = { path = "../pep508-rs" }
 platform-host = { path = "../platform-host" }
 platform-tags = { path = "../platform-tags" }
 puffin-build = { path = "../puffin-build" }
+puffin-cache = { path = "../puffin-cache", features = ["clap"] }
 puffin-client = { path = "../puffin-client" }
 puffin-dispatch = { path = "../puffin-dispatch" }
 puffin-interpreter = { path = "../puffin-interpreter" }
@@ -27,7 +28,6 @@ anstream = { workspace = true }
 anyhow = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 colored = { workspace = true }
-directories = { workspace = true }
 fs-err = { workspace = true }
 futures = { workspace = true }
 indicatif = { workspace = true }
diff --git a/crates/puffin-dev/src/build.rs b/crates/puffin-dev/src/build.rs
index 90e1dafdd..8c86046c6 100644
--- a/crates/puffin-dev/src/build.rs
+++ b/crates/puffin-dev/src/build.rs
@@ -3,10 +3,10 @@ use std::path::PathBuf;
 
 use anyhow::{Context, Result};
 use clap::Parser;
-use directories::ProjectDirs;
 use fs_err as fs;
 
 use platform_host::Platform;
+use puffin_cache::{CacheArgs, CacheDir};
 use puffin_client::RegistryClientBuilder;
 use puffin_dispatch::BuildDispatch;
 use puffin_interpreter::Virtualenv;
@@ -25,6 +25,8 @@ pub(crate) struct BuildArgs {
     sdist: PathBuf,
     /// The subdirectory to build within the source distribution.
     subdirectory: Option<PathBuf>,
+    #[command(flatten)]
+    cache_args: CacheArgs,
 }
 
 /// Build a source distribution to a wheel
@@ -36,19 +38,14 @@ pub(crate) async fn build(args: BuildArgs) -> Result<PathBuf> {
         env::current_dir()?
     };
 
-    let project_dirs = ProjectDirs::from("", "", "puffin");
-    let cache = project_dirs
-        .as_ref()
-        .map(|project_dirs| project_dirs.cache_dir().to_path_buf())
-        .or_else(|| Some(tempfile::tempdir().ok()?.into_path()))
-        .unwrap_or_else(|| PathBuf::from(".puffin_cache"));
+    let cache_dir = CacheDir::try_from(args.cache_args)?;
 
     let platform = Platform::current()?;
-    let venv = Virtualenv::from_env(platform, Some(&cache))?;
+    let venv = Virtualenv::from_env(platform, Some(cache_dir.path()))?;
 
     let build_dispatch = BuildDispatch::new(
-        RegistryClientBuilder::new(cache.clone()).build(),
-        cache,
+        RegistryClientBuilder::new(cache_dir.path().clone()).build(),
+        cache_dir.path().clone(),
         venv.interpreter_info().clone(),
         fs::canonicalize(venv.python_executable())?,
         false,
diff --git a/crates/puffin-dev/src/resolve_cli.rs b/crates/puffin-dev/src/resolve_cli.rs
index 62aa1b51e..4769633cf 100644
--- a/crates/puffin-dev/src/resolve_cli.rs
+++ b/crates/puffin-dev/src/resolve_cli.rs
@@ -3,11 +3,11 @@ use std::path::PathBuf;
 
 use anstream::println;
 use clap::Parser;
-use directories::ProjectDirs;
 use itertools::Itertools;
 
 use pep508_rs::Requirement;
 use platform_host::Platform;
+use puffin_cache::{CacheArgs, CacheDir};
 use puffin_client::RegistryClientBuilder;
 use puffin_dispatch::BuildDispatch;
 use puffin_interpreter::Virtualenv;
@@ -25,21 +25,18 @@ pub(crate) struct ResolveCliArgs {
     /// cached wheels of already built source distributions will be reused.
     #[clap(long)]
     no_build: bool,
+    #[command(flatten)]
+    cache_args: CacheArgs,
 }
 
 pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> anyhow::Result<()> {
-    let project_dirs = ProjectDirs::from("", "", "puffin");
-    let cache = project_dirs
-        .as_ref()
-        .map(|project_dirs| project_dirs.cache_dir().to_path_buf())
-        .or_else(|| Some(tempfile::tempdir().ok()?.into_path()))
-        .unwrap_or_else(|| PathBuf::from(".puffin_cache"));
+    let cache_dir = CacheDir::try_from(args.cache_args)?;
 
     let platform = Platform::current()?;
-    let venv = Virtualenv::from_env(platform, Some(&cache))?;
+    let venv = Virtualenv::from_env(platform, Some(cache_dir.path()))?;
     let build_dispatch = BuildDispatch::new(
-        RegistryClientBuilder::new(cache.clone()).build(),
-        cache.clone(),
+        RegistryClientBuilder::new(cache_dir.path().clone()).build(),
+        cache_dir.path().clone(),
         venv.interpreter_info().clone(),
         fs::canonicalize(venv.python_executable())?,
         args.no_build,
diff --git a/crates/puffin-dev/src/resolve_many.rs b/crates/puffin-dev/src/resolve_many.rs
index 463e7a27c..58d15866c 100644
--- a/crates/puffin-dev/src/resolve_many.rs
+++ b/crates/puffin-dev/src/resolve_many.rs
@@ -2,32 +2,29 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 
+use anyhow::Result;
 use clap::Parser;
-use directories::ProjectDirs;
 use fs_err as fs;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use indicatif::ProgressStyle;
-use tokio::sync::Semaphore;
-use tokio::time::Instant;
-use tracing::{info, info_span, span, Level, Span};
-use tracing_indicatif::span_ext::IndicatifSpanExt;
-
 use pep508_rs::Requirement;
 use platform_host::Platform;
+use puffin_cache::{CacheArgs, CacheDir};
 use puffin_client::RegistryClientBuilder;
 use puffin_dispatch::BuildDispatch;
 use puffin_interpreter::Virtualenv;
 use puffin_traits::BuildContext;
+use tokio::sync::Semaphore;
+use tokio::time::Instant;
+use tracing::{info, info_span, span, Level, Span};
+use tracing_indicatif::span_ext::IndicatifSpanExt;
 
 #[derive(Parser)]
 pub(crate) struct ResolveManyArgs {
     list: PathBuf,
     #[clap(long)]
     limit: Option<usize>,
-    /// Path to the cache directory.
-    #[arg(global = true, long, env = "PUFFIN_CACHE_DIR")]
-    cache_dir: Option<PathBuf>,
     /// Don't build source distributions. This means resolving will not run arbitrary code. The
     /// cached wheels of already built source distributions will be reused.
     #[clap(long)]
@@ -35,19 +32,12 @@ pub(crate) struct ResolveManyArgs {
     /// Run this many tasks in parallel
     #[clap(long, default_value = "50")]
     num_tasks: usize,
+    #[command(flatten)]
+    cache_args: CacheArgs,
 }
 
-pub(crate) async fn resolve_many(args: ResolveManyArgs) -> anyhow::Result<()> {
-    let project_dirs = ProjectDirs::from("", "", "puffin");
-    let cache = args
-        .cache_dir
-        .or_else(|| {
-            project_dirs
-                .as_ref()
-                .map(|project_dirs| project_dirs.cache_dir().to_path_buf())
-        })
-        .or_else(|| Some(tempfile::tempdir().ok()?.into_path()))
-        .unwrap_or_else(|| PathBuf::from(".puffin_cache"));
+pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {
+    let cache_dir = CacheDir::try_from(args.cache_args)?;
 
     let data = fs::read_to_string(&args.list)?;
     let lines = data.lines().map(Requirement::from_str);
@@ -58,10 +48,10 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> anyhow::Result<()> {
     };
 
     let platform = Platform::current()?;
-    let venv = Virtualenv::from_env(platform, Some(&cache))?;
+    let venv = Virtualenv::from_env(platform, Some(cache_dir.path()))?;
     let build_dispatch = BuildDispatch::new(
-        RegistryClientBuilder::new(cache.clone()).build(),
-        cache.clone(),
+        RegistryClientBuilder::new(cache_dir.path().clone()).build(),
+        cache_dir.path().clone(),
         venv.interpreter_info().clone(),
         fs::canonicalize(venv.python_executable())?,
         args.no_build,
diff --git a/crates/puffin-dev/src/wheel_metadata.rs b/crates/puffin-dev/src/wheel_metadata.rs
index 811899251..66c491a5e 100644
--- a/crates/puffin-dev/src/wheel_metadata.rs
+++ b/crates/puffin-dev/src/wheel_metadata.rs
@@ -1,39 +1,24 @@
-use std::borrow::Cow;
-use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
 use clap::Parser;
-use directories::ProjectDirs;
-use tempfile::tempdir;
 use url::Url;
 
+use anyhow::Result;
 use distribution_filename::WheelFilename;
+use puffin_cache::{CacheArgs, CacheDir};
 use puffin_client::RegistryClientBuilder;
 
 #[derive(Parser)]
 pub(crate) struct WheelMetadataArgs {
     url: Url,
-    /// Avoid reading from or writing to the cache.
-    #[arg(global = true, long, short)]
-    no_cache: bool,
-    /// Path to the cache directory.
-    #[arg(global = true, long, env = "PUFFIN_CACHE_DIR")]
-    cache_dir: Option<PathBuf>,
+    #[command(flatten)]
+    cache_args: CacheArgs,
 }
 
-pub(crate) async fn wheel_metadata(args: WheelMetadataArgs) -> anyhow::Result<()> {
-    let project_dirs = ProjectDirs::from("", "", "puffin");
-    // https://github.com/astral-sh/puffin/issues/366
-    let cache_dir = if args.no_cache {
-        Cow::Owned(tempdir()?.into_path())
-    } else if let Some(cache_dir) = args.cache_dir {
-        Cow::Owned(cache_dir)
-    } else if let Some(project_dirs) = project_dirs.as_ref() {
-        Cow::Borrowed(project_dirs.cache_dir())
-    } else {
-        Cow::Borrowed(Path::new(".puffin_cache"))
-    };
-    let client = RegistryClientBuilder::new(cache_dir).build();
+pub(crate) async fn wheel_metadata(args: WheelMetadataArgs) -> Result<()> {
+    let cache_dir = CacheDir::try_from(args.cache_args)?;
+
+    let client = RegistryClientBuilder::new(cache_dir.path().clone()).build();
 
     let filename = WheelFilename::from_str(
         args.url
diff --git a/crates/puffin-installer/src/builder.rs b/crates/puffin-installer/src/builder.rs
index 62cfe1d0d..4d9c065da 100644
--- a/crates/puffin-installer/src/builder.rs
+++ b/crates/puffin-installer/src/builder.rs
@@ -97,6 +97,7 @@ async fn build_sdist<T: BuildContext + Send + Sync>(
     Ok(WheelDownload::Disk(DiskWheel {
         dist: dist.dist,
         path: wheel_filename,
+        temp_dir: None,
     }))
 }
 
diff --git a/crates/puffin-installer/src/downloader.rs b/crates/puffin-installer/src/downloader.rs
index df5bdf20c..be0fcb963 100644
--- a/crates/puffin-installer/src/downloader.rs
+++ b/crates/puffin-installer/src/downloader.rs
@@ -4,6 +4,7 @@ use std::sync::Arc;
 
 use anyhow::{bail, Result};
 use bytesize::ByteSize;
+use tempfile::TempDir;
 use tokio::task::JoinSet;
 use tokio_util::compat::FuturesAsyncReadCompatExt;
 use tracing::debug;
@@ -146,15 +147,16 @@ async fn fetch(
                 debug!("Fetching disk-based wheel from registry: {dist} ({size})");
 
                 // Download the wheel to a temporary file.
-                let temp_dir = tempfile::tempdir_in(cache)?.into_path();
+                let temp_dir = tempfile::tempdir_in(cache)?;
                 let wheel_filename = &wheel.file.filename;
-                let wheel_file = temp_dir.join(wheel_filename);
+                let wheel_file = temp_dir.path().join(wheel_filename);
                 let mut writer = tokio::fs::File::create(&wheel_file).await?;
                 tokio::io::copy(&mut reader.compat(), &mut writer).await?;
 
                 Ok(Download::Wheel(WheelDownload::Disk(DiskWheel {
                     dist,
                     path: wheel_file,
+                    temp_dir: Some(temp_dir),
                 })))
             }
         }
@@ -166,15 +168,16 @@ async fn fetch(
             let reader = client.stream_external(&wheel.url).await?;
 
             // Download the wheel to a temporary file.
-            let temp_dir = tempfile::tempdir_in(cache)?.into_path();
+            let temp_dir = tempfile::tempdir_in(cache)?;
             let wheel_filename = wheel.filename()?;
-            let wheel_file = temp_dir.join(wheel_filename);
+            let wheel_file = temp_dir.path().join(wheel_filename);
             let mut writer = tokio::fs::File::create(&wheel_file).await?;
             tokio::io::copy(&mut reader.compat(), &mut writer).await?;
 
             Ok(Download::Wheel(WheelDownload::Disk(DiskWheel {
                 dist,
                 path: wheel_file,
+                temp_dir: Some(temp_dir),
             })))
         }
 
@@ -188,9 +191,9 @@ async fn fetch(
             let reader = client.stream_external(&url).await?;
 
             // Download the source distribution.
-            let temp_dir = tempfile::tempdir_in(cache)?.into_path();
+            let temp_dir = tempfile::tempdir_in(cache)?;
             let sdist_filename = sdist.filename()?;
-            let sdist_file = temp_dir.join(sdist_filename);
+            let sdist_file = temp_dir.path().join(sdist_filename);
             let mut writer = tokio::fs::File::create(&sdist_file).await?;
             tokio::io::copy(&mut reader.compat(), &mut writer).await?;
 
@@ -198,6 +201,7 @@ async fn fetch(
                 dist,
                 sdist_file,
                 subdirectory: None,
+                temp_dir: Some(temp_dir),
             }))
         }
 
@@ -210,9 +214,9 @@ async fn fetch(
             let mut reader = tokio::io::BufReader::new(reader.compat());
 
             // Download the source distribution.
-            let temp_dir = tempfile::tempdir_in(cache)?.into_path();
+            let temp_dir = tempfile::tempdir_in(cache)?;
             let sdist_filename = sdist.filename()?;
-            let sdist_file = temp_dir.join(sdist_filename);
+            let sdist_file = temp_dir.path().join(sdist_filename);
             let mut writer = tokio::fs::File::create(&sdist_file).await?;
             tokio::io::copy(&mut reader, &mut writer).await?;
 
@@ -220,6 +224,7 @@ async fn fetch(
                 dist,
                 sdist_file,
                 subdirectory,
+                temp_dir: Some(temp_dir),
             }))
         }
 
@@ -238,6 +243,7 @@ async fn fetch(
                 dist,
                 sdist_file,
                 subdirectory,
+                temp_dir: None,
             }))
         }
     }
@@ -267,6 +273,9 @@ pub struct DiskWheel {
     pub(crate) dist: Dist,
     /// The path to the downloaded wheel.
     pub(crate) path: PathBuf,
+    /// The download location, to be dropped after use.
+    #[allow(dead_code)] // We only want the drop implementation
+    pub(crate) temp_dir: Option<TempDir>,
 }
 
 /// A downloaded wheel.
@@ -287,7 +296,7 @@ impl WheelDownload {
 }
 
 /// A downloaded source distribution.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct SourceDistDownload {
     /// The remote distribution from which this source distribution was downloaded.
     pub(crate) dist: Dist,
@@ -295,6 +304,10 @@ pub struct SourceDistDownload {
     pub(crate) sdist_file: PathBuf,
     /// The subdirectory within the archive or directory.
     pub(crate) subdirectory: Option<PathBuf>,
+    /// We can't use source dist archives, we build them into wheels which we persist and then drop
+    /// the source distribution. This field is non for git dependencies, which we keep in the cache.
+    #[allow(dead_code)] // We only keep it for the drop impl
+    pub(crate) temp_dir: Option<TempDir>,
 }
 
 /// A downloaded distribution, either a wheel or a source distribution.
diff --git a/crates/puffin-resolver/src/distribution/source_dist.rs b/crates/puffin-resolver/src/distribution/source_dist.rs
index d5a8c9f7e..b82017718 100644
--- a/crates/puffin-resolver/src/distribution/source_dist.rs
+++ b/crates/puffin-resolver/src/distribution/source_dist.rs
@@ -78,7 +78,7 @@ impl<'a, T: BuildContext> SourceDistFetcher<'a, T> {
             bail!("Building source distributions is disabled");
         }
 
-        let (sdist_file, subdirectory) = match dist {
+        let (temp_dir, sdist_file, subdirectory) = match dist {
             SourceDist::Registry(sdist) => {
                 debug!(
                     "Fetching source distribution from registry: {}",
@@ -89,13 +89,13 @@ impl<'a, T: BuildContext> SourceDistFetcher<'a, T> {
                 let reader = client.stream_external(&url).await?;
 
                 // Download the source distribution.
-                let temp_dir = tempfile::tempdir_in(self.build_context.cache())?.into_path();
+                let temp_dir = tempfile::tempdir_in(self.build_context.cache())?;
                 let sdist_filename = sdist.filename()?;
-                let sdist_file = temp_dir.join(sdist_filename);
+                let sdist_file = temp_dir.path().join(sdist_filename);
                 let mut writer = tokio::fs::File::create(&sdist_file).await?;
                 tokio::io::copy(&mut reader.compat(), &mut writer).await?;
 
-                (sdist_file, None)
+                (Some(temp_dir), sdist_file, None)
             }
 
             SourceDist::DirectUrl(sdist) => {
@@ -107,13 +107,13 @@ impl<'a, T: BuildContext> SourceDistFetcher<'a, T> {
                 let mut reader = tokio::io::BufReader::new(reader.compat());
 
                 // Download the source distribution.
-                let temp_dir = tempfile::tempdir_in(self.build_context.cache())?.into_path();
+                let temp_dir = tempfile::tempdir_in(self.build_context.cache())?;
                 let sdist_filename = sdist.filename()?;
-                let sdist_file = temp_dir.join(sdist_filename);
+                let sdist_file = temp_dir.path().join(sdist_filename);
                 let mut writer = tokio::fs::File::create(&sdist_file).await?;
                 tokio::io::copy(&mut reader, &mut writer).await?;
 
-                (sdist_file, subdirectory)
+                (Some(temp_dir), sdist_file, subdirectory)
             }
 
             SourceDist::Git(sdist) => {
@@ -131,7 +131,7 @@ impl<'a, T: BuildContext> SourceDistFetcher<'a, T> {
                     .await??
                     .into();
 
-                (sdist_file, subdirectory)
+                (None, sdist_file, subdirectory)
             }
         };
 
@@ -154,6 +154,10 @@ impl<'a, T: BuildContext> SourceDistFetcher<'a, T> {
             )
             .await?;
 
+        if let Some(temp_dir) = temp_dir {
+            temp_dir.close()?;
+        }
+
         // Read the metadata from the wheel.
         let wheel = CachedWheel::new(
             wheel_dir.join(&disk_filename),