Include all site packages directories in ephemeral environment overlays (#15121)

Related to https://github.com/astral-sh/uv/issues/15113

The case in the linked issue is that we perhaps should not be allowing
`uv run --with` with system interpreters at all. I think we can consider
that, but the issue highlighted that `uv run --with` for a system
interpreter is broken if the base interpreter has custom site packages.
This generalizes beyond system interpreters so we should probably fix
our overlays.

.github/workflows/ci.yml

@@ -1916,7 +1916,7 @@ jobs:
           persist-credentials: false
 
       - name: "Install Python"
-        run: apt-get update && apt-get install -y python3.11 python3-pip python3.11-venv
+        run: apt-get update && apt-get install -y python3.11 python3-pip python3.11-venv python3-debian
 
       - name: "Download binary"
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -1932,6 +1932,11 @@ jobs:
       - name: "Validate global Python install"
         run: python3.11 scripts/check_system_python.py --uv ./uv --externally-managed
 
+      - name: "Test `uv run` with system Python"
+        run: |
+          ./uv run -p python3.11 -v python -c "import debian"
+          ./uv run -p python3.11 -v --with anyio python -c "import debian"
+
   system-test-fedora:
     timeout-minutes: 10
     needs: build-binary-linux-libc

crates/uv-python/python/get_interpreter_info.py

@@ -4,6 +4,7 @@ Queries information about the current Python interpreter and prints it as JSON.
 The script will exit with status 0 on known error that are turned into rust errors.
 """
 
+import site
 import sys
 
 import json
@@ -637,6 +638,7 @@ def main() -> None:
         # temporary path to `sys.path` so we can import it, which we have to strip later
         # to avoid having this now-deleted path around.
         "sys_path": sys.path[1:],
+        "site_packages": site.getsitepackages(),
         "stdlib": sysconfig.get_path("stdlib"),
         # Prior to the introduction of `sysconfig` patching, python-build-standalone installations would always use
         # "/install" as the prefix. With `sysconfig` patching, we rewrite the prefix to match the actual installation

crates/uv-python/src/interpreter.rs

@@ -51,6 +51,7 @@ pub struct Interpreter {
     sys_base_executable: Option<PathBuf>,
     sys_executable: PathBuf,
     sys_path: Vec<PathBuf>,
+    site_packages: Vec<PathBuf>,
     stdlib: PathBuf,
     standalone: bool,
     tags: OnceLock<Tags>,
@@ -86,6 +87,7 @@ impl Interpreter {
             sys_base_executable: info.sys_base_executable,
             sys_executable: info.sys_executable,
             sys_path: info.sys_path,
+            site_packages: info.site_packages,
             stdlib: info.stdlib,
             standalone: info.standalone,
             tags: OnceLock::new(),
@@ -439,10 +441,21 @@ impl Interpreter {
     }
 
     /// Return the `sys.path` for this Python interpreter.
-    pub fn sys_path(&self) -> &Vec<PathBuf> {
+    pub fn sys_path(&self) -> &[PathBuf] {
         &self.sys_path
     }
 
+    /// Return the `site.getsitepackages` for this Python interpreter.
+    ///
+    /// These are the paths Python will search for packages in at runtime. We use this for
+    /// environment layering, but not for checking for installed packages. We could use these paths
+    /// to check for installed packages, but it introduces a lot of complexity, so instead we use a
+    /// simplified version that does not respect customized site-packages. See
+    /// [`Interpreter::site_packages`].
+    pub fn runtime_site_packages(&self) -> &[PathBuf] {
+        &self.site_packages
+    }
+
     /// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
     pub fn stdlib(&self) -> &Path {
         &self.stdlib
@@ -567,6 +580,9 @@ impl Interpreter {
     ///
     /// Some distributions also create symbolic links from `purelib` to `platlib`; in such cases, we
     /// still deduplicate the entries, returning a single path.
+    ///
+    /// Note this does not include all runtime site-packages directories if the interpreter has been
+    /// customized. See [`Interpreter::runtime_site_packages`].
     pub fn site_packages(&self) -> impl Iterator<Item = Cow<'_, Path>> {
         let target = self.target().map(Target::site_packages);
 
@@ -870,6 +886,7 @@ struct InterpreterInfo {
     sys_base_executable: Option<PathBuf>,
     sys_executable: PathBuf,
     sys_path: Vec<PathBuf>,
+    site_packages: Vec<PathBuf>,
     stdlib: PathBuf,
     standalone: bool,
     pointer_size: PointerSize,
@@ -1247,6 +1264,9 @@ mod tests {
                 "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/lib/python3.12",
                 "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages"
             ],
+            "site_packages": [
+                "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages"
+            ],
             "stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12",
             "scheme": {
                 "data": "/home/ferris/.pyenv/versions/3.12.0",

crates/uv-python/src/lib.rs

@@ -272,6 +272,9 @@ mod tests {
                         "/home/ferris/.pyenv/versions/{FULL_VERSION}/lib/python{VERSION}/lib/python{VERSION}",
                         "/home/ferris/.pyenv/versions/{FULL_VERSION}/lib/python{VERSION}/site-packages"
                     ],
+                    "site_packages": [
+                        "/home/ferris/.pyenv/versions/{FULL_VERSION}/lib/python{VERSION}/site-packages"
+                    ],
                     "stdlib": "/home/ferris/.pyenv/versions/{FULL_VERSION}/lib/python{VERSION}",
                     "scheme": {
                         "data": "/home/ferris/.pyenv/versions/{FULL_VERSION}",

crates/uv/src/commands/project/run.rs

@@ -1077,16 +1077,28 @@ hint: If you are running a script with `{}` in the shebang, you may need to incl
                 requirements_env.site_packages().next().ok_or_else(|| {
                     anyhow!("Requirements environment has no site packages directory")
                 })?;
-            let base_site_packages = base_interpreter
+            let mut base_site_packages = base_interpreter
-                .site_packages()
+                .runtime_site_packages()
-                .next()
+                .iter()
-                .ok_or_else(|| anyhow!("Base environment has no site packages directory"))?;
+                .map(|path| Cow::Borrowed(path.as_path()))
+                .chain(base_interpreter.site_packages())
+                .peekable();
+            if base_site_packages.peek().is_none() {
+                return Err(anyhow!("Base environment has no site packages directory"));
+            }
 
-            ephemeral_env.set_overlay(format!(
+            let overlay_content = format!(
-                "import site; site.addsitedir(\"{}\"); site.addsitedir(\"{}\");",
+                "import site; {}",
-                requirements_site_packages.escape_for_python(),
+                std::iter::once(requirements_site_packages)
-                base_site_packages.escape_for_python(),
+                    .chain(base_site_packages)
-            ))?;
+                    .dedup()
+                    .inspect(|path| debug!("Adding `{}` to site packages", path.display()))
+                    .map(|path| format!("site.addsitedir(\"{}\")", path.escape_for_python()))
+                    .collect::<Vec<_>>()
+                    .join("; ")
+            );
+
+            ephemeral_env.set_overlay(overlay_content)?;
 
             // N.B. The order here matters — earlier interpreters take precedence over the
             // later ones.

crates/uv/tests/it/run.rs

@@ -5334,8 +5334,7 @@ fn run_repeated() -> Result<()> {
     Resolved 1 package in [TIME]
     "###);
 
-    // Re-running as a tool doesn't require reinstalling `typing-extensions`, since the environment
+    // Import `iniconfig` in the context of a `tool run` command, which should fail.
-    // is cached.
     uv_snapshot!(
         context.filters(),
         context.tool_run().arg("--with").arg("typing-extensions").arg("python").arg("-c").arg("import typing_extensions; import iniconfig"), @r#"