From 8122d809a4df2a48c8955bdeef830ecf6dfbfe95 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 29 Feb 2024 14:23:17 -0500 Subject: [PATCH] virtualenv: determine 'site-packages' based on implementation name I'm not at all sure whether this is a correct fix or not, but it does seem to make `pypy` work in at least some cases with `uv`. Previously, I couldn't get it to work at all. Namely the virtualenv was created with a `lib/python3.10/site-packages`, but whenever I did a `uv pip install` in that virtualenv, it was looking for a non-existent `lib/pypy3.10/site-packages` directory. With this PR, the workflow reported as not working in #1488 now works for me: ``` $ pypy3 --version Python 3.10.13 (fc59e61cfbff, Jan 17 2024, 05:35:45) [PyPy 7.3.15 with GCC 13.2.1 20230801] $ uv venv --python $(which pypy3) --seed Using Python 3.10.13 interpreter at: /usr/bin/pypy3 Creating virtualenv at: .venv + pip==24.0 + setuptools==69.1.1 + wheel==0.42.0 Activate with: source .venv/bin/activate $ uv pip install 'alembic==1.0.11' Resolved 9 packages in 8ms Installed 9 packages in 14ms + alembic==1.0.11 + greenlet==3.0.3 + mako==1.3.2 + markupsafe==2.1.5 + python-dateutil==2.8.2 + python-editor==1.0.4 + six==1.16.0 + sqlalchemy==2.0.27 + typing-extensions==4.10.0 ``` Where as previously (current `main`), I was hitting this error: ``` $ uv venv --python $(which pypy3) --seed Using Python 3.10.13 interpreter at: /usr/bin/pypy3 Creating virtualenv at: .venv + pip==24.0 + setuptools==69.1.1 + wheel==0.42.0 Activate with: source .venv/bin/activate $ uv pip install 'alembic==1.0.11' error: Failed to list installed packages Caused by: failed to read directory `/home/andrew/astral/issues/uv/i1488/.venv/lib/pypy3.10/site-packages` Caused by: No such file or directory (os error 2) ``` Notice though that neither outcome above matches the error reported in #1488, so this is likely not a complete fix. There are perhaps other lurking issues. Ref #1488 --- crates/gourgeist/src/bare.rs | 6 ++- crates/uv-interpreter/src/interpreter.rs | 48 +++++++++++++++++-- .../uv-interpreter/src/virtualenv_layout.rs | 18 +++++-- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/crates/gourgeist/src/bare.rs b/crates/gourgeist/src/bare.rs index 0ebaa5852..fac848f53 100644 --- a/crates/gourgeist/src/bare.rs +++ b/crates/gourgeist/src/bare.rs @@ -191,7 +191,8 @@ pub fn create_bare_venv( .replace( "{{ RELATIVE_SITE_PACKAGES }}", &format!( - "../lib/python{}.{}/site-packages", + "../lib/{}{}.{}/site-packages", + interpreter.site_packages_python(), interpreter.python_major(), interpreter.python_minor(), ), @@ -278,7 +279,8 @@ pub fn create_bare_venv( location .join("lib") .join(format!( - "python{}.{}", + "{}{}.{}", + interpreter.site_packages_python(), interpreter.python_major(), interpreter.python_minor(), )) diff --git a/crates/uv-interpreter/src/interpreter.rs b/crates/uv-interpreter/src/interpreter.rs index 3f0d16ad6..f9ebce00c 100644 --- a/crates/uv-interpreter/src/interpreter.rs +++ b/crates/uv-interpreter/src/interpreter.rs @@ -93,9 +93,21 @@ impl Interpreter { // structure, which allows us to avoid querying the interpreter for the `sysconfig` // paths. sysconfig_paths: SysconfigPaths { - purelib: layout.site_packages(&venv_root, self.python_tuple()), - platlib: layout.site_packages(&venv_root, self.python_tuple()), - platstdlib: layout.platstdlib(&venv_root, self.python_tuple()), + purelib: layout.site_packages( + &venv_root, + self.site_packages_python(), + self.python_tuple(), + ), + platlib: layout.site_packages( + &venv_root, + self.site_packages_python(), + self.python_tuple(), + ), + platstdlib: layout.platstdlib( + &venv_root, + self.site_packages_python(), + self.python_tuple(), + ), scripts: layout.scripts(&venv_root), data: layout.data(&venv_root), ..self.sysconfig_paths @@ -399,6 +411,33 @@ impl Interpreter { &self.sysconfig_paths.stdlib } + /// Return the name of the Python directory used to build the path to the + /// `site-packages` directory. + /// + /// If one could not be determined, then `python` is returned. + pub fn site_packages_python(&self) -> &str { + if self.implementation_name() == "pypy" { + "pypy" + } else { + "python" + } + + // This implementation sniffs out what the directory name + // might be from sysconfig, but at the time of writing, this + // seems a little more risky than the simpler but less robust + // implementation above. + /* + const FALLBACK: &str = "python"; + let Some(base) = self.include().file_name().and_then(|name| name.to_str()) else { + return FALLBACK; + }; + base.char_indices() + .take_while(|(_, ch)| ch.is_ascii_alphabetic()) + .last() + .map_or(FALLBACK, |(end, ch)| &base[..end + ch.len_utf8()]) + */ + } + /// Return the [`Layout`] environment used to install wheels into this interpreter. pub fn layout(&self) -> Layout { Layout { @@ -412,7 +451,8 @@ impl Interpreter { // If the interpreter is a venv, then the `include` directory has a different structure. // See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172 self.prefix.join("include").join("site").join(format!( - "python{}.{}", + "{}{}.{}", + self.site_packages_python(), self.python_major(), self.python_minor() )) diff --git a/crates/uv-interpreter/src/virtualenv_layout.rs b/crates/uv-interpreter/src/virtualenv_layout.rs index 81673657b..a05535de7 100644 --- a/crates/uv-interpreter/src/virtualenv_layout.rs +++ b/crates/uv-interpreter/src/virtualenv_layout.rs @@ -41,13 +41,18 @@ impl<'a> VirtualenvLayout<'a> { } /// Returns the path to the `site-packages` directory inside a virtual environment. - pub(crate) fn site_packages(&self, venv_root: impl AsRef, version: (u8, u8)) -> PathBuf { + pub(crate) fn site_packages( + &self, + venv_root: impl AsRef, + site_packages_python: &str, + version: (u8, u8), + ) -> PathBuf { let venv = venv_root.as_ref(); if matches!(self.0.os(), Os::Windows) { venv.join("Lib").join("site-packages") } else { venv.join("lib") - .join(format!("python{}.{}", version.0, version.1)) + .join(format!("{site_packages_python}{}.{}", version.0, version.1)) .join("site-packages") } } @@ -60,13 +65,18 @@ impl<'a> VirtualenvLayout<'a> { /// Returns the path to the `platstdlib` directory inside a virtual environment. #[allow(clippy::unused_self)] - pub(crate) fn platstdlib(&self, venv_root: impl AsRef, version: (u8, u8)) -> PathBuf { + pub(crate) fn platstdlib( + &self, + venv_root: impl AsRef, + site_packages_python: &str, + version: (u8, u8), + ) -> PathBuf { let venv = venv_root.as_ref(); if matches!(self.0.os(), Os::Windows) { venv.join("Lib") } else { venv.join("lib") - .join(format!("python{}.{}", version.0, version.1)) + .join(format!("{site_packages_python}{}.{}", version.0, version.1)) .join("site-packages") } }