Python/uv
1
0
Fork 0
mirror of https://github.com/astral-sh/uv synced 2026-01-22 22:10:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

Adjust the process ulimit to the maximum allowed on startup (#17464)

Browse Source 
Closes #16999

See the commentary at
https://github.com/astral-sh/uv/issues/16999#issuecomment-3641417484 for
precedence in other tools.

This is non-fatal on error, but there are various scary behaviors from
increased ulimit sizes so I've gated this with preview.

---------

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Zanie Blue
2026-01-15 12:17:46 -06:00
committed by GitHub
parent f0fd568c76
commit d673867919
9 changed files with 172 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Cargo.lock generated
View File

@@ -5707,6 +5707,7 @@ dependencies = [
"uv-torch",
"uv-trampoline-builder",
"uv-types",
"uv-unix",
"uv-version",
"uv-virtualenv",
"uv-warnings",
@@ -7136,6 +7137,14 @@ dependencies = [
"uv-workspace",
]
[[package]]
name = "uv-unix"
version = "0.0.14"
dependencies = [
"nix",
"thiserror 2.0.17",
]
[[package]]
name = "uv-version"
version = "0.9.25"

3
Cargo.toml
View File

@@ -70,6 +70,7 @@ uv-tool = { version = "0.0.14", path = "crates/uv-tool" }
uv-torch = { version = "0.0.14", path = "crates/uv-torch" }
uv-trampoline-builder = { version = "0.0.14", path = "crates/uv-trampoline-builder" }
uv-types = { version = "0.0.14", path = "crates/uv-types" }
uv-unix = { version = "0.0.14", path = "crates/uv-unix" }
uv-version = { version = "0.9.25", path = "crates/uv-version" }
uv-virtualenv = { version = "0.0.14", path = "crates/uv-virtualenv" }
uv-warnings = { version = "0.0.14", path = "crates/uv-warnings" }
@@ -134,7 +135,7 @@ md-5 = { version = "0.10.6" }
memchr = { version = "2.7.4" }
miette = { version = "7.2.0", features = ["fancy-no-backtrace"] }
nanoid = { version = "0.4.0" }
nix = { version = "0.30.0", features = ["signal"] }
nix = { version = "0.30.0", features = ["resource", "signal"] }
open = { version = "5.3.2" }
owo-colors = { version = "4.1.0" }
path-slash = { version = "0.2.1" }

7
crates/uv-preview/src/lib.rs
View File

@@ -31,6 +31,7 @@ bitflags::bitflags! {
const TARGET_WORKSPACE_DISCOVERY = 1 << 19;
const METADATA_JSON = 1 << 20;
const GCS_ENDPOINT = 1 << 21;
const ADJUST_ULIMIT = 1 << 22;
}
}
@@ -62,6 +63,7 @@ impl PreviewFeatures {
Self::TARGET_WORKSPACE_DISCOVERY => "target-workspace-discovery",
Self::METADATA_JSON => "metadata-json",
Self::GCS_ENDPOINT => "gcs-endpoint",
Self::ADJUST_ULIMIT => "adjust-ulimit",
_ => panic!("`flag_as_str` can only be used for exactly one feature flag"),
}
}
@@ -121,6 +123,7 @@ impl FromStr for PreviewFeatures {
"direct-publish" => Self::DIRECT_PUBLISH,
"target-workspace-discovery" => Self::TARGET_WORKSPACE_DISCOVERY,
"metadata-json" => Self::METADATA_JSON,
"adjust-ulimit" => Self::ADJUST_ULIMIT,
_ => {
warn_user_once!("Unknown preview feature: `{part}`");
continue;
@@ -310,6 +313,10 @@ mod tests {
PreviewFeatures::METADATA_JSON.flag_as_str(),
"metadata-json"
);
assert_eq!(
PreviewFeatures::ADJUST_ULIMIT.flag_as_str(),
"adjust-ulimit"
);
}
#[test]

20
crates/uv-unix/Cargo.toml Normal file
View File

@@ -0,0 +1,20 @@
[package]
name = "uv-unix"
version = "0.0.14"
description = "Unix-specific functionality for uv"
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
[lib]
doctest = false
[lints]
workspace = true
[target.'cfg(unix)'.dependencies]
nix = { workspace = true }
thiserror = { workspace = true }

9
crates/uv-unix/src/lib.rs Normal file
View File

@@ -0,0 +1,9 @@
//! Unix-specific functionality for uv.
//!
//! This crate is only functional on Unix platforms.
#![cfg(unix)]
mod resource_limits;
pub use resource_limits::{OpenFileLimitError, adjust_open_file_limit};

111
crates/uv-unix/src/resource_limits.rs Normal file
View File

@@ -0,0 +1,111 @@
//! Helper for adjusting Unix resource limits.
//!
//! Linux has a historically low default limit of 1024 open file descriptors per process.
//! macOS also defaults to a low soft limit (typically 256), though its hard limit is much
//! higher. On modern multi-core machines, these low defaults can cause "too many open files"
//! errors because uv infers concurrency limits from CPU count and may schedule more concurrent
//! work than the default file descriptor limit allows.
//!
//! This module attempts to raise the soft limit to the hard limit at startup to avoid these
//! errors without requiring users to manually configure their shell's `ulimit` settings.
//! The raised limit is inherited by child processes, which is important for commands like
//! `uv run` that spawn Python interpreters.
//!
//! See: <https://github.com/astral-sh/uv/issues/16999>
use nix::errno::Errno;
use nix::sys::resource::{Resource, getrlimit, rlim_t, setrlimit};
use thiserror::Error;
/// Errors that can occur when adjusting resource limits.
#[derive(Debug, Error)]
pub enum OpenFileLimitError {
#[error("failed to get open file limit: {}", .0.desc())]
GetLimitFailed(Errno),
#[error("encountered unexpected negative soft limit: {value}")]
NegativeSoftLimit { value: rlim_t },
#[error("soft limit ({current}) already meets the target ({target})")]
AlreadySufficient { current: u64, target: u64 },
#[error("failed to raise open file limit from {current} to {target}: {}", source.desc())]
SetLimitFailed {
current: u64,
target: u64,
source: Errno,
},
}
/// Maximum file descriptor limit to request.
///
/// We cap at 0x100000 (1,048,576) to match the typical Linux default (`/proc/sys/fs/nr_open`)
/// and to avoid issues with extremely high limits.
///
/// `OpenJDK` uses this same cap because:
///
/// 1. Some code breaks if `RLIMIT_NOFILE` exceeds `i32::MAX` (despite the type being `u64`)
/// 2. Code that iterates over all possible FDs, e.g., to close them, can timeout
///
/// See: <https://bugs.openjdk.org/browse/JDK-8324577>
/// See: <https://github.com/oracle/graal/issues/11136>
///
/// Note: `rlim_t` is platform-specific (`u64` on Linux/macOS, `i64` on FreeBSD).
const MAX_NOFILE_LIMIT: rlim_t = 0x0010_0000;
/// Attempt to raise the open file descriptor limit to the maximum allowed.
///
/// This function tries to set the soft limit to `min(hard_limit, 0x100000)`. If the operation
/// fails, it returns an error since the default limits may still be sufficient for the
/// current workload.
///
/// Returns [`Ok`] with the new soft limit on successful adjustment, or an appropriate
/// [`OpenFileLimitError`] if adjustment failed.
///
/// Note the type of `rlim_t` is platform-specific (`u64` on Linux/macOS, `i64` on FreeBSD), but
/// this function always returns a [`u64`].
pub fn adjust_open_file_limit() -> Result<u64, OpenFileLimitError> {
let (soft, hard) =
getrlimit(Resource::RLIMIT_NOFILE).map_err(OpenFileLimitError::GetLimitFailed)?;
// Convert `rlim_t` to `u64`. On FreeBSD, `rlim_t` is `i64` which may fail.
// On Linux and macOS, `rlim_t` is a `u64`, and the conversion is infallible.
let Some(soft) = rlim_t_to_u64(soft) else {
return Err(OpenFileLimitError::NegativeSoftLimit { value: soft });
};
// Cap the target limit to avoid issues with extremely high values.
// If hard is negative or exceeds MAX_NOFILE_LIMIT, use MAX_NOFILE_LIMIT.
#[allow(clippy::unnecessary_cast)]
let target = rlim_t_to_u64(hard.min(MAX_NOFILE_LIMIT)).unwrap_or(MAX_NOFILE_LIMIT as u64);
if soft >= target {
return Err(OpenFileLimitError::AlreadySufficient {
current: soft,
target,
});
}
// Try to raise the soft limit to the target.
// Safe because target <= MAX_NOFILE_LIMIT which fits in both i64 and u64.
let target_rlim = target as rlim_t;
setrlimit(Resource::RLIMIT_NOFILE, target_rlim, hard).map_err(|err| {
OpenFileLimitError::SetLimitFailed {
current: soft,
target,
source: err,
}
})?;
Ok(target)
}
/// Convert `rlim_t` to `u64`, returning `None` if negative.
///
/// On Linux/macOS, `rlim_t` is `u64` so this always succeeds.
/// On FreeBSD, `rlim_t` is `i64` so negative values return `None`.
#[allow(clippy::unnecessary_cast, clippy::useless_conversion)]
fn rlim_t_to_u64(value: rlim_t) -> Option<u64> {
u64::try_from(value).ok()
}

1
crates/uv/Cargo.toml
View File

@@ -149,6 +149,7 @@ zip = { workspace = true }
[target.'cfg(unix)'.dependencies]
nix = { workspace = true }
uv-unix = { workspace = true }
[features]
default = ["performance", "uv-distribution/static", "default-tests"]

11
crates/uv/src/lib.rs
View File

@@ -338,6 +338,17 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
&environment,
);
// Adjust open file limits on Unix if the preview feature is enabled.
#[cfg(unix)]
if globals.preview.is_enabled(PreviewFeatures::ADJUST_ULIMIT) {
match uv_unix::adjust_open_file_limit() {
Ok(_) | Err(uv_unix::OpenFileLimitError::AlreadySufficient { .. }) => {}
// TODO(zanieb): When moving out of preview, consider changing this to a log instead of
// a warning because it's okay if we fail here.
Err(err) => warn_user!("{err}"),
}
}
// Resolve the cache settings.
let cache_settings = CacheSettings::resolve(*cli.top_level.cache_args, filesystem.as_ref());

4
crates/uv/tests/it/show_settings.rs
View File

@@ -7982,7 +7982,7 @@ fn preview_features() {
show_settings: true,
preview: Preview {
flags: PreviewFeatures(
PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT,
PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT | ADJUST_ULIMIT,
),
},
python_preference: Managed,
@@ -8220,7 +8220,7 @@ fn preview_features() {
show_settings: true,
preview: Preview {
flags: PreviewFeatures(
PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT,
PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT | ADJUST_ULIMIT,
),
},
python_preference: Managed,