diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fce3bceaa..9a3527f00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -305,9 +305,6 @@ jobs: - name: "Smoke test" working-directory: ${{ env.UV_WORKSPACE }} - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows run: | Set-Alias -Name uv -Value ./target/debug/uv uv venv -v @@ -316,9 +313,6 @@ jobs: - name: "Smoke test completion" working-directory: ${{ env.UV_WORKSPACE }} shell: powershell - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows run: | Set-Alias -Name uv -Value ./target/debug/uv Set-Alias -Name uvx -Value ./target/debug/uvx @@ -744,9 +738,6 @@ jobs: needs: build-binary-windows name: "integration test | free-threaded on windows" runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows steps: - name: "Download binary" @@ -908,9 +899,6 @@ jobs: & .venv\Scripts\python.exe --version - name: "Check install" - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows run: | .\uv.exe pip install anyio @@ -1040,9 +1028,6 @@ jobs: & .venv\Scripts\python.exe --version - name: "Check install" - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows run: | .\uv.exe pip install anyio @@ -1534,9 +1519,6 @@ jobs: needs: build-binary-windows name: "check system | python3.10 on windows" runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows steps: - uses: actions/checkout@v4 @@ -1560,9 +1542,6 @@ jobs: needs: build-binary-windows name: "check system | python3.10 on windows x86" runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows steps: - uses: actions/checkout@v4 @@ -1587,9 +1566,6 @@ jobs: needs: build-binary-windows name: "check system | python3.13 on windows" runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows steps: - uses: actions/checkout@v4 @@ -1615,9 +1591,6 @@ jobs: needs: build-binary-windows name: "check system | python3.12 via chocolatey" runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows steps: - uses: actions/checkout@v4 @@ -1750,9 +1723,6 @@ jobs: - name: "Validate global Python install" shell: bash -el {0} - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows run: python ./scripts/check_system_python.py --uv ./uv system-test-amazonlinux: @@ -1789,9 +1759,6 @@ jobs: needs: build-binary-windows name: "check system | embedded python3.10 on windows" runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 3000000 # 3 megabyte, triple the default on windows steps: - uses: actions/checkout@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ffc00b4b8..428298290 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,18 +82,6 @@ cargo run -- venv cargo run -- pip install requests ``` -### Testing on Windows - -When testing debug builds on Windows, the stack can overflow resulting in a `STATUS_STACK_OVERFLOW` -error code. This is due to a small stack size limit on Windows that we encounter when running -unoptimized builds — the release builds do not have this problem. We -[added a `UV_STACK_SIZE` variable](https://github.com/astral-sh/uv/pull/941) to bypass this problem -during testing. We recommend bumping the stack size from the default of 1MB to 3MB, for example: - -```powershell -$Env:UV_STACK_SIZE = '3000000' -``` - ## Running inside a Docker container Source distributions can run arbitrary code on build and can make unwanted modifications to your diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index ebc60db4e..1bfe16ae0 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -249,9 +249,6 @@ impl EnvVars { /// Use to disable line wrapping for diagnostics. pub const UV_NO_WRAP: &'static str = "UV_NO_WRAP"; - /// Use to increase the stack size used by uv in debug builds on Windows. - pub const UV_STACK_SIZE: &'static str = "UV_STACK_SIZE"; - /// Generates the environment variable key for the HTTP Basic authentication username. #[attr_env_var_pattern("UV_INDEX_{name}_USERNAME")] pub fn index_username(name: &str) -> String { @@ -505,6 +502,15 @@ impl EnvVars { /// for more. pub const RUST_LOG: &'static str = "RUST_LOG"; + /// Use to set the stack size used by uv. + /// + /// The value is in bytes, and the default is typically 2MB (2097152). + /// Unlike the normal `RUST_MIN_STACK` semantics, this can affect main thread + /// stack size, because we actually spawn our own main2 thread to work around + /// the fact that Windows' real main thread is only 1MB. That thread has size + /// `max(RUST_MIN_STACK, 4MB)`. + pub const RUST_MIN_STACK: &'static str = "RUST_MIN_STACK"; + /// The directory containing the `Cargo.toml` manifest for a package. #[attr_hidden] pub const CARGO_MANIFEST_DIR: &'static str = "CARGO_MANIFEST_DIR"; diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 8d5fbea4f..fcc6cb6b7 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1819,45 +1819,51 @@ where } }; - // Windows has a default stack size of 1MB, which is lower than the linux and mac default. + // Running out of stack has been an issue for us. We box types and futures in various places + // to mitigate this, with this being an especially important case. + // + // Non-main threads should all have 2MB, as Rust forces platform consistency there, + // but that can be overridden with the RUST_MIN_STACK environment variable if you need more. + // + // Main thread stack-size is the real issue. There's BIG variety here across platforms + // and it's harder to control (which is why Rust doesn't by default). Notably + // on macOS and Linux you will typically get 8MB main thread, while on Windows you will + // typically get 1MB, which is *tiny*: // https://learn.microsoft.com/en-us/cpp/build/reference/stack-stack-allocations?view=msvc-170 - // We support increasing the stack size to avoid stack overflows in debug mode on Windows. In - // addition, we box types and futures in various places. This includes the `Box::pin(run())` - // here, which prevents the large (non-send) main future alone from overflowing the stack. - let result = if let Ok(stack_size) = std::env::var(EnvVars::UV_STACK_SIZE) { - let stack_size = stack_size.parse().expect("Invalid stack size"); - let tokio_main = move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .thread_stack_size(stack_size) - .build() - .expect("Failed building the Runtime"); - // Box the large main future to avoid stack overflows. - let result = runtime.block_on(Box::pin(run(cli))); - // Avoid waiting for pending tasks to complete. - // - // The resolver may have kicked off HTTP requests during resolution that - // turned out to be unnecessary. Waiting for those to complete can cause - // the CLI to hang before exiting. - runtime.shutdown_background(); - result - }; - std::thread::Builder::new() - .stack_size(stack_size) - .spawn(tokio_main) - .expect("Tokio executor failed, was there a panic?") - .join() - .expect("Tokio executor failed, was there a panic?") - } else { + // + // To normalize this we just spawn a new thread called main2 with a size we can set + // ourselves. 2MB is typically too small (especially for our debug builds), while 4MB + // seems fine. Also we still try to respect RUST_MIN_STACK if it's set, in case useful, + // but don't let it ask for a smaller stack to avoid messy misconfiguration since we + // know we use quite a bit of main stack space. + let main_stack_size = std::env::var(EnvVars::RUST_MIN_STACK) + .ok() + .and_then(|var| var.parse::().ok()) + .unwrap_or(0) + .max(4 * 1024 * 1024); + + let main2 = move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .expect("Failed building the Runtime"); // Box the large main future to avoid stack overflows. let result = runtime.block_on(Box::pin(run(cli))); + // Avoid waiting for pending tasks to complete. + // + // The resolver may have kicked off HTTP requests during resolution that + // turned out to be unnecessary. Waiting for those to complete can cause + // the CLI to hang before exiting. runtime.shutdown_background(); result }; + let result = std::thread::Builder::new() + .name("main2".to_owned()) + .stack_size(main_stack_size) + .spawn(main2) + .expect("Tokio executor failed, was there a panic?") + .join() + .expect("Tokio executor failed, was there a panic?"); match result { Ok(code) => code.into(), diff --git a/crates/uv/tests/it/common/mod.rs b/crates/uv/tests/it/common/mod.rs index 16eb660c9..46bdb94ca 100644 --- a/crates/uv/tests/it/common/mod.rs +++ b/crates/uv/tests/it/common/mod.rs @@ -489,12 +489,6 @@ impl TestContext { // Avoid locale issues in tests command.env(EnvVars::LC_ALL, "C"); } - - if cfg!(all(windows, debug_assertions)) { - // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the - // default windows stack of 1MB - command.env(EnvVars::UV_STACK_SIZE, (4 * 1024 * 1024).to_string()); - } } /// Create a `pip compile` command for testing. @@ -581,13 +575,6 @@ impl TestContext { let mut command = self.new_command(); command.arg("help"); command.env_remove(EnvVars::UV_CACHE_DIR); - - if cfg!(all(windows, debug_assertions)) { - // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the - // default windows stack of 1MB - command.env(EnvVars::UV_STACK_SIZE, (4 * 1024 * 1024).to_string()); - } - command } @@ -636,13 +623,6 @@ impl TestContext { pub fn publish(&self) -> Command { let mut command = self.new_command(); command.arg("publish"); - - if cfg!(all(windows, debug_assertions)) { - // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the - // default windows stack of 1MB - command.env(EnvVars::UV_STACK_SIZE, (4 * 1024 * 1024).to_string()); - } - command } diff --git a/crates/uv/tests/it/ecosystem.rs b/crates/uv/tests/it/ecosystem.rs index 5d29e77ff..b21ed5dfd 100644 --- a/crates/uv/tests/it/ecosystem.rs +++ b/crates/uv/tests/it/ecosystem.rs @@ -104,11 +104,6 @@ fn lock_ecosystem_package(python_version: &str, name: &str) -> Result<()> { .env(EnvVars::UV_EXCLUDE_NEWER, EXCLUDE_NEWER) .current_dir(context.temp_dir.path()); - if cfg!(all(windows, debug_assertions)) { - // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the - // default windows stack of 1MB - command.env(EnvVars::UV_STACK_SIZE, (4 * 1024 * 1024).to_string()); - } let (snapshot, _) = common::run_and_format( &mut command, context.filters(), diff --git a/crates/uv/tests/it/show_settings.rs b/crates/uv/tests/it/show_settings.rs index 3cf5d5ddc..e7e400ba6 100644 --- a/crates/uv/tests/it/show_settings.rs +++ b/crates/uv/tests/it/show_settings.rs @@ -24,13 +24,6 @@ fn add_shared_args(mut command: Command, cwd: &Path) -> Command { // Avoid locale issues in tests command.env(EnvVars::LC_ALL, "C"); } - - if cfg!(all(windows, debug_assertions)) { - // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the - // default windows stack of 1MB - command.env(EnvVars::UV_STACK_SIZE, (4 * 1024 * 1024).to_string()); - } - command } diff --git a/docs/configuration/environment.md b/docs/configuration/environment.md index af640be28..4b8876b5f 100644 --- a/docs/configuration/environment.md +++ b/docs/configuration/environment.md @@ -320,10 +320,6 @@ uv will require that all dependencies have a hash specified in the requirements Equivalent to the `--resolution` command-line argument. For example, if set to `lowest-direct`, uv will install the lowest compatible versions of all direct dependencies. -### `UV_STACK_SIZE` - -Use to increase the stack size used by uv in debug builds on Windows. - ### `UV_SYSTEM_PYTHON` Equivalent to the `--system` command-line argument. If set to `true`, uv will @@ -484,6 +480,16 @@ For example: See the [tracing documentation](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#example-syntax) for more. +### `RUST_MIN_STACK` + +Use to set the stack size used by uv. + +The value is in bytes, and the default is typically 2MB (2097152). +Unlike the normal `RUST_MIN_STACK` semantics, this can affect main thread +stack size, because we actually spawn our own main2 thread to work around +the fact that Windows' real main thread is only 1MB. That thread has size +`max(RUST_MIN_STACK, 4MB)`. + ### `SHELL` The standard `SHELL` posix env var.