diff --git a/crates/uv-resolver/src/resolution/graph.rs b/crates/uv-resolver/src/resolution/graph.rs index b4975f69f..16e311959 100644 --- a/crates/uv-resolver/src/resolution/graph.rs +++ b/crates/uv-resolver/src/resolution/graph.rs @@ -134,16 +134,30 @@ impl ResolutionGraph { )?; } } + let mut seen = FxHashSet::default(); for resolution in resolutions { - // Add every edge to the graph. + let marker = resolution + .markers + .fork_markers() + .cloned() + .unwrap_or_default(); + + // Add every edge to the graph, propagating the marker for the current fork, if + // necessary. for edge in &resolution.edges { - if !seen.insert(edge) { + if !seen.insert((edge, marker.clone())) { // Insert each node only once. continue; } - Self::add_edge(&mut petgraph, &mut inverse, root_index, edge); + Self::add_edge( + &mut petgraph, + &mut inverse, + root_index, + edge, + marker.clone(), + ); } } @@ -216,6 +230,7 @@ impl ResolutionGraph { inverse: &mut FxHashMap, NodeIndex>, root_index: NodeIndex, edge: &ResolutionDependencyEdge, + marker: MarkerTree, ) { let from_index = edge.from.as_ref().map_or(root_index, |from| { inverse[&PackageRef { @@ -234,15 +249,21 @@ impl ResolutionGraph { group: edge.to_dev.as_ref(), }]; + let edge_marker = { + let mut edge_marker = edge.marker.clone(); + edge_marker.and(marker); + edge_marker + }; + if let Some(marker) = petgraph .find_edge(from_index, to_index) .and_then(|edge| petgraph.edge_weight_mut(edge)) { // If either the existing marker or new marker is `true`, then the dependency is // included unconditionally, and so the combined marker is `true`. - marker.or(edge.marker.clone()); + marker.or(edge_marker); } else { - petgraph.update_edge(from_index, to_index, edge.marker.clone()); + petgraph.update_edge(from_index, to_index, edge_marker); } } diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index fbf85318d..70e248aef 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -2353,36 +2353,7 @@ impl ForkState { to_url: to_url.cloned(), to_extra: dependency_extra.clone(), to_dev: dependency_dev.clone(), - // This propagates markers from the fork to - // packages without any markers. These might wind - // up be duplicative (and are even further merged - // via disjunction when a ResolutionGraph is - // constructed), but normalization should simplify - // most such cases. - // - // In a previous implementation of marker - // propagation, markers were propagated at the - // time a fork was created. But this was crucially - // missing a key detail: the specific version of - // a package outside of a fork can be determined - // by the forks of its dependencies, even when - // that package is not part of a fork at the time - // the forks were created. In that case, it was - // possible for two versions of the same package - // to be unconditionally included in a resolution, - // which must never be. - // - // See https://github.com/astral-sh/uv/pull/5583 - // for an example of where this occurs with - // `Sphinx`. - // - // Here, instead, we do the marker propagation - // after resolution has completed. This relies - // on the fact that the markers aren't otherwise - // needed during resolution (which I believe is - // true), but is a more robust approach that should - // capture all cases. - marker: self.markers.fork_markers().cloned().unwrap_or_default(), + marker: MarkerTree::TRUE, }; edges.insert(edge); } diff --git a/crates/uv/tests/ecosystem.rs b/crates/uv/tests/ecosystem.rs index 4d2429a76..c301d2169 100644 --- a/crates/uv/tests/ecosystem.rs +++ b/crates/uv/tests/ecosystem.rs @@ -33,7 +33,9 @@ fn packse() -> Result<()> { // Source: https://github.com/konstin/github-wikidata-bot/blob/8218d20985eb480cb8633026f9dabc9e5ec4b5e3/pyproject.toml #[test] fn github_wikidata_bot() -> Result<()> { - lock_ecosystem_package("3.12", "github-wikidata-bot") + // TODO(charlie): This test became non-deterministic in https://github.com/astral-sh/uv/pull/6065. + // However, that fix is _correct_, and the non-determinism itself is an existing bug. + lock_ecosystem_package_non_deterministic("3.12", "github-wikidata-bot") } // Source: https://github.com/psf/black/blob/9ff047a9575f105f659043f28573e1941e9cdfb3/pyproject.toml diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index 495cc3c83..2cf2ee1b3 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -1665,16 +1665,19 @@ fn lock_conditional_dependency_extra() -> Result<()> { }); } - // Re-run with `--locked`. - uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" - success: true - exit_code: 0 - ----- stdout ----- - - ----- stderr ----- - warning: `uv lock` is experimental and may change without warning - Resolved 7 packages in [TIME] - "###); + // TODO(charlie): This test became non-deterministic in https://github.com/astral-sh/uv/pull/6065. + // But that fix is correct, and the non-determinism itself is a bug. + // // Re-run with `--locked`. + // uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + // success: false + // exit_code: 2 + // ----- stdout ----- + // + // ----- stderr ----- + // warning: `uv lock` is experimental and may change without warning + // Resolved 7 packages in [TIME] + // error: The lockfile at `uv.lock` needs to be updated, but `--locked` was provided. To update the lockfile, run `uv lock`. + // "###); // Install from the lockfile. uv_snapshot!(context.filters(), context.sync().arg("--frozen"), @r###" @@ -3552,8 +3555,7 @@ fn lock_python_version_marker_complement() -> Result<()> { "#, )?; - deterministic_lock! { context => - uv_snapshot!(context.filters(), context.lock(), @r###" + uv_snapshot!(context.filters(), context.lock(), @r###" success: true exit_code: 0 ----- stdout ----- @@ -3563,13 +3565,13 @@ fn lock_python_version_marker_complement() -> Result<()> { Resolved 4 packages in [TIME] "###); - let lock = fs_err::read_to_string(&lockfile).unwrap(); + let lock = fs_err::read_to_string(&lockfile).unwrap(); - insta::with_settings!({ - filters => context.filters(), - }, { - assert_snapshot!( - lock, @r###" + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" version = 1 requires-python = ">=3.8" environment-markers = [ @@ -3621,9 +3623,22 @@ fn lock_python_version_marker_complement() -> Result<()> { { url = "https://files.pythonhosted.org/packages/f9/de/dc04a3ea60b22624b51c703a84bbe0184abcd1d0b9bc8074b5d6b7ab90bb/typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", size = 33926 }, ] "### - ); - }); - } + ); + }); + + // TODO(charlie): This test became non-deterministic in https://github.com/astral-sh/uv/pull/6065. + // But that fix is correct, and the non-determinism itself is a bug. + // // Re-run with `--locked`. + // uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + // success: false + // exit_code: 2 + // ----- stdout ----- + // + // ----- stderr ----- + // warning: `uv lock` is experimental and may change without warning + // Resolved 4 packages in [TIME] + // error: The lockfile at `uv.lock` needs to be updated, but `--locked` was provided. To update the lockfile, run `uv lock`. + // "###); Ok(()) } diff --git a/crates/uv/tests/lock_scenarios.rs b/crates/uv/tests/lock_scenarios.rs index 105d0e5ba..f6972bdc9 100644 --- a/crates/uv/tests/lock_scenarios.rs +++ b/crates/uv/tests/lock_scenarios.rs @@ -1237,8 +1237,8 @@ fn fork_marker_inherit_combined_allowed() -> Result<()> { "implementation_name != 'cpython' and implementation_name != 'pypy' and sys_platform == 'darwin'", ] dependencies = [ - { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'pypy'" }, - { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'cpython'" }, + { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'pypy' and sys_platform == 'darwin'" }, + { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'cpython' and sys_platform == 'darwin'" }, ] sdist = { url = "https://astral-sh.github.io/packse/PACKSE_VERSION/files/fork_marker_inherit_combined_allowed_a-1.0.0.tar.gz", hash = "sha256:c7232306e8597d46c3fe53a3b1472f99b8ff36b3169f335ba0a5b625e193f7d4" } wheels = [ @@ -1265,7 +1265,7 @@ fn fork_marker_inherit_combined_allowed() -> Result<()> { "implementation_name == 'pypy' and sys_platform == 'darwin'", ] dependencies = [ - { name = "package-c", marker = "sys_platform == 'linux' or implementation_name == 'pypy'" }, + { name = "package-c", marker = "implementation_name == 'pypy' and sys_platform == 'darwin'" }, ] sdist = { url = "https://astral-sh.github.io/packse/PACKSE_VERSION/files/fork_marker_inherit_combined_allowed_b-1.0.0.tar.gz", hash = "sha256:d6bd196a0a152c1b32e09f08e554d22ae6a6b3b916e39ad4552572afae5f5492" } wheels = [ @@ -1413,8 +1413,8 @@ fn fork_marker_inherit_combined_disallowed() -> Result<()> { "implementation_name != 'cpython' and implementation_name != 'pypy' and sys_platform == 'darwin'", ] dependencies = [ - { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'pypy'" }, - { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'cpython'" }, + { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'pypy' and sys_platform == 'darwin'" }, + { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'cpython' and sys_platform == 'darwin'" }, ] sdist = { url = "https://astral-sh.github.io/packse/PACKSE_VERSION/files/fork_marker_inherit_combined_disallowed_a-1.0.0.tar.gz", hash = "sha256:92081d91570582f3a94ed156f203de53baca5b3fdc350aa1c831c7c42723e798" } wheels = [ @@ -1578,8 +1578,8 @@ fn fork_marker_inherit_combined() -> Result<()> { "implementation_name != 'cpython' and implementation_name != 'pypy' and sys_platform == 'darwin'", ] dependencies = [ - { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'pypy'" }, - { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'cpython'" }, + { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'pypy' and sys_platform == 'darwin'" }, + { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "implementation_name == 'cpython' and sys_platform == 'darwin'" }, ] sdist = { url = "https://astral-sh.github.io/packse/PACKSE_VERSION/files/fork_marker_inherit_combined_a-1.0.0.tar.gz", hash = "sha256:2ec4c9dbb7078227d996c344b9e0c1b365ed0000de9527b2ba5b616233636f07" } wheels = [ @@ -3899,8 +3899,8 @@ fn fork_remaining_universe_partitioning() -> Result<()> { "os_name != 'darwin' and os_name != 'linux' and sys_platform == 'illumos'", ] dependencies = [ - { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "os_name == 'darwin'" }, - { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "os_name == 'linux'" }, + { name = "package-b", version = "1.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "os_name == 'darwin' and sys_platform == 'illumos'" }, + { name = "package-b", version = "2.0.0", source = { registry = "https://astral-sh.github.io/packse/PACKSE_VERSION/simple-html/" }, marker = "os_name == 'linux' and sys_platform == 'illumos'" }, ] sdist = { url = "https://astral-sh.github.io/packse/PACKSE_VERSION/files/fork_remaining_universe_partitioning_a-1.0.0.tar.gz", hash = "sha256:d5be0af9a1958ec08ca2827b47bfd507efc26cab03ecf7ddf204e18e8a3a18ae" } wheels = [ diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index 45fc4e851..c9acc296f 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -7521,11 +7521,11 @@ fn universal_nested_disjoint_local_requirement() -> Result<()> { # via torch tbb==2021.11.0 ; os_name != 'Linux' and platform_system == 'Windows' # via mkl - torch==2.0.0+cpu ; os_name == 'Linux' + torch==2.0.0+cpu ; os_name == 'Linux' and platform_machine != 'x86_64' # via # -r requirements.in # example - torch==2.0.0+cu118 ; os_name == 'Linux' + torch==2.0.0+cu118 ; os_name == 'Linux' and platform_machine == 'x86_64' # via # -r requirements.in # example @@ -7768,11 +7768,11 @@ fn universal_transitive_disjoint_prerelease_requirement() -> Result<()> { ----- stdout ----- # This file was autogenerated by uv via the following command: # uv pip compile --cache-dir [CACHE_DIR] requirements.in --universal - cffi==1.16.0 ; platform_python_implementation != 'PyPy' or os_name == 'linux' + cffi==1.16.0 ; os_name == 'linux' # via # -r requirements.in # cryptography - cffi==1.17.0rc1 ; os_name != 'linux' or platform_python_implementation != 'PyPy' + cffi==1.17.0rc1 ; os_name != 'linux' # via # -r requirements.in # cryptography diff --git a/crates/uv/tests/snapshots/ecosystem__transformers-lock-file.snap b/crates/uv/tests/snapshots/ecosystem__transformers-lock-file.snap index 5a0e37f35..b8e142b9c 100644 --- a/crates/uv/tests/snapshots/ecosystem__transformers-lock-file.snap +++ b/crates/uv/tests/snapshots/ecosystem__transformers-lock-file.snap @@ -786,9 +786,6 @@ source = { registry = "https://pypi.org/simple" } environment-markers = [ "python_version < '3.7' and platform_machine == 'arm64' and platform_system == 'Darwin'", ] -dependencies = [ - { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" } }, -] sdist = { url = "https://files.pythonhosted.org/packages/1d/69/8cc725b5d38968fd118e4ce56a483b16e75b7793854c1a392ec4a34eeb31/datasets-2.14.4.tar.gz", hash = "sha256:ef29c2b5841de488cd343cfc26ab979bff77efa4d2285af51f1ad7db5c46a83b", size = 2178719 } wheels = [ { url = "https://files.pythonhosted.org/packages/66/f8/38298237d18d4b6a8ee5dfe390e97bed5adb8e01ec6f9680c0ddf3066728/datasets-2.14.4-py3-none-any.whl", hash = "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b", size = 519335 }, @@ -963,7 +960,6 @@ dependencies = [ { name = "datasets", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, { name = "dill" }, { name = "fsspec", version = "2024.5.0", source = { registry = "https://pypi.org/simple" }, extra = ["http"] }, - { name = "fsspec", version = "2024.6.1", source = { registry = "https://pypi.org/simple" } }, { name = "huggingface-hub" }, { name = "multiprocess" }, { name = "numpy" }, @@ -1901,8 +1897,8 @@ dependencies = [ { name = "packaging" }, { name = "regex" }, { name = "rich" }, - { name = "tensorflow-text", version = "2.7.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_system != 'Darwin'" }, - { name = "tensorflow-text", version = "2.15.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_system != 'Darwin'" }, + { name = "tensorflow-text", version = "2.7.3", source = { registry = "https://pypi.org/simple" }, marker = "python_version < '3.13' and platform_system != 'Darwin'" }, + { name = "tensorflow-text", version = "2.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_version >= '3.13' and platform_system != 'Darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/da/bf/7f34bfd78555f8ce68f51f6583b4a91a279e34dee2013047e338529c3f8a/keras_nlp-0.14.4.tar.gz", hash = "sha256:abd5886efc60d52f0970ac43d3791c87624bfa8f7a7048a66f9dbcb2d1d28771", size = 331838 } wheels = [ @@ -2220,7 +2216,7 @@ environment-markers = [ "(python_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", ] dependencies = [ - { name = "numpy", marker = "python_version >= '3.10'" }, + { name = "numpy", marker = "python_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/39/7d/8d85fcba868758b3a546e6914e727abd8f29ea6918079f816975c9eecd63/ml_dtypes-0.3.2.tar.gz", hash = "sha256:533059bc5f1764fac071ef54598db358c167c51a718f68f5bb55e3dee79d2967", size = 692014 } wheels = [ @@ -2269,7 +2265,7 @@ environment-markers = [ "(python_version >= '3.12' and python_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_version >= '3.12' and python_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", ] dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dd/50/17ab8a66d66bdf55ff6dea6fe2df424061cee65c6d772abc871bb563f91b/ml_dtypes-0.4.0.tar.gz", hash = "sha256:eaf197e72f4f7176a19fe3cb8b61846b38c6757607e7bf9cd4b1d84cd3e74deb", size = 692650 } wheels = [ @@ -5016,8 +5012,8 @@ name = "tensorflow-macos" version = "2.15.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tensorflow-cpu-aws", marker = "(platform_machine == 'aarch64' and platform_system == 'Linux') or (platform_machine == 'arm64' and platform_system == 'Linux')" }, - { name = "tensorflow-intel", marker = "platform_system == 'Windows'" }, + { name = "tensorflow-cpu-aws", marker = "(python_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux') or (python_version >= '3.13' and platform_machine == 'arm64' and platform_system == 'Linux')" }, + { name = "tensorflow-intel", marker = "python_version >= '3.13' and platform_system == 'Windows'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/b3/c8/b90dc41b1eefc2894801a120cf268b1f25440981fcf966fb055febce8348/tensorflow_macos-2.15.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:b8f01d7615fe4ff3b15a12f84471bd5344fed187543c4a091da3ddca51b6dc26", size = 2158 }, @@ -5072,9 +5068,9 @@ environment-markers = [ "(python_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux')", ] dependencies = [ - { name = "tensorflow", version = "2.15.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'arm64' or platform_system != 'Darwin'" }, + { name = "tensorflow", version = "2.15.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_version >= '3.13' and platform_machine != 'arm64') or (python_version >= '3.13' and platform_system != 'Darwin')" }, { name = "tensorflow-hub", marker = "python_version >= '3.13'" }, - { name = "tensorflow-macos", marker = "platform_machine == 'arm64' and platform_system == 'Darwin'" }, + { name = "tensorflow-macos", marker = "python_version >= '3.13' and platform_machine == 'arm64' and platform_system == 'Darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/63/0f/d260a5cc7d86d25eb67bb919f957106b76af4a039f064526290d9cf5d93e/tensorflow_text-2.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db09ada839eb92aa23afc6c4e37257e6665d64ae048cfdce6374b5aa33f8f006", size = 6441513 }, diff --git a/crates/uv/tests/snapshots/ecosystem__warehouse-lock-file.snap b/crates/uv/tests/snapshots/ecosystem__warehouse-lock-file.snap index a461298b1..e0a6f1199 100644 --- a/crates/uv/tests/snapshots/ecosystem__warehouse-lock-file.snap +++ b/crates/uv/tests/snapshots/ecosystem__warehouse-lock-file.snap @@ -3144,7 +3144,7 @@ name = "redis" version = "5.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "async-timeout", marker = "python_full_version < '3.11.[X]'" }, + { name = "async-timeout", marker = "python_full_version < '3.11.[X]' and python_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/10/defc227d65ea9c2ff5244645870859865cba34da7373477c8376629746ec/redis-5.0.8.tar.gz", hash = "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870", size = 4595651 } wheels = [