From 4f81e4b8af0afb533e7dd21429892d31f9ea1f2a Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 15 Dec 2024 16:53:22 -0500 Subject: [PATCH] Respect all platforms --- .../src/prioritized_distribution.rs | 86 +++++++++- .../uv-resolver/src/resolver/environment.rs | 47 +++++- crates/uv-resolver/src/resolver/mod.rs | 62 ++++++- crates/uv/tests/it/lock.rs | 156 +++++++++++++++++- 4 files changed, 342 insertions(+), 9 deletions(-) diff --git a/crates/uv-distribution-types/src/prioritized_distribution.rs b/crates/uv-distribution-types/src/prioritized_distribution.rs index 54503a3317280..8dbcbeeaea2d1 100644 --- a/crates/uv-distribution-types/src/prioritized_distribution.rs +++ b/crates/uv-distribution-types/src/prioritized_distribution.rs @@ -1,7 +1,8 @@ use std::fmt::{Display, Formatter}; -use uv_distribution_filename::BuildTag; +use uv_distribution_filename::{BuildTag, WheelFilename}; use uv_pep440::VersionSpecifiers; +use uv_pep508::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValueString}; use uv_platform_tags::{IncompatibleTag, TagPriority}; use uv_pypi_types::{HashDigest, Yanked}; @@ -14,7 +15,7 @@ use crate::{ pub struct PrioritizedDist(Box); /// [`PrioritizedDist`] is boxed because [`Dist`] is large. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] struct PrioritizedDistInner { /// The highest-priority source distribution. Between compatible source distributions this priority is arbitrary. source: Option<(RegistrySourceDist, SourceDistCompatibility)>, @@ -25,6 +26,20 @@ struct PrioritizedDistInner { wheels: Vec<(RegistryBuiltWheel, WheelCompatibility)>, /// The hashes for each distribution. hashes: Vec, + /// The implied markers. + markers: MarkerTree, +} + +impl Default for PrioritizedDistInner { + fn default() -> Self { + Self { + source: None, + best_wheel_index: None, + wheels: Vec::new(), + hashes: Vec::new(), + markers: MarkerTree::FALSE, + } + } } /// A distribution that can be used for both resolution and installation. @@ -70,6 +85,15 @@ impl CompatibleDist<'_> { CompatibleDist::IncompatibleWheel { sdist, .. } => sdist.file.requires_python.as_ref(), } } + + pub fn implied_markers(&self) -> MarkerTree { + match self { + CompatibleDist::InstalledDist(_) => MarkerTree::TRUE, + CompatibleDist::SourceDist { prioritized, .. } => prioritized.0.markers, + CompatibleDist::CompatibleWheel { prioritized, .. } => prioritized.0.markers, + CompatibleDist::IncompatibleWheel { prioritized, .. } => prioritized.0.markers, + } + } } #[derive(Debug, PartialEq, Eq, Clone)] @@ -261,6 +285,7 @@ impl PrioritizedDist { compatibility: WheelCompatibility, ) -> Self { Self(Box::new(PrioritizedDistInner { + markers: implied_markers(&dist.filename), best_wheel_index: Some(0), wheels: vec![(dist, compatibility)], source: None, @@ -275,6 +300,7 @@ impl PrioritizedDist { compatibility: SourceDistCompatibility, ) -> Self { Self(Box::new(PrioritizedDistInner { + markers: MarkerTree::TRUE, best_wheel_index: None, wheels: vec![], source: Some((dist, compatibility)), @@ -297,8 +323,11 @@ impl PrioritizedDist { } else { self.0.best_wheel_index = Some(self.0.wheels.len()); } - self.0.wheels.push((dist, compatibility)); self.0.hashes.extend(hashes); + if !self.0.markers.is_true() { + self.0.markers.or(implied_markers(&dist.filename)); + } + self.0.wheels.push((dist, compatibility)); } /// Insert the given source distribution into the [`PrioritizedDist`]. @@ -316,7 +345,9 @@ impl PrioritizedDist { } else { self.0.source = Some((dist, compatibility)); } - + if !self.0.markers.is_true() { + self.0.markers.or(MarkerTree::TRUE); + } self.0.hashes.extend(hashes); } @@ -603,3 +634,50 @@ impl IncompatibleWheel { } } } + +/// Given a wheel filename, determine the set of supported platforms, in terms of their markers. +pub fn implied_markers(filename: &WheelFilename) -> MarkerTree { + let mut marker = MarkerTree::FALSE; + for platform_tag in &filename.platform_tag { + match platform_tag.as_str() { + "any" => marker.or(MarkerTree::TRUE), + tag if tag.starts_with("win") => { + marker.or(MarkerTree::expression(MarkerExpression::String { + key: MarkerValueString::SysPlatform, + operator: MarkerOperator::Equal, + value: "win32".to_string(), + })); + } + tag if tag.starts_with("manylinux") => { + marker.or(MarkerTree::expression(MarkerExpression::String { + key: MarkerValueString::SysPlatform, + operator: MarkerOperator::Equal, + value: "linux".to_string(), + })); + } + tag if tag.starts_with("musllinux") => { + marker.or(MarkerTree::expression(MarkerExpression::String { + key: MarkerValueString::SysPlatform, + operator: MarkerOperator::Equal, + value: "linux".to_string(), + })); + } + tag if tag.starts_with("linux") => { + marker.or(MarkerTree::expression(MarkerExpression::String { + key: MarkerValueString::SysPlatform, + operator: MarkerOperator::Equal, + value: "linux".to_string(), + })); + } + tag if tag.starts_with("macosx") => { + marker.or(MarkerTree::expression(MarkerExpression::String { + key: MarkerValueString::SysPlatform, + operator: MarkerOperator::Equal, + value: "darwin".to_string(), + })); + } + _ => {} + } + } + marker +} diff --git a/crates/uv-resolver/src/resolver/environment.rs b/crates/uv-resolver/src/resolver/environment.rs index a29570d7f2e83..7512d53e93346 100644 --- a/crates/uv-resolver/src/resolver/environment.rs +++ b/crates/uv-resolver/src/resolver/environment.rs @@ -512,7 +512,7 @@ impl<'d> Forker<'d> { } /// Fork the resolver based on a `Requires-Python` specifier. -pub(crate) fn fork_python_requirement( +pub(crate) fn fork_version_by_python_requirement( requires_python: &VersionSpecifiers, python_requirement: &PythonRequirement, env: &ResolverEnvironment, @@ -555,6 +555,51 @@ pub(crate) fn fork_python_requirement( envs } +/// Fork the resolver based on a marker. +pub(crate) fn fork_version_by_marker( + env: &ResolverEnvironment, + marker: &MarkerTree, +) -> Vec { + let Kind::Universal { + markers: ref env_marker, + .. + } = env.kind + else { + panic!("resolver must be in universal mode for forking") + }; + + // Attempt to split based on the marker. + // + // For example, given `python_version >= '3.10'` and the split marker `sys_platform == 'linux'`, + // the result will be: + // + // `python_version >= '3.10' and sys_platform == 'linux'` + // `python_version >= '3.10' and sys_platform != 'linux'` + // + // If the marker is disjoint with the current environment, then we should return an empty list. + // If the marker complement is disjoint with the current environment, then we should also return + // an empty list. + // + // For example, given `python_version >= '3.10' and sys_platform == 'linux'` and the split marker + // `sys_platform == 'win32'`, return an empty list, since the following isn't satisfiable: + // + // python_version >= '3.10' and sys_platform == 'linux' and sys_platform == 'win32' + + let mut envs = vec![]; + if env_marker.is_disjoint(*marker) { + return vec![]; + } + envs.push(env.narrow_environment(*marker)); + + let complement = marker.negate(); + if env_marker.is_disjoint(complement) { + return vec![]; + } + envs.push(env.narrow_environment(complement)); + + envs +} + #[cfg(test)] mod tests { use std::ops::Bound; diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index f041a1d6f104a..06751b9122d9e 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -31,7 +31,7 @@ use uv_distribution_types::{ use uv_git::GitResolver; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::{release_specifiers_to_ranges, Version, VersionSpecifiers, MIN_VERSION}; -use uv_pep508::MarkerTree; +use uv_pep508::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValueString}; use uv_platform_tags::Tags; use uv_pypi_types::{ ConflictItem, ConflictItemRef, Conflicts, Requirement, ResolutionMetadata, VerbatimParsedUrl, @@ -61,7 +61,9 @@ pub(crate) use crate::resolver::availability::{ use crate::resolver::batch_prefetch::BatchPrefetcher; pub use crate::resolver::derivation::DerivationChainBuilder; pub use crate::resolver::environment::ResolverEnvironment; -use crate::resolver::environment::{fork_python_requirement, ForkingPossibility}; +use crate::resolver::environment::{ + fork_version_by_marker, fork_version_by_python_requirement, ForkingPossibility, +}; pub(crate) use crate::resolver::fork_map::{ForkMap, ForkSet}; pub(crate) use crate::resolver::urls::Urls; use crate::universal_marker::{ConflictMarker, UniversalMarker}; @@ -1127,7 +1129,11 @@ impl ResolverState ResolverState>() + .join(", ") + ); + Ok(Some(ResolverVersion::Forked(forks))) + }; + } + } + } + } + let filename = match dist.for_installation() { ResolvedDistRef::InstallableRegistrySourceDist { sdist, .. } => sdist .filename() diff --git a/crates/uv/tests/it/lock.rs b/crates/uv/tests/it/lock.rs index fec00f2b9e91c..e3dcb075a5cdc 100644 --- a/crates/uv/tests/it/lock.rs +++ b/crates/uv/tests/it/lock.rs @@ -14723,9 +14723,75 @@ fn lock_named_index_cli() -> Result<()> { ----- stdout ----- ----- stderr ----- - Resolved 3 packages in [TIME] + Resolved 4 packages in [TIME] "###); + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + resolution-markers = [ + "sys_platform != 'darwin' and sys_platform != 'win32'", + "sys_platform == 'darwin'", + "sys_platform == 'win32'", + ] + + [[package]] + name = "jinja2" + version = "3.1.2" + source = { registry = "https://download.pytorch.org/whl/cu121" } + dependencies = [ + { name = "markupsafe", version = "2.1.5", source = { registry = "https://download.pytorch.org/whl/cu121" }, marker = "sys_platform == 'darwin' or sys_platform == 'win32'" }, + { name = "markupsafe", version = "3.0.2", source = { registry = "https://download.pytorch.org/whl/cu121" }, marker = "sys_platform != 'darwin' and sys_platform != 'win32'" }, + ] + wheels = [ + { url = "https://download.pytorch.org/whl/Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" }, + ] + + [[package]] + name = "markupsafe" + version = "2.1.5" + source = { registry = "https://download.pytorch.org/whl/cu121" } + resolution-markers = [ + "sys_platform == 'darwin'", + "sys_platform == 'win32'", + ] + wheels = [ + { url = "https://download.pytorch.org/whl/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1" }, + { url = "https://download.pytorch.org/whl/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4" }, + { url = "https://download.pytorch.org/whl/MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb" }, + ] + + [[package]] + name = "markupsafe" + version = "3.0.2" + source = { registry = "https://download.pytorch.org/whl/cu121" } + resolution-markers = [ + "sys_platform != 'darwin' and sys_platform != 'win32'", + ] + wheels = [ + { url = "https://download.pytorch.org/whl/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396" }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "jinja2" }, + ] + + [package.metadata] + requires-dist = [{ name = "jinja2", specifier = "==3.1.2", index = "https://download.pytorch.org/whl/cu121" }] + "### + ); + }); + Ok(()) } @@ -20776,6 +20842,94 @@ fn lock_self_marker_incompatible() -> Result<()> { Ok(()) } +/// When resolving `PyQt5-Qt5`, we should choose the latest version for macOS, but backtrack to +/// a prior version for Windows and Linux. +#[test] +fn lock_split_on_windows() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["pyqt5-qt5"] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + let lock = context.read("uv.lock"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + resolution-markers = [ + "sys_platform != 'linux' and sys_platform != 'win32'", + "sys_platform == 'win32'", + "sys_platform == 'linux'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "pyqt5-qt5", version = "5.15.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pyqt5-qt5", version = "5.15.13", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + ] + + [package.metadata] + requires-dist = [{ name = "pyqt5-qt5" }] + + [[package]] + name = "pyqt5-qt5" + version = "5.15.2" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform == 'linux'", + ] + wheels = [ + { url = "https://files.pythonhosted.org/packages/83/d4/241a6a518d0bcf0a9fcdcbad5edfed18d43e884317eab8d5230a2b27e206/PyQt5_Qt5-5.15.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:1988f364ec8caf87a6ee5d5a3a5210d57539988bf8e84714c7d60972692e2f4a", size = 59921716 }, + { url = "https://files.pythonhosted.org/packages/1c/7e/ce7c66a541a105fa98b41d6405fe84940564695e29fc7dccf6d9e8c5f898/PyQt5_Qt5-5.15.2-py3-none-win32.whl", hash = "sha256:9cc7a768b1921f4b982ebc00a318ccb38578e44e45316c7a4a850e953e1dd327", size = 43447358 }, + { url = "https://files.pythonhosted.org/packages/37/97/5d3b222b924fa2ed4c2488925155cd0b03fd5d09ee1cfcf7c553c11c9f66/PyQt5_Qt5-5.15.2-py3-none-win_amd64.whl", hash = "sha256:750b78e4dba6bdf1607febedc08738e318ea09e9b10aea9ff0d73073f11f6962", size = 50075158 }, + ] + + [[package]] + name = "pyqt5-qt5" + version = "5.15.13" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "sys_platform != 'linux' and sys_platform != 'win32'", + ] + wheels = [ + { url = "https://files.pythonhosted.org/packages/40/dc/96d9d0ba0d13256343b53efffe8729f278e62409ab4c937bb22e70ab98ac/PyQt5_Qt5-5.15.13-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:92575a9e96a27c4ed67c56c7048ded7461a1655d5d21f0e05064664e6e9fcbdf", size = 38771962 }, + { url = "https://files.pythonhosted.org/packages/c9/8b/4441c208c8ca29b50fab6467ebfa32b6401d16c5c915a031a48dc85dfa7a/PyQt5_Qt5-5.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:141859f2ffe04cc6c5db970e2b6ad9f98897805d886a14c52614e3799daab6d6", size = 36663754 }, + ] + "### + ); + }); + + Ok(()) +} + #[test] fn lock_missing_git_prefix() -> Result<()> { let context = TestContext::new("3.12");