Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add pip-compatible --group flag to uv pip install #11686

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion crates/uv-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use uv_configuration::{
ProjectBuildBackend, TargetTriple, TrustedHost, TrustedPublishing, VersionControlSystem,
};
use uv_distribution_types::{Index, IndexUrl, Origin, PipExtraIndex, PipFindLinks, PipIndex};
use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_normalize::{ExtraName, GroupName, PackageName, PipGroupName};
use uv_pep508::Requirement;
use uv_pypi_types::VerbatimParsedUrl;
use uv_python::{PythonDownloads, PythonPreference, PythonVersion};
Expand Down Expand Up @@ -1586,6 +1586,12 @@ pub struct PipInstallArgs {
#[arg(long, overrides_with("no_deps"), hide = true)]
pub deps: bool,

/// Ignore the package and it's dependencies, only install from the specified dependency group.
///
/// May be provided multiple times.
#[arg(long, group = "sources", conflicts_with_all = ["requirements", "package", "editable"])]
pub group: Vec<PipGroupName>,

/// Require a matching hash for each requirement.
///
/// By default, uv will verify any available hashes in the requirements file, but will not
Expand Down
71 changes: 71 additions & 0 deletions crates/uv-normalize/src/group_name.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::fmt;
use std::fmt::{Display, Formatter};
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::LazyLock;

Expand Down Expand Up @@ -64,6 +65,76 @@ impl AsRef<str> for GroupName {
}
}

/// The pip variant of a `GroupName`
///
/// Either <groupname> or <path>:<groupname>.
/// If <path> is omitted it defaults to "pyproject.toml".
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct PipGroupName {
pub path: PathBuf,
pub name: GroupName,
}

impl FromStr for PipGroupName {
type Err = InvalidNameError;

fn from_str(path_and_name: &str) -> Result<Self, Self::Err> {
// The syntax is `<path>:<name>`.
//
// `:` isn't valid as part of a dependency-group name, but it can appear in a path.
// Therefore we look for the first `:` starting from the end to find the delimiter.
// If there is no `:` then there's no path and we use the default one.
if let Some((path, name)) = path_and_name.rsplit_once(':') {
// pip hard errors if the path does not end with pyproject.toml
if !path.ends_with("pyproject.toml") {
return Err(InvalidNameError(format!(
"--group path did not end with 'pyproject.toml': {path}'"
)));
}

let name = GroupName::from_str(name)?;
let path = PathBuf::from(path);
Ok(Self { path, name })
} else {
let name = GroupName::from_str(path_and_name)?;
let path = PathBuf::from("pyproject.toml");
Ok(Self { path, name })
}
}
}

impl<'de> Deserialize<'de> for PipGroupName {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(serde::de::Error::custom)
}
}

impl Serialize for PipGroupName {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let string = self.to_string();
string.serialize(serializer)
}
}

impl Display for PipGroupName {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
let path = self.path.to_string_lossy();
if path == "pyproject.toml" {
self.name.fmt(f)
} else {
write!(f, "{}:{}", path, self.name)
}
}
}

/// The name of the global `dev-dependencies` group.
///
/// Internally, we model dependency groups as a generic concept; but externally, we only expose the
Expand Down
2 changes: 1 addition & 1 deletion crates/uv-normalize/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter};

pub use dist_info_name::DistInfoName;
pub use extra_name::ExtraName;
pub use group_name::{GroupName, DEV_DEPENDENCIES};
pub use group_name::{GroupName, PipGroupName, DEV_DEPENDENCIES};
pub use package_name::PackageName;

use uv_small_str::SmallString;
Expand Down
40 changes: 28 additions & 12 deletions crates/uv-requirements/src/source_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ use uv_distribution_types::{
BuildableSource, DirectorySourceUrl, HashGeneration, HashPolicy, SourceUrl, VersionId,
};
use uv_fs::Simplified;
use uv_normalize::{ExtraName, PackageName};
use uv_normalize::{ExtraName, PackageName, PipGroupName};
use uv_pep508::RequirementOrigin;
use uv_pypi_types::Requirement;
use uv_resolver::{InMemoryIndex, MetadataResponse};
use uv_types::{BuildContext, HashStrategy};
use uv_warnings::warn_user_once;

#[derive(Debug, Clone)]
pub struct SourceTreeResolution {
/// The requirements sourced from the source trees.
Expand All @@ -37,7 +37,7 @@ pub struct SourceTreeResolver<'a, Context: BuildContext> {
/// The extras to include when resolving requirements.
extras: &'a ExtrasSpecification,
/// The groups to include when resolving requirements.
groups: &'a DevGroupsSpecification,
groups: &'a [PipGroupName],
/// The hash policy to enforce.
hasher: &'a HashStrategy,
/// The in-memory index for resolving dependencies.
Expand All @@ -50,7 +50,7 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
/// Instantiate a new [`SourceTreeResolver`] for a given set of `source_trees`.
pub fn new(
extras: &'a ExtrasSpecification,
groups: &'a DevGroupsSpecification,
groups: &'a [PipGroupName],
hasher: &'a HashStrategy,
index: &'a InMemoryIndex,
database: DistributionDatabase<'a, Context>,
Expand Down Expand Up @@ -100,9 +100,28 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {

let mut requirements = Vec::new();

// pip --group is equivalent to --only-group
// but only use the ones that match this path
let only_groups = self
.groups
.iter()
.filter(|group| group.path == path)
.map(|group| group.name.clone())
.collect();
let groups = DevGroupsSpecification::from_args(
false,
false,
false,
Vec::new(),
Vec::new(),
false,
only_groups,
false,
);

// Flatten any transitive extras and include dependencies
// (unless something like --only-group was passed)
if self.groups.prod() {
if groups.prod() {
requirements.extend(
FlatRequiresDist::from_requirements(metadata.requires_dist, &metadata.name)
.into_iter()
Expand All @@ -116,20 +135,17 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {

// Apply dependency-groups
for (group_name, group) in &metadata.dependency_groups {
if self.groups.contains(group_name) {
if groups.contains(group_name) {
requirements.extend(group.iter().cloned());
}
}
// Complain if dependency groups are named that don't appear.
// This is only a warning because *technically* we support passing in
// multiple pyproject.tomls, but at this level of abstraction we can't see them all,
// so hard erroring on "no pyproject.toml mentions this" is a bit difficult.
for name in self.groups.explicit_names() {
for name in groups.explicit_names() {
if !metadata.dependency_groups.contains_key(name) {
warn_user_once!(
return Err(anyhow::anyhow!(
"The dependency-group '{name}' is not defined in {}",
path.display()
);
));
}
}

Expand Down
12 changes: 11 additions & 1 deletion crates/uv-settings/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use uv_distribution_types::{
};
use uv_install_wheel::LinkMode;
use uv_macros::{CombineOptions, OptionsMetadata};
use uv_normalize::{ExtraName, PackageName};
use uv_normalize::{ExtraName, PackageName, PipGroupName};
use uv_pep508::Requirement;
use uv_pypi_types::{SupportedEnvironments, VerbatimParsedUrl};
use uv_python::{PythonDownloads, PythonPreference, PythonVersion};
Expand Down Expand Up @@ -1127,6 +1127,16 @@ pub struct PipOptions {
"#
)]
pub no_deps: Option<bool>,
/// Ignore package dependencies, instead only add those packages explicitly listed
/// on the command line to the resulting requirements file.
#[option(
default = "None",
value_type = "list[str]",
example = r#"
group = ["dev", "docs"]
"#
)]
pub group: Option<Vec<PipGroupName>>,
/// Allow `uv pip sync` with empty requirements, which will clear the environment of all
/// packages.
#[option(
Expand Down
8 changes: 3 additions & 5 deletions crates/uv/src/commands/pip/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ use tracing::debug;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, Constraints, DevGroupsSpecification,
ExtrasSpecification, IndexStrategy, NoBinary, NoBuild, PreviewMode, Reinstall, SourceStrategy,
TrustedHost, Upgrade,
BuildOptions, Concurrency, ConfigSettings, Constraints, ExtrasSpecification, IndexStrategy,
NoBinary, NoBuild, PreviewMode, Reinstall, SourceStrategy, TrustedHost, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::{BuildDispatch, SharedState};
Expand Down Expand Up @@ -59,7 +58,6 @@ pub(crate) async fn pip_compile(
build_constraints_from_workspace: Vec<Requirement>,
environments: SupportedEnvironments,
extras: ExtrasSpecification,
groups: DevGroupsSpecification,
output_file: Option<&Path>,
resolution_mode: ResolutionMode,
prerelease_mode: PrereleaseMode,
Expand Down Expand Up @@ -420,7 +418,7 @@ pub(crate) async fn pip_compile(
project,
BTreeSet::default(),
&extras,
&groups,
&[],
preferences,
EmptyInstalledPackages,
&hasher,
Expand Down
8 changes: 4 additions & 4 deletions crates/uv/src/commands/pip/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ use tracing::{debug, enabled, Level};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, Constraints, DevGroupsSpecification, DryRun,
ExtrasSpecification, HashCheckingMode, IndexStrategy, PreviewMode, Reinstall, SourceStrategy,
TrustedHost, Upgrade,
BuildOptions, Concurrency, ConfigSettings, Constraints, DryRun, ExtrasSpecification,
HashCheckingMode, IndexStrategy, PreviewMode, Reinstall, SourceStrategy, TrustedHost, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::{BuildDispatch, SharedState};
Expand All @@ -22,6 +21,7 @@ use uv_distribution_types::{
use uv_fs::Simplified;
use uv_install_wheel::LinkMode;
use uv_installer::{SatisfiesResult, SitePackages};
use uv_normalize::PipGroupName;
use uv_pep508::PackageName;
use uv_pypi_types::{Conflicts, Requirement};
use uv_python::{
Expand Down Expand Up @@ -53,7 +53,7 @@ pub(crate) async fn pip_install(
overrides_from_workspace: Vec<Requirement>,
build_constraints_from_workspace: Vec<Requirement>,
extras: &ExtrasSpecification,
groups: &DevGroupsSpecification,
groups: &[PipGroupName],
resolution_mode: ResolutionMode,
prerelease_mode: PrereleaseMode,
dependency_mode: DependencyMode,
Expand Down
39 changes: 26 additions & 13 deletions crates/uv/src/commands/pip/operations.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ use uv_tool::InstalledTools;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, RegistryClient};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, Constraints, DevGroupsSpecification, DryRun,
ExtrasSpecification, Overrides, Reinstall, Upgrade,
BuildOptions, Concurrency, ConfigSettings, Constraints, DryRun, ExtrasSpecification, Overrides,
Reinstall, Upgrade,
};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
Expand All @@ -28,7 +28,7 @@ use uv_distribution_types::{
use uv_fs::Simplified;
use uv_install_wheel::LinkMode;
use uv_installer::{Plan, Planner, Preparer, SitePackages};
use uv_normalize::PackageName;
use uv_normalize::{PackageName, PipGroupName};
use uv_platform_tags::Tags;
use uv_pypi_types::{Conflicts, ResolverMarkerEnvironment};
use uv_python::{PythonEnvironment, PythonInstallation};
Expand All @@ -50,13 +50,33 @@ use crate::printer::Printer;

/// Consolidate the requirements for an installation.
pub(crate) async fn read_requirements(
requirements: &[RequirementsSource],
mut requirements: &[RequirementsSource],
constraints: &[RequirementsSource],
overrides: &[RequirementsSource],
extras: &ExtrasSpecification,
groups: &DevGroupsSpecification,
groups: &[PipGroupName],
client_builder: &BaseClientBuilder<'_>,
) -> Result<RequirementsSpecification, Error> {
// pip `--group` flags specify their own sources, and basically disable everything else.
// So if we encounter them, we desugar them to `-r` inputs and proceed as normal.
let group_requirements;
if !groups.is_empty() {
debug_assert!(
requirements.is_empty(),
"-r should be exclusive with --group in `uv pip`"
);
// Deduplicate in a stable way to get deterministic behaviour
let deduped_paths = groups
.iter()
.map(|group| &group.path)
.collect::<BTreeSet<_>>();
group_requirements = deduped_paths
.into_iter()
.map(|path| RequirementsSource::PyprojectToml(path.to_owned()))
.collect::<Vec<_>>();
requirements = &group_requirements[..];
Comment on lines +68 to +77
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I started doing a cleanup pass to compute a BTreeMap<PathBuf, Vec<GroupName>> that would be used by subsequent steps but it was just more code and value threading for no real benefit over the current impl that just does a filter(group.path == path) later.

}

// If the user requests `extras` but does not provide a valid source (e.g., a `pyproject.toml`),
// return an error.
if !extras.is_empty() && !requirements.iter().any(RequirementsSource::allows_extras) {
Expand All @@ -75,13 +95,6 @@ pub(crate) async fn read_requirements(
)
.into());
}
if !groups.is_empty() && !requirements.iter().any(RequirementsSource::allows_groups) {
let flags = groups.history().as_flags_pretty().join(" ");
return Err(anyhow!(
"Requesting groups requires a `pyproject.toml`. Requested via: {flags}"
)
.into());
}

// Read all requirements from the provided sources.
Ok(RequirementsSpecification::from_sources(
Expand Down Expand Up @@ -114,7 +127,7 @@ pub(crate) async fn resolve<InstalledPackages: InstalledPackagesProvider>(
mut project: Option<PackageName>,
workspace_members: BTreeSet<PackageName>,
extras: &ExtrasSpecification,
groups: &DevGroupsSpecification,
groups: &[PipGroupName],
preferences: Vec<Preference>,
installed_packages: InstalledPackages,
hasher: &HashStrategy,
Expand Down
7 changes: 3 additions & 4 deletions crates/uv/src/commands/pip/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ use tracing::debug;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, Constraints, DevGroupsSpecification, DryRun,
ExtrasSpecification, HashCheckingMode, IndexStrategy, PreviewMode, Reinstall, SourceStrategy,
TrustedHost, Upgrade,
BuildOptions, Concurrency, ConfigSettings, Constraints, DryRun, ExtrasSpecification,
HashCheckingMode, IndexStrategy, PreviewMode, Reinstall, SourceStrategy, TrustedHost, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::{BuildDispatch, SharedState};
Expand Down Expand Up @@ -88,7 +87,7 @@ pub(crate) async fn pip_sync(
// Initialize a few defaults.
let overrides = &[];
let extras = ExtrasSpecification::default();
let groups = DevGroupsSpecification::default();
let groups = Vec::new();
let upgrade = Upgrade::default();
let resolution_mode = ResolutionMode::default();
let prerelease_mode = PrereleaseMode::default();
Expand Down
Loading
Loading