diff --git a/crates/cargo-test-support/src/compare.rs b/crates/cargo-test-support/src/compare.rs index e822bb3822c..1d1cfa9bd17 100644 --- a/crates/cargo-test-support/src/compare.rs +++ b/crates/cargo-test-support/src/compare.rs @@ -300,6 +300,7 @@ static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[ ("[DOCTEST]", " Doc-tests"), ("[PACKAGING]", " Packaging"), ("[PACKAGED]", " Packaged"), + ("[PATCHING]", " Patching"), ("[DOWNLOADING]", " Downloading"), ("[DOWNLOADED]", " Downloaded"), ("[UPLOADING]", " Uploading"), diff --git a/crates/cargo-util-schemas/src/core/mod.rs b/crates/cargo-util-schemas/src/core/mod.rs index e8a878aa77c..d8e209111d7 100644 --- a/crates/cargo-util-schemas/src/core/mod.rs +++ b/crates/cargo-util-schemas/src/core/mod.rs @@ -7,4 +7,6 @@ pub use package_id_spec::PackageIdSpecError; pub use partial_version::PartialVersion; pub use partial_version::PartialVersionError; pub use source_kind::GitReference; +pub use source_kind::PatchInfo; +pub use source_kind::PatchInfoError; pub use source_kind::SourceKind; diff --git a/crates/cargo-util-schemas/src/core/package_id_spec.rs b/crates/cargo-util-schemas/src/core/package_id_spec.rs index 72d72149e2a..fc397fc8ca0 100644 --- a/crates/cargo-util-schemas/src/core/package_id_spec.rs +++ b/crates/cargo-util-schemas/src/core/package_id_spec.rs @@ -7,6 +7,7 @@ use url::Url; use crate::core::GitReference; use crate::core::PartialVersion; use crate::core::PartialVersionError; +use crate::core::PatchInfo; use crate::core::SourceKind; use crate::manifest::PackageName; use crate::restricted_names::NameValidationError; @@ -145,6 +146,14 @@ impl PackageIdSpec { kind = Some(SourceKind::Path); url = strip_url_protocol(&url); } + "patched" => { + let patch_info = + PatchInfo::from_query(url.query_pairs()).map_err(ErrorKind::PatchInfo)?; + url.set_query(None); + kind = Some(SourceKind::Patched(patch_info)); + // We don't strip protocol and leave `patch` as part of URL + // in order to distinguish them. + } kind => return Err(ErrorKind::UnsupportedProtocol(kind.into()).into()), } } else { @@ -232,10 +241,16 @@ impl fmt::Display for PackageIdSpec { write!(f, "{protocol}+")?; } write!(f, "{}", url)?; - if let Some(SourceKind::Git(git_ref)) = self.kind.as_ref() { - if let Some(pretty) = git_ref.pretty_ref(true) { - write!(f, "?{}", pretty)?; + match self.kind.as_ref() { + Some(SourceKind::Git(git_ref)) => { + if let Some(pretty) = git_ref.pretty_ref(true) { + write!(f, "?{pretty}")?; + } } + Some(SourceKind::Patched(patch_info)) => { + write!(f, "?{}", patch_info.as_query())?; + } + _ => {} } if url.path_segments().unwrap().next_back().unwrap() != &*self.name { printed_name = true; @@ -314,13 +329,16 @@ enum ErrorKind { #[error(transparent)] PartialVersion(#[from] crate::core::PartialVersionError), + + #[error(transparent)] + PatchInfo(#[from] crate::core::PatchInfoError), } #[cfg(test)] mod tests { use super::ErrorKind; use super::PackageIdSpec; - use crate::core::{GitReference, SourceKind}; + use crate::core::{GitReference, PatchInfo, SourceKind}; use url::Url; #[test] @@ -599,6 +617,18 @@ mod tests { }, "path+file:///path/to/my/project/foo#1.1.8", ); + + // Unstable + ok( + "patched+https://crates.io/foo?name=bar&version=1.2.0&patch=%2Fto%2Fa.patch&patch=%2Fb.patch#bar@1.2.0", + PackageIdSpec { + name: String::from("bar"), + version: Some("1.2.0".parse().unwrap()), + url: Some(Url::parse("patched+https://crates.io/foo").unwrap()), + kind: Some(SourceKind::Patched(PatchInfo::new("bar".into(), "1.2.0".into(), vec!["/to/a.patch".into(), "/b.patch".into()]))), + }, + "patched+https://crates.io/foo?name=bar&version=1.2.0&patch=%2Fto%2Fa.patch&patch=%2Fb.patch#bar@1.2.0", + ); } #[test] @@ -651,5 +681,17 @@ mod tests { err!("@1.2.3", ErrorKind::NameValidation(_)); err!("registry+https://github.com", ErrorKind::NameValidation(_)); err!("https://crates.io/1foo#1.2.3", ErrorKind::NameValidation(_)); + err!( + "patched+https://crates.io/foo?version=1.2.0&patch=%2Fb.patch#bar@1.2.0", + ErrorKind::PatchInfo(_) + ); + err!( + "patched+https://crates.io/foo?name=bar&patch=%2Fb.patch#bar@1.2.0", + ErrorKind::PatchInfo(_) + ); + err!( + "patched+https://crates.io/foo?name=bar&version=1.2.0&#bar@1.2.0", + ErrorKind::PatchInfo(_) + ); } } diff --git a/crates/cargo-util-schemas/src/core/source_kind.rs b/crates/cargo-util-schemas/src/core/source_kind.rs index 7b2ecaeec8c..e129d72ccb9 100644 --- a/crates/cargo-util-schemas/src/core/source_kind.rs +++ b/crates/cargo-util-schemas/src/core/source_kind.rs @@ -1,4 +1,5 @@ use std::cmp::Ordering; +use std::path::PathBuf; /// The possible kinds of code source. #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -15,6 +16,8 @@ pub enum SourceKind { LocalRegistry, /// A directory-based registry. Directory, + /// A source with paths to patch files (unstable). + Patched(PatchInfo), } impl SourceKind { @@ -27,6 +30,8 @@ impl SourceKind { SourceKind::SparseRegistry => None, SourceKind::LocalRegistry => Some("local-registry"), SourceKind::Directory => Some("directory"), + // Patched source URL already includes the `patched+` prefix, see `SourceId::new` + SourceKind::Patched(_) => None, } } } @@ -107,6 +112,10 @@ impl Ord for SourceKind { (SourceKind::Directory, _) => Ordering::Less, (_, SourceKind::Directory) => Ordering::Greater, + (SourceKind::Patched(a), SourceKind::Patched(b)) => a.cmp(b), + (SourceKind::Patched(_), _) => Ordering::Less, + (_, SourceKind::Patched(_)) => Ordering::Greater, + (SourceKind::Git(a), SourceKind::Git(b)) => a.cmp(b), } } @@ -199,3 +208,101 @@ impl<'a> std::fmt::Display for PrettyRef<'a> { Ok(()) } } + +/// Information to find the source package and patch files. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct PatchInfo { + /// Name of the package to be patched. + name: String, + /// Verision of the package to be patched. + version: String, + /// Absolute paths to patch files. + /// + /// These are absolute to ensure Cargo can locate them in the patching phase. + patches: Vec, +} + +impl PatchInfo { + pub fn new(name: String, version: String, patches: Vec) -> PatchInfo { + PatchInfo { + name, + version, + patches, + } + } + + /// Collects patch information from query string. + /// + /// * `name` --- Package name + /// * `version` --- Package exact version + /// * `patch` --- Paths to patch files. Mutiple occurrences allowed. + pub fn from_query( + query_pairs: impl Iterator, impl AsRef)>, + ) -> Result { + let mut name = None; + let mut version = None; + let mut patches = Vec::new(); + for (k, v) in query_pairs { + let v = v.as_ref(); + match k.as_ref() { + "name" => name = Some(v.to_owned()), + "version" => version = Some(v.to_owned()), + "patch" => patches.push(PathBuf::from(v)), + _ => {} + } + } + let name = name.ok_or_else(|| PatchInfoError("name"))?; + let version = version.ok_or_else(|| PatchInfoError("version"))?; + if patches.is_empty() { + return Err(PatchInfoError("path")); + } + Ok(PatchInfo::new(name, version, patches)) + } + + /// As a URL query string. + pub fn as_query(&self) -> PatchInfoQuery<'_> { + PatchInfoQuery(self) + } + + pub fn name(&self) -> &str { + self.name.as_str() + } + + pub fn version(&self) -> &str { + self.version.as_str() + } + + pub fn patches(&self) -> &[PathBuf] { + self.patches.as_slice() + } +} + +/// A [`PatchInfo`] that can be `Display`ed as URL query string. +pub struct PatchInfoQuery<'a>(&'a PatchInfo); + +impl<'a> std::fmt::Display for PatchInfoQuery<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "name=")?; + for value in url::form_urlencoded::byte_serialize(self.0.name.as_bytes()) { + write!(f, "{value}")?; + } + write!(f, "&version=")?; + for value in url::form_urlencoded::byte_serialize(self.0.version.as_bytes()) { + write!(f, "{value}")?; + } + for path in &self.0.patches { + write!(f, "&patch=")?; + let path = path.to_str().expect("utf8 patch").replace("\\", "/"); + for value in url::form_urlencoded::byte_serialize(path.as_bytes()) { + write!(f, "{value}")?; + } + } + + Ok(()) + } +} + +/// Error parsing patch info from URL query string. +#[derive(Debug, thiserror::Error)] +#[error("missing query string `{0}`")] +pub struct PatchInfoError(&'static str); diff --git a/crates/cargo-util-schemas/src/manifest/mod.rs b/crates/cargo-util-schemas/src/manifest/mod.rs index fe954f0f4ca..e9bcc003126 100644 --- a/crates/cargo-util-schemas/src/manifest/mod.rs +++ b/crates/cargo-util-schemas/src/manifest/mod.rs @@ -777,6 +777,13 @@ pub struct TomlDetailedDependency { #[serde(rename = "default_features")] pub default_features2: Option, pub package: Option, + /// `patches = [, ...]` for specifying patch files (unstable). + /// + /// Paths of patches are relative to the file it appears in. + /// If that's a `Cargo.toml`, they'll be relative to that TOML file, + /// and if it's a `.cargo/config.toml` file, they'll be relative to the + /// parent directory of that file. + pub patches: Option>, pub public: Option, /// One or more of `bin`, `cdylib`, `staticlib`, `bin:`. @@ -815,6 +822,7 @@ impl Default for TomlDetailedDependency

{ default_features: Default::default(), default_features2: Default::default(), package: Default::default(), + patches: Default::default(), public: Default::default(), artifact: Default::default(), lib: Default::default(), diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index f613bdf9094..9176ed542b3 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -513,6 +513,9 @@ features! { /// Allow multiple packages to participate in the same API namespace (unstable, open_namespaces, "", "reference/unstable.html#open-namespaces"), + + /// Allow patching dependencies with patch files. + (unstable, patch_files, "", "reference/unstable.html#patch-files"), } /// Status and metadata for a single unstable feature. @@ -775,6 +778,7 @@ unstable_cli_options!( next_lockfile_bump: bool, no_index_update: bool = ("Do not update the registry index even if the cache is outdated"), panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"), + patch_files: bool = ("Allow patching dependencies with patch files"), profile_rustflags: bool = ("Enable the `rustflags` option in profiles in .cargo/config.toml file"), public_dependency: bool = ("Respect a dependency's `public` field in Cargo.toml to control public/private dependencies"), publish_timeout: bool = ("Enable the `publish.timeout` key in .cargo/config.toml file"), @@ -1277,6 +1281,7 @@ impl CliUnstable { "mtime-on-use" => self.mtime_on_use = parse_empty(k, v)?, "no-index-update" => self.no_index_update = parse_empty(k, v)?, "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?, + "patch-files" => self.patch_files = parse_empty(k, v)?, "public-dependency" => self.public_dependency = parse_empty(k, v)?, "profile-rustflags" => self.profile_rustflags = parse_empty(k, v)?, "trim-paths" => self.trim_paths = parse_empty(k, v)?, diff --git a/src/cargo/core/source_id.rs b/src/cargo/core/source_id.rs index d03a0a5769c..edf18875e64 100644 --- a/src/cargo/core/source_id.rs +++ b/src/cargo/core/source_id.rs @@ -1,6 +1,7 @@ use crate::core::GitReference; use crate::core::PackageId; use crate::core::SourceKind; +use crate::sources::patched::PatchedSource; use crate::sources::registry::CRATES_IO_HTTP_INDEX; use crate::sources::source::Source; use crate::sources::{DirectorySource, CRATES_IO_DOMAIN, CRATES_IO_INDEX, CRATES_IO_REGISTRY}; @@ -8,6 +9,7 @@ use crate::sources::{GitSource, PathSource, RegistrySource}; use crate::util::interning::InternedString; use crate::util::{context, CanonicalUrl, CargoResult, GlobalContext, IntoUrl}; use anyhow::Context as _; +use cargo_util_schemas::core::PatchInfo; use serde::de; use serde::ser; use std::cmp::{self, Ordering}; @@ -176,6 +178,14 @@ impl SourceId { let url = url.into_url()?; SourceId::new(SourceKind::Path, url, None) } + "patched" => { + let mut url = url.into_url()?; + let patch_info = PatchInfo::from_query(url.query_pairs()) + .with_context(|| format!("parse `{url}`"))?; + url.set_fragment(None); + url.set_query(None); + SourceId::for_patches(SourceId::from_url(url.as_str())?, patch_info) + } kind => Err(anyhow::format_err!("unsupported source protocol: {}", kind)), } } @@ -245,6 +255,16 @@ impl SourceId { SourceId::new(SourceKind::Directory, url, None) } + pub fn for_patches(orig_source_id: SourceId, patch_info: PatchInfo) -> CargoResult { + let url = orig_source_id.as_encoded_url(); + // `Url::set_scheme` disallow conversions between non-special and speicial schemes, + // so parse the url from string again. + let url = format!("patched+{url}") + .parse() + .with_context(|| format!("cannot set patched scheme on `{url}`"))?; + SourceId::new(SourceKind::Patched(patch_info), url, None) + } + /// Returns the `SourceId` corresponding to the main repository. /// /// This is the main cargo registry by default, but it can be overridden in @@ -419,6 +439,7 @@ impl SourceId { .expect("path sources cannot be remote"); Ok(Box::new(DirectorySource::new(&path, self, gctx))) } + SourceKind::Patched(_) => Ok(Box::new(PatchedSource::new(self, gctx)?)), } } @@ -665,6 +686,13 @@ impl fmt::Display for SourceId { } SourceKind::LocalRegistry => write!(f, "registry `{}`", url_display(&self.inner.url)), SourceKind::Directory => write!(f, "dir {}", url_display(&self.inner.url)), + SourceKind::Patched(ref patch_info) => { + let n = patch_info.patches().len(); + let plural = if n == 1 { "" } else { "s" }; + let name = patch_info.name(); + let version = patch_info.version(); + write!(f, "{name}@{version} with {n} patch file{plural}") + } } } } @@ -730,6 +758,14 @@ impl<'a> fmt::Display for SourceIdAsUrl<'a> { write!(f, "#{}", precise)?; } } + + if let SourceIdInner { + kind: SourceKind::Patched(patch_info), + .. + } = &self.inner + { + write!(f, "?{}", patch_info.as_query())?; + } Ok(()) } } @@ -806,6 +842,8 @@ mod tests { use std::hash::Hasher; use std::path::Path; + use cargo_util_schemas::core::PatchInfo; + let gen_hash = |source_id: SourceId| { let mut hasher = std::collections::hash_map::DefaultHasher::new(); source_id.stable_hash(Path::new("/tmp/ws"), &mut hasher); @@ -850,6 +888,12 @@ mod tests { let source_id = SourceId::for_directory(path).unwrap(); assert_eq!(gen_hash(source_id), 17459999773908528552); assert_eq!(crate::util::hex::short_hash(&source_id), "6568fe2c2fab5bfe"); + + let patch_info = PatchInfo::new("foo".into(), "1.0.0".into(), vec![path.into()]); + let registry_source_id = SourceId::for_registry(&url).unwrap(); + let source_id = SourceId::for_patches(registry_source_id, patch_info).unwrap(); + assert_eq!(gen_hash(source_id), 10476212805277277232); + assert_eq!(crate::util::hex::short_hash(&source_id), "45f3b913ab447282"); } #[test] diff --git a/src/cargo/core/workspace.rs b/src/cargo/core/workspace.rs index 4ac8777bd62..16baf6df16a 100644 --- a/src/cargo/core/workspace.rs +++ b/src/cargo/core/workspace.rs @@ -484,7 +484,7 @@ impl<'gctx> Workspace<'gctx> { })?, }; patch.insert( - url, + url.clone(), deps.iter() .map(|(name, dep)| { crate::util::toml::to_dependency( @@ -498,6 +498,7 @@ impl<'gctx> Workspace<'gctx> { // any relative paths are resolved before they'd be joined with root. Path::new("unused-relative-path"), /* kind */ None, + &url, ) }) .collect::>>()?, diff --git a/src/cargo/sources/mod.rs b/src/cargo/sources/mod.rs index 9c98cc49eaa..7a181f83aec 100644 --- a/src/cargo/sources/mod.rs +++ b/src/cargo/sources/mod.rs @@ -40,6 +40,7 @@ pub mod config; pub mod directory; pub mod git; pub mod overlay; +pub mod patched; pub mod path; pub mod registry; pub mod replaced; diff --git a/src/cargo/sources/patched.rs b/src/cargo/sources/patched.rs new file mode 100644 index 00000000000..e470b5ad061 --- /dev/null +++ b/src/cargo/sources/patched.rs @@ -0,0 +1,349 @@ +//! A source that takes other source and patches it with local patch files. +//! See [`PatchedSource`] for details. + +use std::path::Path; +use std::path::PathBuf; +use std::task::Poll; + +use anyhow::Context as _; +use cargo_util::paths; +use cargo_util::ProcessBuilder; +use cargo_util::Sha256; +use cargo_util_schemas::core::PatchInfo; +use cargo_util_schemas::core::SourceKind; +use lazycell::LazyCell; + +use crate::core::Dependency; +use crate::core::Package; +use crate::core::PackageId; +use crate::core::SourceId; +use crate::core::Verbosity; +use crate::sources::source::MaybePackage; +use crate::sources::source::QueryKind; +use crate::sources::source::Source; +use crate::sources::IndexSummary; +use crate::sources::PathSource; +use crate::sources::SourceConfigMap; +use crate::util::cache_lock::CacheLockMode; +use crate::util::hex; +use crate::util::OptVersionReq; +use crate::CargoResult; +use crate::GlobalContext; + +/// A file indicates that if present, the patched source is ready to use. +const READY_LOCK: &str = ".cargo-ok"; + +/// `PatchedSource` is a source that, when fetching, it patches a paticular +/// package with given local patch files. +/// +/// This could only be created from [the `[patch]` section][patch] with any +/// entry carrying `{ .., patches = ["..."] }` field. Other kinds of dependency +/// sections (normal, dev, build) shouldn't allow to create any `PatchedSource`. +/// +/// [patch]: https://doc.rust-lang.org/nightly/cargo/reference/overriding-dependencies.html#the-patch-section +/// +/// ## Filesystem layout +/// +/// When Cargo fetches a package from a `PatchedSource`, it'll copy everything +/// from the original source to a dedicated patched source directory. That +/// directory is located under `$CARGO_HOME`. The patched source of each package +/// would be put under: +/// +/// ```text +/// $CARGO_HOME/patched-src//-//`. +/// ``` +/// +/// The file tree of the patched source directory roughly looks like: +/// +/// ```text +/// $CARGO_HOME/patched-src/github.com-6d038ece37e82ae2 +/// ├── gimli-0.29.0/ +/// │ ├── a0d193bd15a5ed96/ # checksum of all patch files from a patch to gimli@0.29.0 +/// │ ├── c58e1db3de7c154d/ +/// └── serde-1.0.197/ +/// └── deadbeef12345678/ +/// ``` +/// +/// ## `SourceId` for tracking the original package +/// +/// Due to the nature that a patched source is actually locked to a specific +/// version of one package, the SourceId URL of a `PatchedSource` needs to +/// carry such information. It looks like: +/// +/// ```text +/// patched+registry+https://github.com/rust-lang/crates.io-index?name=foo&version=1.0.0&patch=0001-bugfix.patch +/// ``` +/// +/// where the `patched+` protocol is essential for Cargo to distinguish between +/// a patched source and the source it patches. The query string contains the +/// name and version of the package being patched. We want patches to be as +/// reproducible as it could, so lock to one specific version here. +/// See [`PatchInfo::from_query`] to learn what are being tracked. +/// +/// To achieve it, the version specified in any of the entry in `[patch]` must +/// be an exact version via the `=` SemVer comparsion operator. For example, +/// this will fetch source of serde@1.2.3 from crates.io, and apply patches to it. +/// +/// ```toml +/// [patch.crates-io] +/// serde = { version = "=1.2.3", patches = ["patches/0001-serde-bug.patch"] } +/// ``` +/// +/// ## Patch tools +/// +/// When patching a package, Cargo will change the working directory to +/// the root directory of the copied source code, and then execute the tool +/// specified via the `patchtool.path` config value in the Cargo configuration. +/// Paths of patch files will be provided as absolute paths to the tool. +pub struct PatchedSource<'gctx> { + source_id: SourceId, + /// The source of the package we're going to patch. + original_source: Box, + /// Checksum from all patch files. + patches_checksum: LazyCell, + /// For respecting `[source]` replacement configuration. + map: SourceConfigMap<'gctx>, + path_source: Option>, + quiet: bool, + gctx: &'gctx GlobalContext, +} + +impl<'gctx> PatchedSource<'gctx> { + pub fn new( + source_id: SourceId, + gctx: &'gctx GlobalContext, + ) -> CargoResult> { + let original_id = { + let mut url = source_id.url().clone(); + url.set_query(None); + url.set_fragment(None); + let url = url.as_str(); + let Some(url) = url.strip_prefix("patched+") else { + anyhow::bail!("patched source url requires a `patched` scheme, got `{url}`"); + }; + SourceId::from_url(&url)? + }; + let map = SourceConfigMap::new(gctx)?; + let source = PatchedSource { + source_id, + original_source: map.load(original_id, &Default::default())?, + patches_checksum: LazyCell::new(), + map, + path_source: None, + quiet: false, + gctx, + }; + Ok(source) + } + + /// Downloads the package source if needed. + fn download_pkg(&mut self) -> CargoResult { + let patch_info = self.patch_info(); + let exact_req = &format!("={}", patch_info.version()); + let original_id = self.original_source.source_id(); + let dep = Dependency::parse(patch_info.name(), Some(exact_req), original_id)?; + let pkg_id = loop { + match self.original_source.query_vec(&dep, QueryKind::Exact) { + Poll::Ready(deps) => break deps?.remove(0).as_summary().package_id(), + Poll::Pending => self.original_source.block_until_ready()?, + } + }; + + let source = self.map.load(original_id, &Default::default())?; + Box::new(source).download_now(pkg_id, self.gctx) + } + + fn copy_pkg_src(&self, pkg: &Package, dst: &Path) -> CargoResult<()> { + let src = pkg.root(); + for entry in walkdir::WalkDir::new(src) { + let entry = entry?; + let path = entry.path().strip_prefix(src).unwrap(); + let src = entry.path(); + let dst = dst.join(path); + if entry.file_type().is_dir() { + paths::create_dir_all(dst)?; + } else { + // TODO: handle symlink? + paths::copy(src, dst)?; + } + } + Ok(()) + } + + fn apply_patches(&self, pkg: &Package, dst: &Path) -> CargoResult<()> { + let patches = self.patch_info().patches(); + let n = patches.len(); + assert!(n > 0, "must have at least one patch, got {n}"); + + self.gctx.shell().status("Patching", pkg)?; + + let patchtool_config = self.gctx.patchtool_config()?; + let Some(tool) = patchtool_config.path.as_ref() else { + anyhow::bail!("missing `[patchtool]` for patching dependencies"); + }; + + let program = tool.path.resolve_program(self.gctx); + let mut cmd = ProcessBuilder::new(program); + cmd.cwd(dst).args(&tool.args); + + for patch_path in patches { + let patch_path = self.gctx.cwd().join(patch_path); + let mut cmd = cmd.clone(); + cmd.arg(patch_path); + if matches!(self.gctx.shell().verbosity(), Verbosity::Verbose) { + self.gctx.shell().status("Running", &cmd)?; + cmd.exec()?; + } else { + cmd.exec_with_output()?; + } + } + + Ok(()) + } + + /// Gets the destination directory we put the patched source at. + fn dest_src_dir(&self, pkg: &Package) -> CargoResult { + let patched_src_root = self.gctx.patched_source_path(); + let patched_src_root = self + .gctx + .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &patched_src_root); + let pkg_id = pkg.package_id(); + let source_id = pkg_id.source_id(); + let ident = source_id.url().host_str().unwrap_or_default(); + let hash = hex::short_hash(&source_id); + let name = pkg_id.name(); + let version = pkg_id.version(); + let mut dst = patched_src_root.join(format!("{ident}-{hash}")); + dst.push(format!("{name}-{version}")); + dst.push(self.patches_checksum()?); + Ok(dst) + } + + fn patches_checksum(&self) -> CargoResult<&String> { + self.patches_checksum.try_borrow_with(|| { + let mut cksum = Sha256::new(); + for patch in self.patch_info().patches() { + cksum.update_path(patch)?; + } + let mut cksum = cksum.finish_hex(); + // TODO: is it safe to truncate sha256? + cksum.truncate(16); + Ok(cksum) + }) + } + + fn patch_info(&self) -> &PatchInfo { + let SourceKind::Patched(info) = self.source_id.kind() else { + panic!("patched source must be SourceKind::Patched"); + }; + info + } +} + +impl<'gctx> Source for PatchedSource<'gctx> { + fn source_id(&self) -> SourceId { + self.source_id + } + + fn supports_checksums(&self) -> bool { + false + } + + fn requires_precise(&self) -> bool { + false + } + + fn query( + &mut self, + dep: &Dependency, + kind: QueryKind, + f: &mut dyn FnMut(IndexSummary), + ) -> Poll> { + // Version requirement here is still the `=` exact one for fetching + // the source to patch, so switch it to a wildchard requirement. + // It is safe because this source contains one and the only package. + let mut dep = dep.clone(); + dep.set_version_req(OptVersionReq::Any); + if let Some(src) = self.path_source.as_mut() { + src.query(&dep, kind, f) + } else { + Poll::Pending + } + } + + fn invalidate_cache(&mut self) { + // No cache for a patched source + } + + fn set_quiet(&mut self, quiet: bool) { + self.quiet = quiet; + } + + fn download(&mut self, id: PackageId) -> CargoResult { + self.path_source + .as_mut() + .expect("path source must exist") + .download(id) + } + + fn finish_download(&mut self, _pkg_id: PackageId, _contents: Vec) -> CargoResult { + panic!("no download should have started") + } + + fn fingerprint(&self, pkg: &Package) -> CargoResult { + let fingerprint = self.original_source.fingerprint(pkg)?; + let cksum = self.patches_checksum()?; + Ok(format!("{fingerprint}/{cksum}")) + } + + fn describe(&self) -> String { + use std::fmt::Write as _; + let mut desc = self.original_source.describe(); + let n = self.patch_info().patches().len(); + let plural = if n == 1 { "" } else { "s" }; + write!(desc, " with {n} patch file{plural}").unwrap(); + desc + } + + fn add_to_yanked_whitelist(&mut self, _pkgs: &[PackageId]) { + // There is no yanked package for a patched source + } + + fn is_yanked(&mut self, _pkg: PackageId) -> Poll> { + // There is no yanked package for a patched source + Poll::Ready(Ok(false)) + } + + fn block_until_ready(&mut self) -> CargoResult<()> { + if self.path_source.is_some() { + return Ok(()); + } + + let pkg = self.download_pkg().context("failed to download source")?; + let dst = self.dest_src_dir(&pkg)?; + + let ready_lock = dst.join(READY_LOCK); + let cksum = self.patches_checksum()?; + match paths::read(&ready_lock) { + Ok(prev_cksum) if &prev_cksum == cksum => { + // We've applied patches. Assume they never change. + } + _ => { + // Either we were interrupted, or never get started. + // We just start over here. + if let Err(e) = paths::remove_dir_all(&dst) { + tracing::trace!("failed to remove `{}`: {e}", dst.display()); + } + self.copy_pkg_src(&pkg, &dst) + .context("failed to copy source")?; + self.apply_patches(&pkg, &dst) + .context("failed to apply patches")?; + paths::write(&ready_lock, cksum)?; + } + } + + self.path_source = Some(PathSource::new(&dst, self.source_id, self.gctx)); + + Ok(()) + } +} diff --git a/src/cargo/util/canonical_url.rs b/src/cargo/util/canonical_url.rs index 7516e035691..4981ab2eadd 100644 --- a/src/cargo/util/canonical_url.rs +++ b/src/cargo/util/canonical_url.rs @@ -39,7 +39,12 @@ impl CanonicalUrl { // almost certainly not using the same case conversion rules that GitHub // does. (See issue #84) if url.host_str() == Some("github.com") { - url = format!("https{}", &url[url::Position::AfterScheme..]) + let proto = if url.scheme().starts_with("patched+") { + "patched+https" + } else { + "https" + }; + url = format!("{proto}{}", &url[url::Position::AfterScheme..]) .parse() .unwrap(); let path = url.path().to_lowercase(); diff --git a/src/cargo/util/context/mod.rs b/src/cargo/util/context/mod.rs index c4fa1a5947d..ec096f24e75 100644 --- a/src/cargo/util/context/mod.rs +++ b/src/cargo/util/context/mod.rs @@ -228,6 +228,7 @@ pub struct GlobalContext { doc_extern_map: LazyCell, progress_config: ProgressConfig, env_config: LazyCell, + patchtool_config: LazyCell, /// This should be false if: /// - this is an artifact of the rustc distribution process for "stable" or for "beta" /// - this is an `#[test]` that does not opt in with `enable_nightly_features` @@ -322,6 +323,7 @@ impl GlobalContext { doc_extern_map: LazyCell::new(), progress_config: ProgressConfig::default(), env_config: LazyCell::new(), + patchtool_config: LazyCell::new(), nightly_features_allowed: matches!(&*features::channel(), "nightly" | "dev"), ws_roots: RefCell::new(HashMap::new()), global_cache_tracker: LazyCell::new(), @@ -400,6 +402,11 @@ impl GlobalContext { self.registry_base_path().join("src") } + /// Gets the directory containg patched package sources (`/patched-src`). + pub fn patched_source_path(&self) -> Filesystem { + self.home_path.join("patched-src") + } + /// Gets the default Cargo registry. pub fn default_registry(&self) -> CargoResult> { Ok(self @@ -1860,6 +1867,11 @@ impl GlobalContext { Ok(env_config) } + pub fn patchtool_config(&self) -> CargoResult<&PatchtoolConfig> { + self.patchtool_config + .try_borrow_with(|| self.get::("patchtool")) + } + /// This is used to validate the `term` table has valid syntax. /// /// This is necessary because loading the term settings happens very @@ -2778,6 +2790,12 @@ where deserializer.deserialize_option(ProgressVisitor) } +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct PatchtoolConfig { + pub path: Option, +} + #[derive(Debug)] enum EnvConfigValueInner { Simple(String), diff --git a/src/cargo/util/toml/mod.rs b/src/cargo/util/toml/mod.rs index 2fa704a8b87..cfa58b1e0c6 100644 --- a/src/cargo/util/toml/mod.rs +++ b/src/cargo/util/toml/mod.rs @@ -1,4 +1,5 @@ use annotate_snippets::{Level, Snippet}; +use cargo_util_schemas::core::PatchInfo; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -31,6 +32,7 @@ use crate::sources::{CRATES_IO_INDEX, CRATES_IO_REGISTRY}; use crate::util::errors::{CargoResult, ManifestError}; use crate::util::interning::InternedString; use crate::util::lints::{get_span, rel_cwd_manifest_path}; +use crate::util::CanonicalUrl; use crate::util::{self, context::ConfigRelativePath, GlobalContext, IntoUrl, OptVersionReq}; mod embedded; @@ -1344,7 +1346,7 @@ fn to_real_manifest( )?; } let replace = replace(&resolved_toml, &mut manifest_ctx)?; - let patch = patch(&resolved_toml, &mut manifest_ctx)?; + let patch = patch(&resolved_toml, &mut manifest_ctx, &features)?; { let mut names_sources = BTreeMap::new(); @@ -1700,7 +1702,7 @@ fn to_virtual_manifest( }; ( replace(&original_toml, &mut manifest_ctx)?, - patch(&original_toml, &mut manifest_ctx)?, + patch(&original_toml, &mut manifest_ctx, &features)?, ) }; if let Some(profiles) = &original_toml.profile { @@ -1779,7 +1781,7 @@ fn gather_dependencies( for (n, v) in dependencies.iter() { let resolved = v.resolved().expect("previously resolved"); - let dep = dep_to_dependency(&resolved, n, manifest_ctx, kind)?; + let dep = dep_to_dependency(&resolved, n, manifest_ctx, kind, None)?; manifest_ctx.deps.push(dep); } Ok(()) @@ -1813,7 +1815,7 @@ fn replace( ); } - let mut dep = dep_to_dependency(replacement, spec.name(), manifest_ctx, None)?; + let mut dep = dep_to_dependency(replacement, spec.name(), manifest_ctx, None, None)?; let version = spec.version().ok_or_else(|| { anyhow!( "replacements must specify a version \ @@ -1836,7 +1838,9 @@ fn replace( fn patch( me: &manifest::TomlManifest, manifest_ctx: &mut ManifestContext<'_, '_>, + features: &Features, ) -> CargoResult>> { + let patch_files_enabled = features.require(Feature::patch_files()).is_ok(); let mut patch = HashMap::new(); for (toml_url, deps) in me.patch.iter().flatten() { let url = match &toml_url[..] { @@ -1853,7 +1857,7 @@ fn patch( })?, }; patch.insert( - url, + url.clone(), deps.iter() .map(|(name, dep)| { unused_dep_keys( @@ -1862,7 +1866,13 @@ fn patch( dep.unused_keys(), &mut manifest_ctx.warnings, ); - dep_to_dependency(dep, name, manifest_ctx, None) + dep_to_dependency( + dep, + name, + manifest_ctx, + None, + Some((&url, patch_files_enabled)), + ) }) .collect::>>()?, ); @@ -1870,6 +1880,7 @@ fn patch( Ok(patch) } +/// Transforms a `patch` entry to a [`Dependency`]. pub(crate) fn to_dependency( dep: &manifest::TomlDependency

, name: &str, @@ -1879,20 +1890,18 @@ pub(crate) fn to_dependency( platform: Option, root: &Path, kind: Option, + patch_source_url: &Url, ) -> CargoResult { - dep_to_dependency( - dep, - name, - &mut ManifestContext { - deps: &mut Vec::new(), - source_id, - gctx, - warnings, - platform, - root, - }, - kind, - ) + let manifest_ctx = &mut ManifestContext { + deps: &mut Vec::new(), + source_id, + gctx, + warnings, + platform, + root, + }; + let patch_source_url = Some((patch_source_url, gctx.cli_unstable().patch_files)); + dep_to_dependency(dep, name, manifest_ctx, kind, patch_source_url) } fn dep_to_dependency( @@ -1900,6 +1909,7 @@ fn dep_to_dependency( name: &str, manifest_ctx: &mut ManifestContext<'_, '_>, kind: Option, + patch_source_url: Option<(&Url, bool)>, ) -> CargoResult { match *orig { manifest::TomlDependency::Simple(ref version) => detailed_dep_to_dependency( @@ -1910,9 +1920,10 @@ fn dep_to_dependency( name, manifest_ctx, kind, + patch_source_url, ), manifest::TomlDependency::Detailed(ref details) => { - detailed_dep_to_dependency(details, name, manifest_ctx, kind) + detailed_dep_to_dependency(details, name, manifest_ctx, kind, patch_source_url) } } } @@ -1922,6 +1933,7 @@ fn detailed_dep_to_dependency( name_in_toml: &str, manifest_ctx: &mut ManifestContext<'_, '_>, kind: Option, + patch_source_url: Option<(&Url, bool)>, ) -> CargoResult { if orig.version.is_none() && orig.path.is_none() && orig.git.is_none() { anyhow::bail!( @@ -2057,6 +2069,11 @@ fn detailed_dep_to_dependency( ) } } + + if let Some(source_id) = patched_source_id(orig, manifest_ctx, &dep, patch_source_url)? { + dep.set_source_id(source_id); + } + Ok(dep) } @@ -2145,6 +2162,88 @@ fn to_dependency_source_id( } } +// Handle `patches` field for `[patch]` table, if any. +fn patched_source_id( + orig: &manifest::TomlDetailedDependency

, + manifest_ctx: &mut ManifestContext<'_, '_>, + dep: &Dependency, + patch_source_url: Option<(&Url, bool)>, +) -> CargoResult> { + let name_in_toml = dep.name_in_toml().as_str(); + let message = "see https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#patch-files about the status of this feature."; + match (patch_source_url, orig.patches.as_ref()) { + (_, None) => { + // not a SourceKind::Patched dep. + Ok(None) + } + (None, Some(_)) => { + let kind = dep.kind().kind_table(); + manifest_ctx.warnings.push(format!( + "unused manifest key: {kind}.{name_in_toml}.patches; {message}" + )); + Ok(None) + } + (Some((url, false)), Some(_)) => { + manifest_ctx.warnings.push(format!( + "ignoring `patches` on patch for `{name_in_toml}` in `{url}`; {message}" + )); + Ok(None) + } + (Some((url, true)), Some(patches)) => { + let source_id = dep.source_id(); + if !source_id.is_registry() { + bail!( + "patch for `{name_in_toml}` in `{url}` requires a registry source \ + when patching with patch files" + ); + } + if &CanonicalUrl::new(url)? != source_id.canonical_url() { + bail!( + "patch for `{name_in_toml}` in `{url}` must refer to the same source \ + when patching with patch files" + ) + } + let version = match dep.version_req().locked_version() { + Some(v) => Some(v.to_owned()), + None if dep.version_req().is_exact() => { + // Remove the `=` exact operator. + orig.version + .as_deref() + .map(|v| v[1..].trim().parse().ok()) + .flatten() + } + None => None, + }; + let Some(version) = version else { + bail!( + "patch for `{name_in_toml}` in `{url}` requires an exact version \ + when patching with patch files" + ); + }; + let patches: Vec<_> = patches + .iter() + .map(|path| { + let path = path.resolve(manifest_ctx.gctx); + let path = manifest_ctx.root.join(path); + // keep paths inside workspace relative to workspace, otherwise absolute. + path.strip_prefix(manifest_ctx.gctx.cwd()) + .map(Into::into) + .unwrap_or_else(|_| paths::normalize_path(&path)) + }) + .collect(); + if patches.is_empty() { + bail!( + "patch for `{name_in_toml}` in `{url}` requires at least one patch file \ + when patching with patch files" + ); + } + let pkg_name = dep.package_name().to_string(); + let patch_info = PatchInfo::new(pkg_name, version.to_string(), patches); + SourceId::for_patches(source_id, patch_info).map(Some) + } + } +} + pub trait ResolveToPath { fn resolve(&self, gctx: &GlobalContext) -> PathBuf; } diff --git a/tests/testsuite/cargo/z_help/stdout.term.svg b/tests/testsuite/cargo/z_help/stdout.term.svg index a4c8e579b4d..038ad1bd265 100644 --- a/tests/testsuite/cargo/z_help/stdout.term.svg +++ b/tests/testsuite/cargo/z_help/stdout.term.svg @@ -1,4 +1,4 @@ - +