Skip to content

Commit b7c2cfb

Browse files
borsbors
committed
Auto merge of #4834 - aidanhs:aphs-better-backtrack, r=alexcrichton
Make resolution backtracking smarter There's no need to try every candidate for every dependency when backtracking - instead, only try candidates if they *could* change the eventual failure that caused backtracking in the first place, i.e. 1. if we've backtracked past the parent of the dep that failed 2. the number of activations for the dep has changed (activations are only ever added during resolution I believe) The two new tests before: ``` $ /usr/bin/time cargo test --test resolve -- --test-threads 1 --nocapture resolving_with_constrained_sibling_ Finished dev [unoptimized + debuginfo] target(s) in 0.0 secs Running target/debug/deps/resolve-19b2a13e5a19eed8 38.45user 2.16system 0:42.00elapsed 96%CPU (0avgtext+0avgdata 47672maxresident)k 0inputs+1664096outputs (0major+10921minor)pagefaults 0swaps ``` After: ``` $ /usr/bin/time cargo test --test resolve -- --test-threads 1 --nocapture resolving_with_constrained_sibling_ Finished dev [unoptimized + debuginfo] target(s) in 0.0 secs Running target/debug/deps/resolve-19b2a13e5a19eed8 [...] 0.36user 0.01system 0:00.49elapsed 76%CPU (0avgtext+0avgdata 47464maxresident)k 0inputs+32outputs (0major+11602minor)pagefaults 0swaps ``` You observe the issue yourself with the following (it should fail, but hangs for a while instead - I didn't bother timing it and waiting for it to finish. With this PR it terminates almost instantly): ``` $ cargo new --bin x Created binary (application) `x` project $ /bin/echo -e 'serde = "=1.0.9"\nrust-s3 = "0.8"' >> x/Cargo.toml $ cd x && cargo build Updating registry `https://github.com/rust-lang/crates.io-index` Resolving dependency graph... ```
2 parents a997689 + a85f9b6 commit b7c2cfb

File tree

2 files changed

+164
-2
lines changed

2 files changed

+164
-2
lines changed

‎src/cargo/core/resolver/mod.rs

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -745,22 +745,38 @@ fn activate_deps_loop<'a>(mut cx: Context<'a>,
745745
Ok(cx)
746746
}
747747

748-
// Searches up `backtrack_stack` until it finds a dependency with remaining
749-
// candidates. Resets `cx` and `remaining_deps` to that level and returns the
748+
// Looks through the states in `backtrack_stack` for dependencies with
749+
// remaining candidates. For each one, also checks if rolling back
750+
// could change the outcome of the failed resolution that caused backtracking
751+
// in the first place - namely, if we've backtracked past the parent of the
752+
// failed dep, or the previous number of activations of the failed dep has
753+
// changed (possibly relaxing version constraints). If the outcome could differ,
754+
// resets `cx` and `remaining_deps` to that level and returns the
750755
// next candidate. If all candidates have been exhausted, returns None.
756+
// Read https://github.com/rust-lang/cargo/pull/4834#issuecomment-362871537 for
757+
// a more detailed explanation of the logic here.
751758
fn find_candidate<'a>(backtrack_stack: &mut Vec<BacktrackFrame<'a>>,
752759
cx: &mut Context<'a>,
753760
remaining_deps: &mut BinaryHeap<DepsFrame>,
754761
parent: &mut Summary,
755762
cur: &mut usize,
756763
dep: &mut Dependency,
757764
features: &mut Rc<Vec<String>>) -> Option<Candidate> {
765+
let num_dep_prev_active = cx.prev_active(dep).len();
758766
while let Some(mut frame) = backtrack_stack.pop() {
759767
let (next, has_another) = {
760768
let prev_active = frame.context_backup.prev_active(&frame.dep);
761769
(frame.remaining_candidates.next(prev_active),
762770
frame.remaining_candidates.clone().next(prev_active).is_some())
763771
};
772+
let cur_num_dep_prev_active = frame.context_backup.prev_active(dep).len();
773+
// Activations should monotonically decrease during backtracking
774+
assert!(cur_num_dep_prev_active <= num_dep_prev_active);
775+
let maychange = !frame.context_backup.is_active(parent) ||
776+
cur_num_dep_prev_active != num_dep_prev_active;
777+
if !maychange {
778+
continue
779+
}
764780
if let Some(candidate) = next {
765781
*cur = frame.cur;
766782
if has_another {
@@ -1178,6 +1194,14 @@ impl<'a> Context<'a> {
11781194
.unwrap_or(&[])
11791195
}
11801196

1197+
fn is_active(&mut self, summary: &Summary) -> bool {
1198+
let id = summary.package_id();
1199+
self.activations.get(id.name())
1200+
.and_then(|v| v.get(id.source_id()))
1201+
.map(|v| v.iter().any(|s| s == summary))
1202+
.unwrap_or(false)
1203+
}
1204+
11811205
/// Return all dependencies and the features we want from them.
11821206
fn resolve_features<'b>(&mut self,
11831207
s: &'b Summary,

‎tests/resolve.rs

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,13 @@ impl ToPkgId for (&'static str, &'static str) {
7474
}
7575
}
7676

77+
impl ToPkgId for (&'static str, String) {
78+
fn to_pkgid(&self) -> PackageId {
79+
let (name, ref vers) = *self;
80+
PackageId::new(name, vers, &registry_loc()).unwrap()
81+
}
82+
}
83+
7784
macro_rules! pkg {
7885
($pkgid:expr => [$($deps:expr),+]) => ({
7986
let d: Vec<Dependency> = vec![$($deps.to_dep()),+];
@@ -364,6 +371,137 @@ fn resolving_with_deep_backtracking() {
364371
("baz", "1.0.1")])));
365372
}
366373

374+
#[test]
375+
fn resolving_with_constrained_sibling_backtrack_parent() {
376+
// There is no point in considering all of the backtrack_trap{1,2}
377+
// candidates since they can't change the result of failing to
378+
// resolve 'constrained'. Cargo should (ideally) skip past them and resume
379+
// resolution once the activation of the parent, 'bar', is rolled back.
380+
// Note that the traps are slightly more constrained to make sure they
381+
// get picked first.
382+
let mut reglist = vec![
383+
pkg!(("foo", "1.0.0") => [dep_req("bar", "1.0"),
384+
dep_req("constrained", "=1.0.0")]),
385+
386+
pkg!(("bar", "1.0.0") => [dep_req("backtrack_trap1", "1.0.2"),
387+
dep_req("backtrack_trap2", "1.0.2"),
388+
dep_req("constrained", "1.0.0")]),
389+
pkg!(("constrained", "1.0.0")),
390+
pkg!(("backtrack_trap1", "1.0.0")),
391+
pkg!(("backtrack_trap2", "1.0.0")),
392+
];
393+
// Bump this to make the test harder - it adds more versions of bar that will
394+
// fail to resolve, and more versions of the traps to consider.
395+
const NUM_BARS_AND_TRAPS: usize = 50; // minimum 2
396+
for i in 1..NUM_BARS_AND_TRAPS {
397+
let vsn = format!("1.0.{}", i);
398+
reglist.push(pkg!(("bar", vsn.clone()) => [dep_req("backtrack_trap1", "1.0.2"),
399+
dep_req("backtrack_trap2", "1.0.2"),
400+
dep_req("constrained", "1.0.1")]));
401+
reglist.push(pkg!(("backtrack_trap1", vsn.clone())));
402+
reglist.push(pkg!(("backtrack_trap2", vsn.clone())));
403+
reglist.push(pkg!(("constrained", vsn.clone())));
404+
}
405+
let reg = registry(reglist);
406+
407+
let res = resolve(&pkg_id("root"), vec![
408+
dep_req("foo", "1"),
409+
], &reg).unwrap();
410+
411+
assert_that(&res, contains(names(&[("root", "1.0.0"),
412+
("foo", "1.0.0"),
413+
("bar", "1.0.0"),
414+
("constrained", "1.0.0")])));
415+
}
416+
417+
#[test]
418+
fn resolving_with_constrained_sibling_backtrack_activation() {
419+
// It makes sense to resolve most-constrained deps first, but
420+
// with that logic the backtrack traps here come between the two
421+
// attempted resolutions of 'constrained'. When backtracking,
422+
// cargo should skip past them and resume resolution once the
423+
// number of activations for 'constrained' changes.
424+
let mut reglist = vec![
425+
pkg!(("foo", "1.0.0") => [dep_req("bar", "=1.0.0"),
426+
dep_req("backtrack_trap1", "1.0"),
427+
dep_req("backtrack_trap2", "1.0"),
428+
dep_req("constrained", "<=1.0.60")]),
429+
pkg!(("bar", "1.0.0") => [dep_req("constrained", ">=1.0.60")]),
430+
];
431+
// Bump these to make the test harder, but you'll also need to
432+
// change the version constraints on `constrained` above. To correctly
433+
// exercise Cargo, the relationship between the values is:
434+
// NUM_CONSTRAINED - vsn < NUM_TRAPS < vsn
435+
// to make sure the traps are resolved between `constrained`.
436+
const NUM_TRAPS: usize = 45; // min 1
437+
const NUM_CONSTRAINED: usize = 100; // min 1
438+
for i in 0..NUM_TRAPS {
439+
let vsn = format!("1.0.{}", i);
440+
reglist.push(pkg!(("backtrack_trap1", vsn.clone())));
441+
reglist.push(pkg!(("backtrack_trap2", vsn.clone())));
442+
}
443+
for i in 0..NUM_CONSTRAINED {
444+
let vsn = format!("1.0.{}", i);
445+
reglist.push(pkg!(("constrained", vsn.clone())));
446+
}
447+
let reg = registry(reglist);
448+
449+
let res = resolve(&pkg_id("root"), vec![
450+
dep_req("foo", "1"),
451+
], &reg).unwrap();
452+
453+
assert_that(&res, contains(names(&[("root", "1.0.0"),
454+
("foo", "1.0.0"),
455+
("bar", "1.0.0"),
456+
("constrained", "1.0.60")])));
457+
}
458+
459+
#[test]
460+
fn resolving_with_constrained_sibling_transitive_dep_effects() {
461+
// When backtracking due to a failed dependency, if Cargo is
462+
// trying to be clever and skip irrelevant dependencies, care must
463+
// be taken to not miss the transitive effects of alternatives. E.g.
464+
// in the right-to-left resolution of the graph below, B may
465+
// affect whether D is successfully resolved.
466+
//
467+
// A
468+
// / | \
469+
// B C D
470+
// | |
471+
// C D
472+
let reg = registry(vec![
473+
pkg!(("A", "1.0.0") => [dep_req("B", "1.0"),
474+
dep_req("C", "1.0"),
475+
dep_req("D", "1.0.100")]),
476+
477+
pkg!(("B", "1.0.0") => [dep_req("C", ">=1.0.0")]),
478+
pkg!(("B", "1.0.1") => [dep_req("C", ">=1.0.1")]),
479+
480+
pkg!(("C", "1.0.0") => [dep_req("D", "1.0.0")]),
481+
pkg!(("C", "1.0.1") => [dep_req("D", ">=1.0.1,<1.0.100")]),
482+
pkg!(("C", "1.0.2") => [dep_req("D", ">=1.0.2,<1.0.100")]),
483+
484+
pkg!(("D", "1.0.0")),
485+
pkg!(("D", "1.0.1")),
486+
pkg!(("D", "1.0.2")),
487+
pkg!(("D", "1.0.100")),
488+
pkg!(("D", "1.0.101")),
489+
pkg!(("D", "1.0.102")),
490+
pkg!(("D", "1.0.103")),
491+
pkg!(("D", "1.0.104")),
492+
pkg!(("D", "1.0.105")),
493+
]);
494+
495+
let res = resolve(&pkg_id("root"), vec![
496+
dep_req("A", "1"),
497+
], &reg).unwrap();
498+
499+
assert_that(&res, contains(names(&[("A", "1.0.0"),
500+
("B", "1.0.0"),
501+
("C", "1.0.0"),
502+
("D", "1.0.105")])));
503+
}
504+
367505
#[test]
368506
fn resolving_but_no_exists() {
369507
let reg = registry(vec![

0 commit comments

Comments
 (0)