Skip to content

Commit

Permalink
WIP: feat(triage): add PASSing count to per-platform analysis
Browse files Browse the repository at this point in the history
TODO: Actually do the counting properly, and test it.
  • Loading branch information
ErichDonGubler committed Dec 27, 2023
1 parent 98671b6 commit f51908b
Showing 1 changed file with 30 additions and 4 deletions.
34 changes: 30 additions & 4 deletions moz-webgpu-cts/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -805,9 +805,11 @@ fn run(cli: Cli) -> ExitCode {

#[derive(Clone, Debug, Default)]
struct PerPlatformAnalysis {
tests_fully_passing: BTreeSet<Arc<String>>,
tests_with_runner_errors: TestSet,
tests_with_disabled_or_skip: TestSet,
tests_with_crashes: TestSet,
subtests_passing_by_test: SubtestByTestSet,
subtests_with_failures_by_test: SubtestByTestSet,
subtests_with_timeouts_by_test: SubtestByTestSet,
}
Expand Down Expand Up @@ -935,13 +937,21 @@ fn run(cli: Cli) -> ExitCode {
fn analyze_test_outcome<F>(
test_name: &Arc<String>,
expectation: Expectation<TestOutcome>,
subtests: &BTreeMap<SectionHeader, Subtest>,

Check failure on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (ubuntu-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, beta)

unused variable: `subtests`

Check failure on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (ubuntu-latest, stable)

unused variable: `subtests`

Check failure on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (ubuntu-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, beta)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, stable)

unused variable: `subtests`

Check warning on line 940 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, stable)

unused variable: `subtests`
mut receiver: F,
) where
F: FnMut(&mut dyn FnMut(&mut PerPlatformAnalysis)),
{
for outcome in expectation.iter() {
match outcome {
TestOutcome::Ok => (),
TestOutcome::Ok => receiver(&mut |analysis| {
insert_in_test_set(
&mut analysis.tests_with_crashes,
test_name,
expectation,
outcome,
)
}),
// We skip this because this test _should_ contain subtests with
// `TIMEOUT` and `NOTRUN`, so we shouldn't actually miss anything.
TestOutcome::Timeout => (),
Expand Down Expand Up @@ -974,13 +984,13 @@ fn run(cli: Cli) -> ExitCode {
}

let apply_to_all_platforms = |analysis: &mut Analysis, expectation| {
analyze_test_outcome(&test_name, expectation, |f| {
analyze_test_outcome(&test_name, expectation, &subtests, |f| {
analysis.for_each_platform_mut(f)
})
};
let apply_to_specific_platforms =
|analysis: &mut Analysis, platform, expectation| {
analyze_test_outcome(&test_name, expectation, |f| {
analyze_test_outcome(&test_name, expectation, &subtests, |f| {
analysis.for_platform_mut(platform, f)
})
};
Expand Down Expand Up @@ -1047,7 +1057,15 @@ fn run(cli: Cli) -> ExitCode {
{
for outcome in expectation.iter() {
match outcome {
SubtestOutcome::Pass => (),
SubtestOutcome::Pass => receiver(&mut |analysis| {
insert_in_subtest_by_test_set(
&mut analysis.subtests_passing_by_test,
test_name,
subtest_name,
expectation,
outcome,
)
}),
SubtestOutcome::Timeout | SubtestOutcome::NotRun => {
receiver(&mut |analysis| {
insert_in_subtest_by_test_set(
Expand Down Expand Up @@ -1140,13 +1158,20 @@ fn run(cli: Cli) -> ExitCode {
OnZeroItem::Hide => false,
};
let PerPlatformAnalysis {
tests_fully_passing,
tests_with_runner_errors,
tests_with_disabled_or_skip,
tests_with_crashes,
subtests_passing_by_test,

Check failure on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (ubuntu-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (ubuntu-latest, beta)

unused variable: `subtests_passing_by_test`

Check failure on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (ubuntu-latest, stable)

unused variable: `subtests_passing_by_test`

Check failure on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (ubuntu-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (ubuntu-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (macOS-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Check (windows-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, beta)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (windows-latest, stable)

unused variable: `subtests_passing_by_test`

Check warning on line 1165 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Test Suite (macOS-latest, stable)

unused variable: `subtests_passing_by_test`
subtests_with_failures_by_test,
subtests_with_timeouts_by_test,
} = analysis;

let tests_fully_passing = lazy_format!(
"{} test(s) permanently `PASS`ing",
tests_fully_passing.len()
);

let PermaAndIntermittent {
perma: num_tests_with_perma_runner_errors,
intermittent: num_tests_with_intermittent_runner_errors,
Expand Down Expand Up @@ -1321,6 +1346,7 @@ fn run(cli: Cli) -> ExitCode {
item.map(|disp| disp as &dyn Display)
}
let sections = [
Some(Box::new(tests_fully_passing) as Box<dyn Display>),
priority_section(
"HIGH",
[
Expand Down

0 comments on commit f51908b

Please sign in to comment.