Skip to content

Commit fefad06

Browse files
rylevrylev
committed
Update comparison page to injest new API data
1 parent 7e3784d commit fefad06

File tree

2 files changed

+59
-73
lines changed

2 files changed

+59
-73
lines changed

‎site/src/comparison.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -641,7 +641,7 @@ pub struct BenchmarkComparison {
641641
results: (f64, f64),
642642
}
643643

644-
const SIGNIFICANCE_THRESHOLD: f64 = 0.01;
644+
const SIGNIFICANCE_THRESHOLD: f64 = 0.001;
645645
impl BenchmarkComparison {
646646
fn log_change(&self) -> f64 {
647647
let (a, b) = self.results;

‎site/static/compare.html

+58-72
Original file line numberDiff line numberDiff line change
@@ -525,28 +525,23 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
525525
return true;
526526
}
527527
}
528-
function toVariants(name) {
528+
function toVariants(results) {
529529
let variants = [];
530-
for (let d of data.a.data[name]) {
531-
const key = d[0];
532-
const datumA = d[1];
533-
const datumB = data.b.data[name]?.find(x => x[0] == key)?.[1];
534-
if (!datumB) {
535-
continue;
536-
}
530+
for (let r of results) {
531+
const scenarioName = r.scenario;
532+
const datumA = r.statistics[0];
533+
const datumB = r.statistics[1];
534+
const isSignificant = r.is_significant;
537535
let percent = (100 * (datumB - datumA) / datumA);
538-
let isDodgy = false;
539-
if (data.variance) {
540-
let variance = data.variance[name + "-" + key];
541-
isDodgy = (variance?.description?.type ?? "Normal") != "Normal";
542-
}
543-
if (shouldShowBuild(key)) {
536+
let isDodgy = r.is_dodgy;
537+
if (shouldShowBuild(scenarioName)) {
544538
variants.push({
545-
casename: key,
539+
casename: scenarioName,
546540
datumA,
547541
datumB,
548542
percent,
549543
isDodgy,
544+
isSignificant
550545
});
551546
}
552547
}
@@ -555,25 +550,33 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
555550
}
556551

557552
let benches =
558-
Object.keys(data.a.data).
559-
filter(n => filter.name && filter.name.trim() ? n.includes(filter.name.trim()) : true).
560-
map(name => {
561-
const variants = toVariants(name).filter(v => filter.showOnlySignificant ? isSignificant(v.percent, v.isDodgy) : true);
562-
const pcts = variants.map(field => parseFloat(field.percent));
563-
const maxPct = Math.max(...pcts).toFixed(1);
564-
const minPct = Math.min(...pcts).toFixed(1);
565-
if (variants.length > 0) {
566-
variants[0].first = true;
567-
}
568-
569-
return {
570-
name,
571-
variants,
572-
maxPct,
573-
minPct,
574-
};
575-
}).
576-
filter(b => b.variants.length > 0);
553+
data.comparisons.
554+
filter(n => filter.name && filter.name.trim() ? n.benchmark.includes(filter.name.trim()) : true).
555+
reduce((accum, next) => {
556+
accum[next.benchmark + "-" + next.profile] ||= [];
557+
accum[next.benchmark + "-" + next.profile].push(next);
558+
return accum;
559+
}, {});
560+
benches = Object.entries(benches).
561+
map(c => {
562+
const name = c[0];
563+
const comparison = c[1];
564+
const variants = toVariants(comparison).filter(v => filter.showOnlySignificant ? v.isSignificant : true);
565+
const pcts = variants.map(field => parseFloat(field.percent));
566+
const maxPct = Math.max(...pcts).toFixed(1);
567+
const minPct = Math.min(...pcts).toFixed(1);
568+
if (variants.length > 0) {
569+
variants[0].first = true;
570+
}
571+
572+
return {
573+
name,
574+
variants,
575+
maxPct,
576+
minPct,
577+
};
578+
}).
579+
filter(b => b.variants.length > 0);
577580

578581
const largestChange = a => Math.max(Math.abs(a.minPct), Math.abs(a.maxPct));
579582
benches.sort((a, b) => largestChange(b) - largestChange(a));
@@ -634,29 +637,25 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
634637
return findQueryParam("stat") || "instructions:u";
635638
},
636639
summary() {
637-
const filtered = Object.fromEntries(this.benches.map(b => [b.name, Object.fromEntries(b.variants.map(v => [v.casename, true]))]));
640+
// Create object with each test case that is not filtered out as a key
641+
const filtered = Object.fromEntries(this.benches.flatMap(b => b.variants.map(v => [b.name + "-" + v.casename, true])));
638642
const newCount = { regressions: 0, improvements: 0, unchanged: 0 }
639643
let result = { all: { ...newCount }, filtered: { ...newCount } }
640-
for (let benchmarkAndProfile of Object.keys(this.data.a.data)) {
641-
for (let d of this.data.a.data[benchmarkAndProfile]) {
642-
const scenario = d[0];
643-
const datumA = d[1];
644-
const datumB = this.data.b.data[benchmarkAndProfile]?.find(x => x[0] == scenario)?.[1];
645-
if (!datumB) {
646-
continue;
647-
}
648-
let percent = 100 * ((datumB - datumA) / datumA);
649-
const isDodgy = this.isDodgy(benchmarkAndProfile, scenario);
650-
if (percent > 0 && isSignificant(percent, isDodgy)) {
651-
result.all.regressions += 1;
652-
result.filtered.regressions += filtered[benchmarkAndProfile]?.[scenario] || 0;
653-
} else if (percent < 0 && isSignificant(percent, isDodgy)) {
654-
result.all.improvements += 1;
655-
result.filtered.improvements += filtered[benchmarkAndProfile]?.[scenario] || 0;
656-
} else {
657-
result.all.unchanged += 1;
658-
result.filtered.unchanged += filtered[benchmarkAndProfile]?.[scenario] || 0;
659-
}
644+
for (let d of this.data.comparisons) {
645+
const testCase = testCaseString(d)
646+
const scenario = d.scenario;
647+
const datumA = d.statistics[0];
648+
const datumB = d.statistics[1];
649+
let percent = 100 * ((datumB - datumA) / datumA);
650+
if (percent > 0 && d.is_significant) {
651+
result.all.regressions += 1;
652+
result.filtered.regressions += filtered[testCase] || 0;
653+
} else if (percent < 0 && d.is_significant) {
654+
result.all.improvements += 1;
655+
result.filtered.improvements += filtered[testCase] || 0;
656+
} else {
657+
result.all.unchanged += 1;
658+
result.filtered.unchanged += filtered[testCase] || 0;
660659
}
661660
}
662661
return result;
@@ -719,26 +718,9 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
719718
}
720719
return result;
721720
},
722-
isDodgy(benchmarkAndProfile, scenario) {
723-
let variance = this.data.variance;
724-
if (!variance) {
725-
return false;
726-
}
727-
variance = this.data.variance[benchmarkAndProfile + "-" + scenario];
728-
return (variance?.description?.type ?? "Normal") != "Normal";
729-
},
730721
},
731722
});
732723

733-
function isSignificant(percent, isNoisy) {
734-
const perAbs = Math.abs(percent);
735-
if (isNoisy) {
736-
return perAbs > 1.0;
737-
} else {
738-
return perAbs >= 0.2;
739-
}
740-
}
741-
742724
function toggleFilters(id, toggle) {
743725
let styles = document.getElementById(id).style;
744726
let indicator = document.getElementById(toggle);
@@ -753,6 +735,10 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
753735
toggleFilters("filters-content", "filters-toggle-indicator");
754736
toggleFilters("search-content", "search-toggle-indicator");
755737

738+
function testCaseString(testCase) {
739+
return testCase.benchmark + "-" + testCase.profile + "-" + testCase.scenario;
740+
}
741+
756742
document.getElementById("filters-toggle").onclick = (e) => {
757743
toggleFilters("filters-content", "filters-toggle-indicator");
758744
};

0 commit comments

Comments
 (0)