Add torchao dashboard to benchmarks (#5284)

Co-authored-by: Huy Do <[email protected]>
pytorch · Jun 5, 2024 · 270b951 · 270b951
1 parent 1e0f0d5
commit 270b951
Show file tree

Hide file tree

Showing 11 changed files with 703 additions and 7 deletions.
diff --git a/torchci/components/NavBar.tsx b/torchci/components/NavBar.tsx
@@ -40,6 +40,10 @@ function NavBar() {
       name: "TorchInductor",
       href: "/benchmark/compilers",
     },
+    {
+      name: "TorchAO",
+      href: "/benchmark/torchao",
+    },
     {
       name: "TorchBench",
       href: "/torchbench/userbenchmark",

diff --git a/torchci/components/benchmark/compilers/SummaryPanel.tsx b/torchci/components/benchmark/compilers/SummaryPanel.tsx
@@ -11,7 +11,6 @@ import {
   SCALE,
   SPEEDUP_THRESHOLD,
 } from "components/benchmark/compilers/common";
-import { SUITES } from "components/benchmark/compilers/SuitePicker";
 import styles from "components/metrics.module.css";
 import { TablePanelWithData } from "components/metrics/panels/TablePanel";
 import { Granularity } from "components/metrics/panels/TimeSeriesPanel";
@@ -136,6 +135,7 @@ export function SummaryPanel({
   dtype,
   lPerfData,
   rPerfData,
+  all_suites,
 }: {
   startTime: dayjs.Dayjs;
   stopTime: dayjs.Dayjs;
@@ -144,6 +144,7 @@ export function SummaryPanel({
   dtype: string;
   lPerfData: BranchAndCommitPerfData;
   rPerfData: BranchAndCommitPerfData;
+  all_suites: { [key: string]: string };
 }) {
   const fields: { [k: string]: any } = {
     passrate_display: computePassrate,
@@ -167,7 +168,8 @@ export function SummaryPanel({
     fields
   );
 
-  const suites = Object.keys(SUITES);
+  const suites = Object.keys(all_suites);
+
   // Combine both sides
   const passrate = combineLeftAndRight(
     lCommit,
@@ -229,7 +231,7 @@ export function SummaryPanel({
               suites.map((suite: string) => {
                 return {
                   field: suite,
-                  headerName: SUITES[suite],
+                  headerName: all_suites[suite],
                   flex: 1,
                   renderCell: (params: GridRenderCellParams<any>) => {
                     const v = params.value;
@@ -319,7 +321,7 @@ export function SummaryPanel({
               suites.map((suite: string) => {
                 return {
                   field: suite,
-                  headerName: SUITES[suite],
+                  headerName: all_suites[suite],
                   flex: 1,
                   renderCell: (params: GridRenderCellParams<any>) => {
                     const v = params.value;
@@ -412,7 +414,7 @@ export function SummaryPanel({
               suites.map((suite: string) => {
                 return {
                   field: suite,
-                  headerName: SUITES[suite],
+                  headerName: all_suites[suite],
                   flex: 1,
                   renderCell: (params: GridRenderCellParams<any>) => {
                     const v = params.value;
@@ -501,7 +503,7 @@ export function SummaryPanel({
               suites.map((suite: string) => {
                 return {
                   field: suite,
-                  headerName: SUITES[suite],
+                  headerName: all_suites[suite],
                   flex: 1,
                   renderCell: (params: GridRenderCellParams<any>) => {
                     const v = params.value;

diff --git a/torchci/components/benchmark/torchao/SuitePicker.tsx b/torchci/components/benchmark/torchao/SuitePicker.tsx
@@ -0,0 +1,46 @@
+import {
+  FormControl,
+  InputLabel,
+  MenuItem,
+  Select,
+  SelectChangeEvent,
+} from "@mui/material";
+
+export const SUITES: { [k: string]: string } = {
+  torchbench: "Torchbench",
+  huggingface: "Huggingface",
+  timm_models: "TIMM models",
+};
+
+export function SuitePicker({
+  suite,
+  setSuite,
+}: {
+  suite: string;
+  setSuite: any;
+}) {
+  function handleChange(e: SelectChangeEvent<string>) {
+    setSuite(e.target.value);
+  }
+
+  return (
+    <>
+      <FormControl>
+        <InputLabel id="suite-picker-input-label">Suite</InputLabel>
+        <Select
+          value={suite}
+          label="Suite"
+          labelId="suite-picker-select-label"
+          onChange={handleChange}
+          id="suite-picker-select"
+        >
+          {Object.keys(SUITES).map((suite) => (
+            <MenuItem key={suite} value={suite}>
+              {SUITES[suite]}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+    </>
+  );
+}
diff --git a/torchci/components/benchmark/torchao/common.tsx b/torchci/components/benchmark/torchao/common.tsx
@@ -0,0 +1,3 @@
+export const DEFAULT_REPO_NAME = "pytorch/ao";
+export const DTYPES = ["amp", "bfloat16"];
+export const DEFAULT_MODE = "inference";
diff --git a/torchci/pages/benchmark/compilers.tsx b/torchci/pages/benchmark/compilers.tsx
@@ -148,6 +148,7 @@ function Report({
           ...rBranchAndCommit,
           data: rData,
         }}
+        all_suites={SUITES}
       />
       <GraphPanel
         queryParams={queryParams}