From 270b9515650a3218838ca92c4cae714e8d388ec4 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Wed, 5 Jun 2024 19:41:16 -0400 Subject: [PATCH] Add torchao dashboard to benchmarks (#5284) Co-authored-by: Huy Do --- torchci/components/NavBar.tsx | 4 + .../benchmark/compilers/SummaryPanel.tsx | 14 +- .../benchmark/torchao/SuitePicker.tsx | 46 +++ .../components/benchmark/torchao/common.tsx | 3 + torchci/pages/benchmark/compilers.tsx | 1 + torchci/pages/benchmark/torchao.tsx | 359 ++++++++++++++++++ .../rockset/inductor/__sql/torchao_query.sql | 156 ++++++++ .../inductor/__sql/torchao_query_branches.sql | 21 + .../inductor/torchao_query.lambda.json | 66 ++++ .../torchao_query_branches.lambda.json | 36 ++ torchci/rockset/prodVersions.json | 4 +- 11 files changed, 703 insertions(+), 7 deletions(-) create mode 100644 torchci/components/benchmark/torchao/SuitePicker.tsx create mode 100644 torchci/components/benchmark/torchao/common.tsx create mode 100644 torchci/pages/benchmark/torchao.tsx create mode 100644 torchci/rockset/inductor/__sql/torchao_query.sql create mode 100644 torchci/rockset/inductor/__sql/torchao_query_branches.sql create mode 100644 torchci/rockset/inductor/torchao_query.lambda.json create mode 100644 torchci/rockset/inductor/torchao_query_branches.lambda.json diff --git a/torchci/components/NavBar.tsx b/torchci/components/NavBar.tsx index d4d3acd8a8..98043337dd 100644 --- a/torchci/components/NavBar.tsx +++ b/torchci/components/NavBar.tsx @@ -40,6 +40,10 @@ function NavBar() { name: "TorchInductor", href: "/benchmark/compilers", }, + { + name: "TorchAO", + href: "/benchmark/torchao", + }, { name: "TorchBench", href: "/torchbench/userbenchmark", diff --git a/torchci/components/benchmark/compilers/SummaryPanel.tsx b/torchci/components/benchmark/compilers/SummaryPanel.tsx index bc149002ff..fc11c19855 100644 --- a/torchci/components/benchmark/compilers/SummaryPanel.tsx +++ b/torchci/components/benchmark/compilers/SummaryPanel.tsx @@ -11,7 +11,6 @@ import { SCALE, SPEEDUP_THRESHOLD, } from "components/benchmark/compilers/common"; -import { SUITES } from "components/benchmark/compilers/SuitePicker"; import styles from "components/metrics.module.css"; import { TablePanelWithData } from "components/metrics/panels/TablePanel"; import { Granularity } from "components/metrics/panels/TimeSeriesPanel"; @@ -136,6 +135,7 @@ export function SummaryPanel({ dtype, lPerfData, rPerfData, + all_suites, }: { startTime: dayjs.Dayjs; stopTime: dayjs.Dayjs; @@ -144,6 +144,7 @@ export function SummaryPanel({ dtype: string; lPerfData: BranchAndCommitPerfData; rPerfData: BranchAndCommitPerfData; + all_suites: { [key: string]: string }; }) { const fields: { [k: string]: any } = { passrate_display: computePassrate, @@ -167,7 +168,8 @@ export function SummaryPanel({ fields ); - const suites = Object.keys(SUITES); + const suites = Object.keys(all_suites); + // Combine both sides const passrate = combineLeftAndRight( lCommit, @@ -229,7 +231,7 @@ export function SummaryPanel({ suites.map((suite: string) => { return { field: suite, - headerName: SUITES[suite], + headerName: all_suites[suite], flex: 1, renderCell: (params: GridRenderCellParams) => { const v = params.value; @@ -319,7 +321,7 @@ export function SummaryPanel({ suites.map((suite: string) => { return { field: suite, - headerName: SUITES[suite], + headerName: all_suites[suite], flex: 1, renderCell: (params: GridRenderCellParams) => { const v = params.value; @@ -412,7 +414,7 @@ export function SummaryPanel({ suites.map((suite: string) => { return { field: suite, - headerName: SUITES[suite], + headerName: all_suites[suite], flex: 1, renderCell: (params: GridRenderCellParams) => { const v = params.value; @@ -501,7 +503,7 @@ export function SummaryPanel({ suites.map((suite: string) => { return { field: suite, - headerName: SUITES[suite], + headerName: all_suites[suite], flex: 1, renderCell: (params: GridRenderCellParams) => { const v = params.value; diff --git a/torchci/components/benchmark/torchao/SuitePicker.tsx b/torchci/components/benchmark/torchao/SuitePicker.tsx new file mode 100644 index 0000000000..8c2a855442 --- /dev/null +++ b/torchci/components/benchmark/torchao/SuitePicker.tsx @@ -0,0 +1,46 @@ +import { + FormControl, + InputLabel, + MenuItem, + Select, + SelectChangeEvent, +} from "@mui/material"; + +export const SUITES: { [k: string]: string } = { + torchbench: "Torchbench", + huggingface: "Huggingface", + timm_models: "TIMM models", +}; + +export function SuitePicker({ + suite, + setSuite, +}: { + suite: string; + setSuite: any; +}) { + function handleChange(e: SelectChangeEvent) { + setSuite(e.target.value); + } + + return ( + <> + + Suite + + + + ); +} diff --git a/torchci/components/benchmark/torchao/common.tsx b/torchci/components/benchmark/torchao/common.tsx new file mode 100644 index 0000000000..634234159d --- /dev/null +++ b/torchci/components/benchmark/torchao/common.tsx @@ -0,0 +1,3 @@ +export const DEFAULT_REPO_NAME = "pytorch/ao"; +export const DTYPES = ["amp", "bfloat16"]; +export const DEFAULT_MODE = "inference"; diff --git a/torchci/pages/benchmark/compilers.tsx b/torchci/pages/benchmark/compilers.tsx index 0f323c7610..d12fa50231 100644 --- a/torchci/pages/benchmark/compilers.tsx +++ b/torchci/pages/benchmark/compilers.tsx @@ -148,6 +148,7 @@ function Report({ ...rBranchAndCommit, data: rData, }} + all_suites={SUITES} /> ; + } + + return ( +
+ + + + + +
+ ); +} + +export default function Page() { + const router = useRouter(); + + const defaultStartTime = dayjs().subtract(LAST_N_DAYS, "day"); + const [startTime, setStartTime] = useState(defaultStartTime); + const defaultStopTime = dayjs(); + const [stopTime, setStopTime] = useState(defaultStopTime); + const [timeRange, setTimeRange] = useState(LAST_N_DAYS); + + const [granularity, setGranularity] = useState("hour"); + const [suite, setSuite] = useState(Object.keys(SUITES)[0]); + const [mode, setMode] = useState(DEFAULT_MODE); + const [dtype, setDType] = useState(MODES[DEFAULT_MODE]); + const [lBranch, setLBranch] = useState(MAIN_BRANCH); + const [lCommit, setLCommit] = useState(""); + const [rBranch, setRBranch] = useState(MAIN_BRANCH); + const [rCommit, setRCommit] = useState(""); + const [baseUrl, setBaseUrl] = useState(""); + + // Set the dropdown value what is in the param + useEffect(() => { + const startTime: string = (router.query.startTime as string) ?? undefined; + if (startTime !== undefined) { + setStartTime(dayjs(startTime)); + + if (dayjs(startTime).valueOf() !== defaultStartTime.valueOf()) { + setTimeRange(-1); + } + } + + const stopTime: string = (router.query.stopTime as string) ?? undefined; + if (stopTime !== undefined) { + setStopTime(dayjs(stopTime)); + + if (dayjs(stopTime).valueOf() !== defaultStopTime.valueOf()) { + setTimeRange(-1); + } + } + + const granularity: Granularity = + (router.query.granularity as Granularity) ?? undefined; + if (granularity !== undefined) { + setGranularity(granularity); + } + + const suite: string = (router.query.suite as string) ?? undefined; + if (suite !== undefined) { + setSuite(suite); + } + + const mode: string = (router.query.mode as string) ?? undefined; + if (mode !== undefined) { + setMode(mode); + } + + const dtype: string = (router.query.dtype as string) ?? undefined; + if (dtype !== undefined) { + setDType(dtype); + } + + const lBranch: string = (router.query.lBranch as string) ?? undefined; + if (lBranch !== undefined) { + setLBranch(lBranch); + } + + const lCommit: string = (router.query.lCommit as string) ?? undefined; + if (lCommit !== undefined) { + setLCommit(lCommit); + } + + const rBranch: string = (router.query.rBranch as string) ?? undefined; + if (rBranch !== undefined) { + setRBranch(rBranch); + } + + const rCommit: string = (router.query.rCommit as string) ?? undefined; + if (rCommit !== undefined) { + setRCommit(rCommit); + } + + setBaseUrl( + `${window.location.protocol}//${ + window.location.host + }${router.asPath.replace(/\?.+/, "")}` + ); + }, [router.query]); + + const queryParams: RocksetParam[] = [ + { + name: "timezone", + type: "string", + value: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + { + name: "startTime", + type: "string", + value: startTime, + }, + { + name: "stopTime", + type: "string", + value: stopTime, + }, + { + name: "granularity", + type: "string", + value: granularity, + }, + { + name: "mode", + type: "string", + value: mode, + }, + { + name: "dtypes", + type: "string", + value: dtype, + }, + ]; + + return ( +
+ + + TorchAO Performance DashBoard + + + + + + + + + + + + —Diff→ + + + + + +
+ ); +} diff --git a/torchci/rockset/inductor/__sql/torchao_query.sql b/torchci/rockset/inductor/__sql/torchao_query.sql new file mode 100644 index 0000000000..6b8df1aa1a --- /dev/null +++ b/torchci/rockset/inductor/__sql/torchao_query.sql @@ -0,0 +1,156 @@ +WITH performance_results AS ( + SELECT + name, + IF(speedup = 'infra_error', NULL, speedup) AS speedup, -- Handle the recent burst of infra error + REPLACE( + filename, + CONCAT( + '_', : dtypes, '_', : mode, '_', : device, + '_performance' + ) + ) AS filename, + compilation_latency, + compression_ratio, + abs_latency, + mfu, + memory_bandwidth, + dynamo_peak_mem, + eager_peak_mem, + workflow_id, + CAST(job_id AS INT) AS job_id, + FORMAT_ISO8601( + DATE_TRUNC(: granularity, _event_time) + ) AS granularity_bucket, + head_branch, +FROM + inductor.torchao_perf_stats +WHERE + filename LIKE '%_performance' + AND filename LIKE CONCAT( + '%_', : dtypes, '_', : mode, '_', : device, + '_%' + ) + AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) + AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) + AND (workflow_id = :workflowId OR :workflowId = 0) +), +accuracy_results AS ( + SELECT + name, + accuracy, + REPLACE( + filename, + CONCAT( + '_', : dtypes, '_', : mode, '_', : device, + '_accuracy' + ) + ) AS filename, + workflow_id, + CAST(job_id AS INT) AS job_id, + FROM + inductor.torchao_perf_stats + WHERE + filename LIKE '%_accuracy' + AND filename LIKE CONCAT( + '%_', : dtypes, '_', : mode, '_', : device, + '_%' + ) + AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) + AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) + AND (workflow_id = :workflowId OR :workflowId = 0) + AND accuracy != 'model_fail_to_load' + AND accuracy != 'eager_fail_to_run' +), +results AS ( + SELECT + performance_results.granularity_bucket AS granularity_bucket, + performance_results.workflow_id AS workflow_id, + performance_results.job_id AS job_id, + performance_results.head_branch AS head_branch, + CASE + WHEN performance_results.filename LIKE '%_torchbench' THEN 'torchbench' + WHEN performance_results.filename LIKE '%_timm_models' THEN 'timm_models' + WHEN performance_results.filename LIKE '%_huggingface' THEN 'huggingface' + ELSE NULL + END AS suite, + CASE + WHEN performance_results.filename LIKE '%_torchbench' THEN REPLACE( + performance_results.filename, '_torchbench' + ) + WHEN performance_results.filename LIKE '%_timm_models' THEN REPLACE( + performance_results.filename, '_timm_models' + ) + WHEN performance_results.filename LIKE '%_huggingface' THEN REPLACE( + performance_results.filename, '_huggingface' + ) + ELSE NULL + END AS compiler, + performance_results.name, + IF(TRY_CAST(speedup AS FLOAT) IS NOT NULL, + CAST(speedup AS FLOAT), + 0.0 + ) AS speedup, + accuracy_results.accuracy AS accuracy, + IF(TRY_CAST(compilation_latency AS FLOAT) IS NOT NULL, + CAST(compilation_latency AS FLOAT), + 0.0 + ) AS compilation_latency, + IF(TRY_CAST(compression_ratio AS FLOAT) IS NOT NULL, + CAST(compression_ratio AS FLOAT), + 0.0 + ) AS compression_ratio, + IF(TRY_CAST(abs_latency AS FLOAT) IS NOT NULL, + CAST(abs_latency AS FLOAT), + 0.0 + ) AS abs_latency, + IF(TRY_CAST(mfu AS FLOAT) IS NOT NULL, + CAST(mfu AS FLOAT), + 0.0 + ) AS mfu, + IF(TRY_CAST(memory_bandwidth AS FLOAT) IS NOT NULL, + CAST(memory_bandwidth AS FLOAT), + 0.0 + ) AS memory_bandwidth, + IF(TRY_CAST(dynamo_peak_mem AS FLOAT) IS NOT NULL, + CAST(dynamo_peak_mem AS FLOAT), + 0.0 + ) AS dynamo_peak_mem, + IF(TRY_CAST(eager_peak_mem AS FLOAT) IS NOT NULL, + CAST(eager_peak_mem AS FLOAT), + 0.0 + ) AS eager_peak_mem, + FROM + performance_results + LEFT JOIN accuracy_results ON performance_results.name = accuracy_results.name + AND performance_results.filename = accuracy_results.filename + AND performance_results.workflow_id = accuracy_results.workflow_id +) +SELECT DISTINCT + results.workflow_id, + -- As the JSON response is pretty big, only return the field if it's needed + IF(:getJobId, results.job_id, NULL) AS job_id, + results.suite, + results.compiler, + results.name, + results.speedup, + results.accuracy, + results.compilation_latency, + results.compression_ratio, + results.abs_latency, + results.mfu, + results.memory_bandwidth, + results.dynamo_peak_mem, + results.eager_peak_mem, + results.granularity_bucket, +FROM + results +WHERE + ARRAY_CONTAINS(SPLIT(:suites, ','), LOWER(results.suite)) + AND (ARRAY_CONTAINS(SPLIT(:compilers, ','), LOWER(results.compiler)) OR :compilers = '') + AND (ARRAY_CONTAINS(SPLIT(:branches, ','), results.head_branch) OR :branches = '') +ORDER BY + granularity_bucket DESC, + workflow_id DESC, + suite ASC, + compiler ASC, + name ASC \ No newline at end of file diff --git a/torchci/rockset/inductor/__sql/torchao_query_branches.sql b/torchci/rockset/inductor/__sql/torchao_query_branches.sql new file mode 100644 index 0000000000..1659e54a98 --- /dev/null +++ b/torchci/rockset/inductor/__sql/torchao_query_branches.sql @@ -0,0 +1,21 @@ +SELECT + DISTINCT head_branch, + head_sha, + FORMAT_ISO8601( + DATE_TRUNC( + : granularity, _event_time + ) + ) AS event_time, +FROM + inductor.torchao_perf_stats +WHERE + torchao_perf_stats._event_time >= PARSE_DATETIME_ISO8601(: startTime) + AND torchao_perf_stats._event_time < PARSE_DATETIME_ISO8601(: stopTime) + AND torchao_perf_stats.filename LIKE '%_performance' + AND torchao_perf_stats.filename LIKE CONCAT( + '%_', : dtypes, '_', : mode, '_', : device, + '_%' + ) +ORDER BY + head_branch, + event_time DESC diff --git a/torchci/rockset/inductor/torchao_query.lambda.json b/torchci/rockset/inductor/torchao_query.lambda.json new file mode 100644 index 0000000000..c890d7abb3 --- /dev/null +++ b/torchci/rockset/inductor/torchao_query.lambda.json @@ -0,0 +1,66 @@ +{ + "sql_path": "__sql/torchao_query.sql", + "default_parameters": [ + { + "name": "branches", + "type": "string", + "value": "main" + }, + { + "name": "compilers", + "type": "string", + "value": "" + }, + { + "name": "device", + "type": "string", + "value": "cuda" + }, + { + "name": "dtypes", + "type": "string", + "value": "bfloat16" + }, + { + "name": "getJobId", + "type": "bool", + "value": "false" + }, + { + "name": "granularity", + "type": "string", + "value": "day" + }, + { + "name": "mode", + "type": "string", + "value": "inference" + }, + { + "name": "startTime", + "type": "string", + "value": "2024-06-01T00:00:00.00Z" + }, + { + "name": "stopTime", + "type": "string", + "value": "2024-06-06T00:00:00.00Z" + }, + { + "name": "suites", + "type": "string", + "value": "torchbench,huggingface,timm_models" + }, + { + "name": "timezone", + "type": "string", + "value": "America/Los_Angeles" + }, + { + "name": "workflowId", + "type": "string", + "value": "0" + } + ], + "description": "TorchAO Query" +} \ No newline at end of file diff --git a/torchci/rockset/inductor/torchao_query_branches.lambda.json b/torchci/rockset/inductor/torchao_query_branches.lambda.json new file mode 100644 index 0000000000..79428ba5e2 --- /dev/null +++ b/torchci/rockset/inductor/torchao_query_branches.lambda.json @@ -0,0 +1,36 @@ +{ + "sql_path": "__sql/torchao_query_branches.sql", + "default_parameters": [ + { + "name": "device", + "type": "string", + "value": "cuda" + }, + { + "name": "dtypes", + "type": "string", + "value": "bfloat16" + }, + { + "name": "granularity", + "type": "string", + "value": "day" + }, + { + "name": "mode", + "type": "string", + "value": "inference" + }, + { + "name": "startTime", + "type": "string", + "value": "2024-06-01T00:00:00.00Z" + }, + { + "name": "stopTime", + "type": "string", + "value": "2024-06-06T00:00:00.00Z" + } + ], + "description": "TorchAO Query for commit branches" +} \ No newline at end of file diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json index 6b7666c171..af0b8f806d 100644 --- a/torchci/rockset/prodVersions.json +++ b/torchci/rockset/prodVersions.json @@ -83,7 +83,9 @@ }, "inductor": { "compilers_benchmark_performance": "ee1751ed1f0e68cd", - "compilers_benchmark_performance_branches": "8896fe6bbd61e7dc" + "compilers_benchmark_performance_branches": "8896fe6bbd61e7dc", + "torchao_query": "011929830b89f966", + "torchao_query_branches": "dae2141eab66e839" }, "torchbench": { "torchbench_list_userbenchmarks": "fcfa85c5d1056e8f",