Skip to content

Commit

Permalink
Merge pull request #211 from ClusterCockpit/uplot_roofline_scatter
Browse files Browse the repository at this point in the history 
Uplot roofline scatter
  • Loading branch information
@moebiusband73
moebiusband73 authored Sep 5, 2023
2 parents 07acbf6 + 20902f8 commit 84b63af
Show file tree
Hide file tree
Showing 11 changed files with 569 additions and 359 deletions.
30 changes: 16 additions & 14 deletions internal/repository/stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,28 +70,30 @@ func (r *JobRepository) buildStatsQuery(
var query sq.SelectBuilder
castType := r.getCastType()

// fmt.Sprintf(`CAST(ROUND((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / 3600) as %s) as value`, time.Now().Unix(), castType)

if col != "" {
// Scan columns: id, totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
query = sq.Select(col, "COUNT(job.id) as totalJobs",
fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s) as totalWalltime", castType),
fmt.Sprintf("CAST(SUM(job.num_nodes) as %s) as totalNodes", castType),
fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s) as totalNodeHours", castType),
fmt.Sprintf("CAST(SUM(job.num_hwthreads) as %s) as totalCores", castType),
fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s) as totalCoreHours", castType),
fmt.Sprintf("CAST(SUM(job.num_acc) as %s) as totalAccs", castType),
fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s) as totalAccHours", castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as %s) as totalWalltime`, time.Now().Unix(), castType),
fmt.Sprintf(`CAST(SUM(job.num_nodes) as %s) as totalNodes`, castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as %s) as totalNodeHours`, time.Now().Unix(), castType),
fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as %s) as totalCores`, castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as %s) as totalCoreHours`, time.Now().Unix(), castType),
fmt.Sprintf(`CAST(SUM(job.num_acc) as %s) as totalAccs`, castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as %s) as totalAccHours`, time.Now().Unix(), castType),
).From("job").GroupBy(col)

} else {
// Scan columns: totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
query = sq.Select("COUNT(job.id)",
fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s)", castType),
fmt.Sprintf("CAST(SUM(job.num_nodes) as %s)", castType),
fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s)", castType),
fmt.Sprintf("CAST(SUM(job.num_hwthreads) as %s)", castType),
fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s)", castType),
fmt.Sprintf("CAST(SUM(job.num_acc) as %s)", castType),
fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s)", castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as %s)`, time.Now().Unix(), castType),
fmt.Sprintf(`CAST(SUM(job.num_nodes) as %s)`, castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as %s)`, time.Now().Unix(), castType),
fmt.Sprintf(`CAST(SUM(job.num_hwthreads) as %s)`, castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_hwthreads) / 3600) as %s)`, time.Now().Unix(), castType),
fmt.Sprintf(`CAST(SUM(job.num_acc) as %s)`, castType),
fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as %s)`, time.Now().Unix(), castType),
).From("job")
}

Expand Down
7 changes: 7 additions & 0 deletions web/frontend/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions web/frontend/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"@rollup/plugin-commonjs": "^24.1.0",
"@rollup/plugin-node-resolve": "^15.0.2",
"@rollup/plugin-terser": "^0.4.1",
"@timohausmann/quadtree-js": "^1.2.5",
"rollup": "^3.21.0",
"rollup-plugin-css-only": "^4.3.0",
"rollup-plugin-svelte": "^7.1.4",
Expand Down
6 changes: 3 additions & 3 deletions web/frontend/src/Analysis.root.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import { binsFromFootprint } from './utils.js'
import ScatterPlot from './plots/Scatter.svelte'
import PlotTable from './PlotTable.svelte'
import Roofline from './plots/Roofline.svelte'
import RooflineHeatmap from './plots/RooflineHeatmap.svelte'
const { query: initq } = init()
Expand Down Expand Up @@ -293,7 +293,7 @@
{#each $topQuery.data.topList as te, i}
<tr>
<td><Icon name="circle-fill" style="color: {colors[i]};"/></td>
{#if groupSelection.key == 'User'}
{#if groupSelection.key == 'user'}
<th scope="col"><a href="/monitoring/user/{te.id}?cluster={cluster.name}">{te.id}</a></th>
{:else}
<th scope="col"><a href="/monitoring/jobs/?cluster={cluster.name}&project={te.id}&projectMatch=eq">{te.id}</a></th>
Expand All @@ -315,7 +315,7 @@
{:else if $rooflineQuery.data && cluster}
<div bind:clientWidth={colWidth2}>
{#key $rooflineQuery.data}
<Roofline
<RooflineHeatmap
width={colWidth2} height={300}
tiles={$rooflineQuery.data.rooflineHeatmap}
cluster={cluster.subClusters.length == 1 ? cluster.subClusters[0] : null}
Expand Down
26 changes: 10 additions & 16 deletions web/frontend/src/Job.root.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
groupByScope,
fetchMetricsStore,
checkMetricDisabled,
transformDataForRoofline
} from "./utils.js";
import {
Row,
Expand Down Expand Up @@ -131,7 +132,6 @@
let plots = {},
jobTags,
fullWidth,
statsTable;
$: document.title = $initq.fetching
? "Loading..."
Expand Down Expand Up @@ -190,7 +190,6 @@
}));
</script>

<div class="row" bind:clientWidth={fullWidth} />
<Row>
<Col>
{#if $initq.error}
Expand Down Expand Up @@ -245,7 +244,6 @@
{/if}
<Col>
<Polar
size={fullWidth / 4.1}
metrics={ccconfig[
`job_view_polarPlotMetrics:${$initq.data.job.cluster}`
] || ccconfig[`job_view_polarPlotMetrics`]}
Expand All @@ -255,28 +253,26 @@
</Col>
<Col>
<Roofline
width={fullWidth / 3 - 10}
height={fullWidth / 5}
renderTime={true}
cluster={clusters
.find((c) => c.name == $initq.data.job.cluster)
.subClusters.find(
(sc) => sc.name == $initq.data.job.subCluster
)}
flopsAny={$jobMetrics.data.jobMetrics.find(
(m) => m.name == "flops_any" && m.scope == "node"
)}
memBw={$jobMetrics.data.jobMetrics.find(
(m) => m.name == "mem_bw" && m.scope == "node"
)}
data={
transformDataForRoofline (
$jobMetrics.data.jobMetrics.find((m) => m.name == "flops_any" && m.scope == "node").metric,
$jobMetrics.data.jobMetrics.find((m) => m.name == "mem_bw" && m.scope == "node").metric
)
}
/>
</Col>
{:else}
<Col />
<Col />
{/if}
</Row>
<br />
<Row>
<Row class="mb-3">
<Col xs="auto">
{#if $initq.data}
<TagManagement job={$initq.data.job} bind:jobTags />
Expand All @@ -293,7 +289,6 @@
<Zoom timeseriesPlots={plots} />
</Col> -->
</Row>
<br />
<Row>
<Col>
{#if $jobMetrics.error}
Expand Down Expand Up @@ -340,8 +335,7 @@
{/if}
</Col>
</Row>
<br />
<Row>
<Row class="mt-2">
<Col>
{#if $initq.data}
<TabContent>
Expand Down
23 changes: 12 additions & 11 deletions web/frontend/src/Status.root.svelte
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<script>
import { getContext } from "svelte";
import Refresher from "./joblist/Refresher.svelte";
import Roofline, { transformPerNodeData } from "./plots/Roofline.svelte";
import Roofline from "./plots/Roofline.svelte";
import Pie, { colors } from "./plots/Pie.svelte";
import Histogram from "./plots/Histogram.svelte";
import {
Expand All @@ -16,7 +16,7 @@
Progress,
Icon,
} from "sveltestrap";
import { init, convert2uplot } from "./utils.js";
import { init, convert2uplot, transformPerNodeDataForRoofline } from "./utils.js";
import { scaleNumbers } from "./units.js";
import {
queryStore,
Expand All @@ -31,8 +31,8 @@
export let cluster;
let plotWidths = [],
colWidth1 = 0,
colWidth2;
colWidth1,
colWidth2
let from = new Date(Date.now() - 5 * 60 * 1000),
to = new Date(Date.now());
const topOptions = [
Expand Down Expand Up @@ -427,24 +427,25 @@
<div bind:clientWidth={plotWidths[i]}>
{#key $mainQuery.data.nodeMetrics}
<Roofline
allowSizeChange={true}
width={plotWidths[i] - 10}
height={300}
colorDots={true}
showTime={false}
cluster={subCluster}
data={transformPerNodeData(
$mainQuery.data.nodeMetrics.filter(
(data) => data.subCluster == subCluster.name
data={
transformPerNodeDataForRoofline(
$mainQuery.data.nodeMetrics.filter(
(data) => data.subCluster == subCluster.name
)
)
)}
}
/>
{/key}
</div>
</Col>
</Row>
{/each}
<hr style="margin-top: -1em;" />
<hr/>
<!-- Usage Stats as Histograms -->
Expand Down
12 changes: 9 additions & 3 deletions web/frontend/src/plots/Polar.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
LineElement
);
export let size
export let metrics
export let cluster
export let jobMetrics
export let height = 365
const metricConfig = getContext('metrics')
Expand Down Expand Up @@ -89,13 +89,19 @@
// No custom defined options but keep for clarity
const options = {
maintainAspectRatio: false,
animation: false
animation: false,
scales: { // fix scale
r: {
suggestedMin: 0.0,
suggestedMax: 1.0
}
}
}
</script>

<div class="chart-container">
<Radar {data} {options} width={size} height={size}/>
<Radar {data} {options} {height}/>
</div>

<style>
Expand Down
Loading

0 comments on commit 84b63af

Please sign in to comment.