Skip to content

Commit

Permalink
refactor: Use timestamp as query parameter
Browse files Browse the repository at this point in the history 
* Grafana passes ISO time always in UTC zone

* If we want to use local browser time, we need to use timestamp as query parameter

* We simplify /api/jobs end point to accept timestamp as query parameter

* Update tests and fixtures accordingly

Signed-off-by: Mahendra Paipuri <[email protected]>
  • Loading branch information
@mahendrapaipuri
mahendrapaipuri committed Jan 4, 2024
1 parent b574eb9 commit d8d5503
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 73 deletions.
7 changes: 6 additions & 1 deletion pkg/jobstats/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,13 +436,15 @@ func (j *jobStatsDB) prepareInsertStatement(tx *sql.Tx) (*sql.Stmt, error) {

// Insert job stat into DB
func (j *jobStatsDB) insertJobs(statement *sql.Stmt, jobStats []base.BatchJob) {
var err error
for _, jobStat := range jobStats {
// Empty job
if jobStat == (base.BatchJob{}) {
continue
}

// level.Debug(j.logger).Log("msg", "Inserting job", "jobid", jobStat.Jobid)
_, err := statement.Exec(
_, err = statement.Exec(
jobStat.Jobid,
jobStat.Jobuuid,
jobStat.Partition,
Expand All @@ -455,6 +457,9 @@ func (j *jobStatsDB) insertJobs(statement *sql.Stmt, jobStats []base.BatchJob) {
jobStat.Submit,
jobStat.Start,
jobStat.End,
jobStat.SubmitTS,
jobStat.StartTS,
jobStat.EndTS,
jobStat.Elapsed,
jobStat.Exitcode,
jobStat.State,
Expand Down
2 changes: 1 addition & 1 deletion pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query-all.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"},{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"14508","id":"88a46e84-ffce-52ea-37e9-47b39d9ccfb3","partition":"part1","qos":"qos1","account":"acc4","group":"grp4","gid":"1004","user":"usr4","uid":"1004","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1004","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr4"},{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"},{"jobid":"11508","id":"d4956307-af17-870a-2fa0-38375105d257","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr15"},{"jobid":"81510","id":"938832b4-33b4-3303-b002-8150f737de7e","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr23"}]}
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"},{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"14508","id":"88a46e84-ffce-52ea-37e9-47b39d9ccfb3","partition":"part1","qos":"qos1","account":"acc4","group":"grp4","gid":"1004","user":"usr4","uid":"1004","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1004","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr4"},{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"},{"jobid":"11508","id":"d4956307-af17-870a-2fa0-38375105d257","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr15"},{"jobid":"81510","id":"938832b4-33b4-3303-b002-8150f737de7e","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr23"}]}
2 changes: 1 addition & 1 deletion pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"}]}
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"}]}
2 changes: 1 addition & 1 deletion pkg/jobstats/fixtures/output/e2e-test-stats-server-jobid-query.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"}]}
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"}]}
2 changes: 1 addition & 1 deletion pkg/jobstats/fixtures/output/e2e-test-stats-server-jobuuid-query.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"}]}
{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"}]}
18 changes: 9 additions & 9 deletions pkg/jobstats/fixtures/sacct
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/bin/bash

echo """JobID|Partition|QoS|Account|Group|GID|User|UID|Submit|Start|End|Elapsed|ElapsedRaw|ExitCode|State|NNodes|Ncpus|NodeList|JobName|WorkDir
1479763|part1|qos1|acc1|grp1|1001|usr1|1001|2022-02-21T14:37:02|2022-02-21T14:37:07|2022-02-21T15:26:29|00:49:22|3000|0:0|CANCELLED by 1001|1|8|compute-0|test_script1|/home/usr1
1481508|part1|qos1|acc2|grp2|1002|usr2|1002|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:08:17|4500|0:0|CANCELLED by 1002|2|16|compute-[0-2]|test_script2|/home/usr2
1481510|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:00:17|789|0:0|CANCELLED by 1003|2|16|compute-[0-2]|test_script2|/home/usr3
147973|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T14:37:02|2023-02-21T14:37:07|2023-02-21T15:26:29|00:49:22|3000|0:0|CANCELLED by 1003|1|8|compute-0|test_script1|/home/usr3
14508|part1|qos1|acc4|grp4|1004|usr4|1004|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:08:17|4500|0:0|CANCELLED by 1004|2|16|compute-[0-2]|test_script2|/home/usr4
147973|part1|qos1|acc1|gr1|1001|usr1|1001|2023-12-21T15:48:20|2023-12-21T15:49:06|2023-12-21T15:57:23|00:00:17|567|0:0|CANCELLED by 1001|2|16|compute-[0-2]|test_script2|/home/usr1
1479763|part1|qos1|acc1|grp8|1008|usr8|1008|2023-02-21T14:37:02|2023-02-21T14:37:07|2023-02-21T15:26:29|00:49:22|3000|0:0|CANCELLED by 1008|1|8|compute-0|test_script1|/home/usr8
11508|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:08:17|4500|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr15
81510|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:00:17|3533|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr23
1479763|part1|qos1|acc1|grp1|1001|usr1|1001|2022-02-21T14:37:02+0100|2022-02-21T14:37:07+0100|2022-02-21T15:26:29+0100|00:49:22|3000|0:0|CANCELLED by 1001|1|8|compute-0|test_script1|/home/usr1
1481508|part1|qos1|acc2|grp2|1002|usr2|1002|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:08:17|4500|0:0|CANCELLED by 1002|2|16|compute-[0-2]|test_script2|/home/usr2
1481510|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:00:17|789|0:0|CANCELLED by 1003|2|16|compute-[0-2]|test_script2|/home/usr3
147973|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T14:37:02+0100|2023-02-21T14:37:07+0100|2023-02-21T15:26:29+0100|00:49:22|3000|0:0|CANCELLED by 1003|1|8|compute-0|test_script1|/home/usr3
14508|part1|qos1|acc4|grp4|1004|usr4|1004|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:08:17|4500|0:0|CANCELLED by 1004|2|16|compute-[0-2]|test_script2|/home/usr4
147973|part1|qos1|acc1|gr1|1001|usr1|1001|2023-12-21T15:48:20+0100|2023-12-21T15:49:06+0100|2023-12-21T15:57:23+0100|00:00:17|567|0:0|CANCELLED by 1001|2|16|compute-[0-2]|test_script2|/home/usr1
1479763|part1|qos1|acc1|grp8|1008|usr8|1008|2023-02-21T14:37:02+0100|2023-02-21T14:37:07+0100|2023-02-21T15:26:29+0100|00:49:22|3000|0:0|CANCELLED by 1008|1|8|compute-0|test_script1|/home/usr8
11508|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:08:17|4500|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr15
81510|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:00:17|3533|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr23
"""
Loading

0 comments on commit d8d5503

Please sign in to comment.