From d8d5503205971be067781bcf2583cfa1474fd7d5 Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Thu, 4 Jan 2024 13:00:59 +0100 Subject: [PATCH] refactor: Use timestamp as query parameter * Grafana passes ISO time always in UTC zone * If we want to use local browser time, we need to use timestamp as query parameter * We simplify /api/jobs end point to accept timestamp as query parameter * Update tests and fixtures accordingly Signed-off-by: Mahendra Paipuri --- pkg/jobstats/db/db.go | 7 +- .../e2e-test-stats-server-admin-query-all.txt | 2 +- .../e2e-test-stats-server-admin-query.txt | 2 +- .../e2e-test-stats-server-jobid-query.txt | 2 +- .../e2e-test-stats-server-jobuuid-query.txt | 2 +- pkg/jobstats/fixtures/sacct | 18 +-- pkg/jobstats/server/server.go | 112 ++++++++++-------- pkg/jobstats/server/server_test.go | 6 +- scripts/e2e-test.sh | 20 ++-- 9 files changed, 98 insertions(+), 73 deletions(-) diff --git a/pkg/jobstats/db/db.go b/pkg/jobstats/db/db.go index 9af8b87b..4ba1c26d 100644 --- a/pkg/jobstats/db/db.go +++ b/pkg/jobstats/db/db.go @@ -436,13 +436,15 @@ func (j *jobStatsDB) prepareInsertStatement(tx *sql.Tx) (*sql.Stmt, error) { // Insert job stat into DB func (j *jobStatsDB) insertJobs(statement *sql.Stmt, jobStats []base.BatchJob) { + var err error for _, jobStat := range jobStats { // Empty job if jobStat == (base.BatchJob{}) { continue } + // level.Debug(j.logger).Log("msg", "Inserting job", "jobid", jobStat.Jobid) - _, err := statement.Exec( + _, err = statement.Exec( jobStat.Jobid, jobStat.Jobuuid, jobStat.Partition, @@ -455,6 +457,9 @@ func (j *jobStatsDB) insertJobs(statement *sql.Stmt, jobStats []base.BatchJob) { jobStat.Submit, jobStat.Start, jobStat.End, + jobStat.SubmitTS, + jobStat.StartTS, + jobStat.EndTS, jobStat.Elapsed, jobStat.Exitcode, jobStat.State, diff --git a/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query-all.txt b/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query-all.txt index c54dfa1c..76a5b837 100644 --- a/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query-all.txt +++ b/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query-all.txt @@ -1 +1 @@ -{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"},{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"14508","id":"88a46e84-ffce-52ea-37e9-47b39d9ccfb3","partition":"part1","qos":"qos1","account":"acc4","group":"grp4","gid":"1004","user":"usr4","uid":"1004","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1004","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr4"},{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"},{"jobid":"11508","id":"d4956307-af17-870a-2fa0-38375105d257","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr15"},{"jobid":"81510","id":"938832b4-33b4-3303-b002-8150f737de7e","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr23"}]} +{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"},{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"14508","id":"88a46e84-ffce-52ea-37e9-47b39d9ccfb3","partition":"part1","qos":"qos1","account":"acc4","group":"grp4","gid":"1004","user":"usr4","uid":"1004","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1004","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr4"},{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"},{"jobid":"11508","id":"d4956307-af17-870a-2fa0-38375105d257","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr15"},{"jobid":"81510","id":"938832b4-33b4-3303-b002-8150f737de7e","partition":"part1","qos":"qos1","account":"acc1","group":"grp15","gid":"1015","user":"usr15","uid":"1015","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1015","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr23"}]} diff --git a/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query.txt b/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query.txt index fdb3df92..dd44c114 100644 --- a/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query.txt +++ b/pkg/jobstats/fixtures/output/e2e-test-stats-server-admin-query.txt @@ -1 +1 @@ -{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"}]} +{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"147973","id":"d8b28c2c-2011-d572-de94-8ec4facb4a2a","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr3"},{"jobid":"1481510","id":"b76ecf69-4d2f-076b-047d-2bcc8503b4cb","partition":"part1","qos":"qos1","account":"acc3","group":"grp3","gid":"1003","user":"usr3","uid":"1003","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:00:17","exitcode":"0:0","state":"CANCELLED by 1003","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr3"}]} diff --git a/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobid-query.txt b/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobid-query.txt index 1c4637a6..7a62e12e 100644 --- a/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobid-query.txt +++ b/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobid-query.txt @@ -1 +1 @@ -{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02","start":"2023-02-21T14:37:07","end":"2023-02-21T15:26:29","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"}]} +{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1479763","id":"a04088e8-2699-2a9b-bc27-30282679ebb3","partition":"part1","qos":"qos1","account":"acc1","group":"grp8","gid":"1008","user":"usr8","uid":"1008","submit":"2023-02-21T14:37:02+0100","start":"2023-02-21T14:37:07+0100","end":"2023-02-21T15:26:29+0100","submitts":"1676986622","startts":"1676986627","endts":"1676989589","elapsed":"00:49:22","exitcode":"0:0","state":"CANCELLED by 1008","nnodes":"1","ncpus":"8","nodelist":"compute-0","nodelistexp":"compute-0","jobname":"test_script1","workdir":"/home/usr8"}]} diff --git a/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobuuid-query.txt b/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobuuid-query.txt index ff19904e..709912a6 100644 --- a/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobuuid-query.txt +++ b/pkg/jobstats/fixtures/output/e2e-test-stats-server-jobuuid-query.txt @@ -1 +1 @@ -{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20","start":"2023-02-21T15:49:06","end":"2023-02-21T15:57:23","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"}]} +{"status":"success","errorType":"","error":"","warnings":null,"data":[{"jobid":"1481508","id":"baee651d-df44-af2c-fa09-50f5523b5e19","partition":"part1","qos":"qos1","account":"acc2","group":"grp2","gid":"1002","user":"usr2","uid":"1002","submit":"2023-02-21T15:48:20+0100","start":"2023-02-21T15:49:06+0100","end":"2023-02-21T15:57:23+0100","submitts":"1676990900","startts":"1676990946","endts":"1676991443","elapsed":"00:08:17","exitcode":"0:0","state":"CANCELLED by 1002","nnodes":"2","ncpus":"16","nodelist":"compute-[0-2]","nodelistexp":"compute-0|compute-1|compute-2","jobname":"test_script2","workdir":"/home/usr2"}]} diff --git a/pkg/jobstats/fixtures/sacct b/pkg/jobstats/fixtures/sacct index cd522765..3a022a20 100755 --- a/pkg/jobstats/fixtures/sacct +++ b/pkg/jobstats/fixtures/sacct @@ -1,13 +1,13 @@ #!/bin/bash echo """JobID|Partition|QoS|Account|Group|GID|User|UID|Submit|Start|End|Elapsed|ElapsedRaw|ExitCode|State|NNodes|Ncpus|NodeList|JobName|WorkDir -1479763|part1|qos1|acc1|grp1|1001|usr1|1001|2022-02-21T14:37:02|2022-02-21T14:37:07|2022-02-21T15:26:29|00:49:22|3000|0:0|CANCELLED by 1001|1|8|compute-0|test_script1|/home/usr1 -1481508|part1|qos1|acc2|grp2|1002|usr2|1002|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:08:17|4500|0:0|CANCELLED by 1002|2|16|compute-[0-2]|test_script2|/home/usr2 -1481510|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:00:17|789|0:0|CANCELLED by 1003|2|16|compute-[0-2]|test_script2|/home/usr3 -147973|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T14:37:02|2023-02-21T14:37:07|2023-02-21T15:26:29|00:49:22|3000|0:0|CANCELLED by 1003|1|8|compute-0|test_script1|/home/usr3 -14508|part1|qos1|acc4|grp4|1004|usr4|1004|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:08:17|4500|0:0|CANCELLED by 1004|2|16|compute-[0-2]|test_script2|/home/usr4 -147973|part1|qos1|acc1|gr1|1001|usr1|1001|2023-12-21T15:48:20|2023-12-21T15:49:06|2023-12-21T15:57:23|00:00:17|567|0:0|CANCELLED by 1001|2|16|compute-[0-2]|test_script2|/home/usr1 -1479763|part1|qos1|acc1|grp8|1008|usr8|1008|2023-02-21T14:37:02|2023-02-21T14:37:07|2023-02-21T15:26:29|00:49:22|3000|0:0|CANCELLED by 1008|1|8|compute-0|test_script1|/home/usr8 -11508|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:08:17|4500|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr15 -81510|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20|2023-02-21T15:49:06|2023-02-21T15:57:23|00:00:17|3533|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr23 +1479763|part1|qos1|acc1|grp1|1001|usr1|1001|2022-02-21T14:37:02+0100|2022-02-21T14:37:07+0100|2022-02-21T15:26:29+0100|00:49:22|3000|0:0|CANCELLED by 1001|1|8|compute-0|test_script1|/home/usr1 +1481508|part1|qos1|acc2|grp2|1002|usr2|1002|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:08:17|4500|0:0|CANCELLED by 1002|2|16|compute-[0-2]|test_script2|/home/usr2 +1481510|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:00:17|789|0:0|CANCELLED by 1003|2|16|compute-[0-2]|test_script2|/home/usr3 +147973|part1|qos1|acc3|grp3|1003|usr3|1003|2023-02-21T14:37:02+0100|2023-02-21T14:37:07+0100|2023-02-21T15:26:29+0100|00:49:22|3000|0:0|CANCELLED by 1003|1|8|compute-0|test_script1|/home/usr3 +14508|part1|qos1|acc4|grp4|1004|usr4|1004|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:08:17|4500|0:0|CANCELLED by 1004|2|16|compute-[0-2]|test_script2|/home/usr4 +147973|part1|qos1|acc1|gr1|1001|usr1|1001|2023-12-21T15:48:20+0100|2023-12-21T15:49:06+0100|2023-12-21T15:57:23+0100|00:00:17|567|0:0|CANCELLED by 1001|2|16|compute-[0-2]|test_script2|/home/usr1 +1479763|part1|qos1|acc1|grp8|1008|usr8|1008|2023-02-21T14:37:02+0100|2023-02-21T14:37:07+0100|2023-02-21T15:26:29+0100|00:49:22|3000|0:0|CANCELLED by 1008|1|8|compute-0|test_script1|/home/usr8 +11508|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:08:17|4500|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr15 +81510|part1|qos1|acc1|grp15|1015|usr15|1015|2023-02-21T15:48:20+0100|2023-02-21T15:49:06+0100|2023-02-21T15:57:23+0100|00:00:17|3533|0:0|CANCELLED by 1015|2|16|compute-[0-2]|test_script2|/home/usr23 """ diff --git a/pkg/jobstats/server/server.go b/pkg/jobstats/server/server.go index 9d609462..1e3ceb3a 100644 --- a/pkg/jobstats/server/server.go +++ b/pkg/jobstats/server/server.go @@ -8,6 +8,7 @@ import ( "net/http" _ "net/http/pprof" "slices" + "strconv" "strings" "sync" "time" @@ -55,17 +56,17 @@ func (q *Query) query(s string) { } // Add parameter and its placeholder -func (q *Query) param(val string) { - q.builder.WriteString("?") - q.params = append(q.params, val) -} - -// Add multiple parameters and its placeholder -func (q *Query) multiParam(val []string) { +func (q *Query) param(val []string) { q.builder.WriteString(fmt.Sprintf("(%s)", strings.Join(strings.Split(strings.Repeat("?", len(val)), ""), ","))) q.params = append(q.params, val...) } +// Add multiple parameters and its placeholder +// func (q *Query) multiParam(val []string) { +// q.builder.WriteString(fmt.Sprintf("(%s)", strings.Join(strings.Split(strings.Repeat("?", len(val)), ""), ","))) +// q.params = append(q.params, val...) +// } + // Get current query string and its parameters func (q *Query) get() (string, []string) { return q.builder.String(), q.params @@ -260,6 +261,7 @@ func (s *JobstatsServer) jobsErrorResponse(errorString string, errorType string, // GET /api/jobs // Get jobs of a user based on query params func (s *JobstatsServer) jobs(w http.ResponseWriter, r *http.Request) { + var fromTime, toTime time.Time var response base.JobsResponse s.setHeaders(w) w.WriteHeader(http.StatusOK) @@ -281,84 +283,91 @@ func (s *JobstatsServer) jobs(w http.ResponseWriter, r *http.Request) { q.query(fmt.Sprintf("SELECT * FROM %s", s.dbConfig.JobstatsDBTable)) // Add dummy condition at the beginning - q.query(" WHERE id > ") - q.param("0") + q.query(" WHERE id > 0") // Check if logged user is in admin users and if we are quering for "all" jobs // If that is not the case, add user condition in query if !slices.Contains(s.adminUsers, loggedUser) || dashboardUser != "all" { - q.query(" AND Usr = ") - q.param(dashboardUser) + q.query(" AND Usr IN ") + q.param([]string{dashboardUser}) } // Get account query parameters if any if accounts := r.URL.Query()["account"]; len(accounts) > 0 { q.query(" AND Account IN ") - q.multiParam(accounts) + q.param(accounts) } // Check if jobuuid present in query params and add them // If any of jobuuid or jobid query params are present // do not check query window as we are fetching a specific job(s) - // Consequently set defaultQueryWindow to maximum which is retention period if jobuuids := r.URL.Query()["jobuuid"]; len(jobuuids) > 0 { q.query(" AND Jobuuid IN ") - q.multiParam(jobuuids) + q.param(jobuuids) checkQueryWindow = false - defaultQueryWindow = s.dbConfig.RetentionPeriod } // Similarly check for jobid if jobids := r.URL.Query()["jobid"]; len(jobids) > 0 { q.query(" AND Jobid IN ") - q.multiParam(jobids) + q.param(jobids) checkQueryWindow = false - defaultQueryWindow = s.dbConfig.RetentionPeriod } - var from, to string - // If no from provided, use 1 week from now as from - if from = r.URL.Query().Get("from"); from == "" { - from = time.Now().Add(-defaultQueryWindow).Format(time.RFC3339) - } - if to = r.URL.Query().Get("to"); to == "" { - to = time.Now().Format(time.RFC3339) + // If we dont have to specific query window skip next section of code as it becomes + // irrelevant + if !checkQueryWindow { + goto queryJobs } - // Convert "from" and "to" to time.Time and return error response if they - // cannot be parsed - fromTime, err := time.Parse(time.RFC3339, from) - if err != nil { - level.Error(s.logger).Log("msg", "Failed to parse from datestring", "from", from, "err", err) - s.jobsErrorResponse("Internal server error", "Malformed 'from' time string", w) - return + // Get to and from query parameters and do checks on them + if f := r.URL.Query().Get("from"); f == "" { + // If from is not present in query params, use a default query window of 1 week + fromTime = time.Now().Add(-defaultQueryWindow) + } else { + // Return error response if from is not a timestamp + if ts, err := strconv.Atoi(f); err != nil { + level.Error(s.logger).Log("msg", "Failed to parse from timestamp", "from", f, "err", err) + s.jobsErrorResponse("Internal server error", "Malformed 'from' timestamp", w) + return + } else { + fromTime = time.Unix(int64(ts), 0) + } } - toTime, err := time.Parse(time.RFC3339, to) - if err != nil { - level.Error(s.logger).Log("msg", "Failed to parse from datestring", "to", to, "err", err) - s.jobsErrorResponse("Internal server error", "Malformed 'to' time string", w) - return + if t := r.URL.Query().Get("to"); t == "" { + // Use current time as default to + toTime = time.Now() + } else { + // Return error response if to is not a timestamp + if ts, err := strconv.Atoi(t); err != nil { + level.Error(s.logger).Log("msg", "Failed to parse to timestamp", "to", t, "err", err) + s.jobsErrorResponse("Internal server error", "Malformed 'to' timestamp", w) + return + } else { + toTime = time.Unix(int64(ts), 0) + } } - // If difference between from and to is more than 3 months, return with empty + // If difference between from and to is more than 1 months, return with empty // response. This is to prevent users from making "big" requests that can "potentially" // choke server and end up in OOM errors - if toTime.Sub(fromTime) > time.Duration(3*30*24)*time.Hour && checkQueryWindow { + if toTime.Sub(fromTime) > time.Duration(30*24)*time.Hour { level.Error(s.logger).Log( - "msg", "Exceeded maximum query time window of 3 months", "from", from, "to", to, + "msg", "Exceeded maximum query time window of 3 months", + "from", fromTime.Format(time.DateTime), "to", toTime.Format(time.DateTime), "queryWindow", toTime.Sub(fromTime).String(), ) s.jobsErrorResponse("Internal server error", "Maximum query window exceeded", w) return } - // Add from and to to query - if checkQueryWindow { - q.query(" AND Start BETWEEN ") - q.param(from) - q.query(" AND ") - q.param(to) - } + // Add from and to to query only when checkQueryWindow is true + q.query(" AND Start BETWEEN ") + q.param([]string{fromTime.Format(time.DateTime)}) + q.query(" AND ") + q.param([]string{toTime.Format(time.DateTime)}) + +queryJobs: // Get all user jobs in the given time window jobs, err := s.Jobs(q, s.logger) @@ -486,8 +495,9 @@ func fetchJobs( var jobid, jobuuid, partition, qos, account, group, gid, user, uid, - submit, start, end, elapsed, - exitcode, state, + submit, start, end, + submitTs, startTs, endTs, + elapsed, exitcode, state, nnodes, ncpus, nodelist, nodelistExp, jobName, workDir string if err := rows.Scan( @@ -503,6 +513,9 @@ func fetchJobs( &submit, &start, &end, + &submitTs, + &startTs, + &endTs, &elapsed, &exitcode, &state, @@ -528,6 +541,9 @@ func fetchJobs( Submit: submit, Start: start, End: end, + SubmitTS: submitTs, + StartTS: startTs, + EndTS: endTs, Elapsed: elapsed, Exitcode: exitcode, State: state, diff --git a/pkg/jobstats/server/server_test.go b/pkg/jobstats/server/server_test.go index 624f2221..4d35ca49 100644 --- a/pkg/jobstats/server/server_test.go +++ b/pkg/jobstats/server/server_test.go @@ -255,8 +255,8 @@ func TestJobsHandlerWithQueryWindowExceeded(t *testing.T) { req.Header.Set("X-Grafana-User", "foo") // Add from query parameter q := req.URL.Query() - q.Add("from", "2023-01-01T00:00:00Z") - q.Add("to", "2023-06-01T00:00:00Z") + q.Add("from", "1672527600") + q.Add("to", "1685570400") req.URL.RawQuery = q.Encode() // Start recorder @@ -296,8 +296,6 @@ func TestJobsHandlerWithJobuuidsQueryParams(t *testing.T) { req.Header.Set("X-Grafana-User", "foo") // Add from query parameter q := req.URL.Query() - q.Add("from", "2023-01-01T00:00:00Z") - q.Add("to", "2023-06-01T00:00:00Z") q.Add("jobuuid", "foo-bar") req.URL.RawQuery = q.Encode() diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 940c1392..512c75ab 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -155,6 +155,10 @@ get() { fi } +waitport() { + timeout 5 bash -c "while ! curl -s -f "http://localhost:${port}" > /dev/null; do sleep 0.1; done"; +} + if [[ "${scenario}" =~ "exporter" ]] then if [ ! -x ./bin/batchjob_exporter ] @@ -240,7 +244,8 @@ then echo $! > "${pidfile}" - sleep 1 + # sleep 1 + waitport get "127.0.0.1:${port}/metrics" | grep -E -v "${skip_re}" > "${fixture_output}" elif [[ "${scenario}" =~ "stats" ]] @@ -261,26 +266,27 @@ then echo $! > "${pidfile}" - sleep 2 + # sleep 2 + waitport if [ "${scenario}" = "stats-account-query" ] then get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/accounts" > "${fixture_output}" elif [ "${scenario}" = "stats-jobuuid-query" ] then - get -H "X-Grafana-User: usr2" "127.0.0.1:${port}/api/jobs?jobuuid=baee651d-df44-af2c-fa09-50f5523b5e19&account=acc2&from=2023-02-21T00:00:00Z&to=2023-02-21T23:59:59Z" > "${fixture_output}" + get -H "X-Grafana-User: usr2" "127.0.0.1:${port}/api/jobs?jobuuid=baee651d-df44-af2c-fa09-50f5523b5e19&account=acc2" > "${fixture_output}" elif [ "${scenario}" = "stats-jobid-query" ] then - get -H "X-Grafana-User: usr8" "127.0.0.1:${port}/api/jobs?jobid=1479763&account=acc1&from=2023-02-21T00:00:00Z&to=2023-02-22T00:00:00Z" > "${fixture_output}" + get -H "X-Grafana-User: usr8" "127.0.0.1:${port}/api/jobs?jobid=1479763&account=acc1&from=1676934000&to=1677020400" > "${fixture_output}" elif [ "${scenario}" = "stats-jobuuid-jobid-query" ] then - get -H "X-Grafana-User: usr15" "127.0.0.1:${port}/api/jobs?jobuuid=e653f045-73b7-c928-e8df-00c4083cb9bc&jobid=11508&jobid=81510&account=acc1&from=2023-02-21T00:00:00Z&to=2023-02-22T00:00:00Z" > "${fixture_output}" + get -H "X-Grafana-User: usr15" "127.0.0.1:${port}/api/jobs?jobuuid=e653f045-73b7-c928-e8df-00c4083cb9bc&jobid=11508&jobid=81510&account=acc1" > "${fixture_output}" elif [ "${scenario}" = "stats-admin-query" ] then - get -H "X-Grafana-User: grafana" -H "X-Dashboard-User: usr3" "127.0.0.1:${port}/api/jobs?account=acc3&from=2023-01-21T15:49:06Z&to=2023-02-27T15:49:06Z" > "${fixture_output}" + get -H "X-Grafana-User: grafana" -H "X-Dashboard-User: usr3" "127.0.0.1:${port}/api/jobs?account=acc3&from=1676934000&to=1677538800" > "${fixture_output}" elif [ "${scenario}" = "stats-admin-query-all" ] then - get -H "X-Grafana-User: grafana" -H "X-Dashboard-User: all" "127.0.0.1:${port}/api/jobs?from=2023-01-21T15:49:06Z&to=2023-02-27T15:49:06Z" > "${fixture_output}" + get -H "X-Grafana-User: grafana" -H "X-Dashboard-User: all" "127.0.0.1:${port}/api/jobs?from=1676934000&to=1677538800" > "${fixture_output}" fi fi