From 62023923afdc084640be5b1870202c3b22f27f45 Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Wed, 24 Jul 2024 09:02:34 +0200 Subject: [PATCH 1/3] perf: Improve custom SQL funcs perf * Avoid unmarshalling empty JSONs * Make cumulative sum over step method and avoid looping twice Signed-off-by: Mahendra Paipuri --- pkg/sqlite3/sqlite3.go | 43 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/pkg/sqlite3/sqlite3.go b/pkg/sqlite3/sqlite3.go index e147f26b..7eb3c52e 100644 --- a/pkg/sqlite3/sqlite3.go +++ b/pkg/sqlite3/sqlite3.go @@ -227,9 +227,7 @@ func avgMetricMap(existing, new string, existingWeight, newWeight float64) strin // For int or float types, they will be summed up // String types will be ignored and treated as zero type sumMetricMap struct { - metricMaps []models.MetricMap aggMetricMap models.MetricMap - n int64 } // newSumMetricMap returns an instance of sumMetricMap @@ -239,22 +237,22 @@ func newSumMetricMap() *sumMetricMap { // Step adds the element to slice func (g *sumMetricMap) Step(m string) { + // On empty map return + if m == "{}" { + return + } + var metricMap models.MetricMap if err := json.Unmarshal([]byte(m), &metricMap); err != nil { panic(err) } - g.metricMaps = append(g.metricMaps, metricMap) - g.n++ + for metricName, metricValue := range metricMap { + g.aggMetricMap[metricName] += metricValue + } } // Done aggregates all the elements added to slice func (g *sumMetricMap) Done() string { - for _, metricMap := range g.metricMaps { - for metricName, metricValue := range metricMap { - g.aggMetricMap[metricName] += metricValue - } - } - // Finally, marshal the type into string and return aggMetricMapBytes, err := json.Marshal(g.aggMetricMap) if err != nil { @@ -267,11 +265,8 @@ func (g *sumMetricMap) Done() string { // For int or float types, they will be weighed averaged // For string types, they will be ignored type avgMetricMapAgg struct { - metricMaps []models.MetricMap - weights []float64 avgMetricMap models.MetricMap totalWeights map[string]models.JSONFloat - n int64 } // newAvgMetricMap returns an instance of avgMetricMap @@ -284,26 +279,24 @@ func newAvgMetricMapAgg() *avgMetricMapAgg { // Step adds the element to slice func (g *avgMetricMapAgg) Step(m string, w float64) { + // On empty map return + if m == "{}" || w == 0 { + return + } + var metricMap models.MetricMap if err := json.Unmarshal([]byte(m), &metricMap); err != nil { panic(err) } - g.metricMaps = append(g.metricMaps, metricMap) - g.weights = append(g.weights, w) - g.n++ + for metricName, metricValue := range metricMap { + weight := models.JSONFloat(w) + g.avgMetricMap[metricName] += metricValue * weight + g.totalWeights[metricName] += weight + } } // Done aggregates all the elements added to slice func (g *avgMetricMapAgg) Done() string { - // Get weighed sum of all metric maps - for imetricMap, metricMap := range g.metricMaps { - for metricName, metricValue := range metricMap { - weight := models.JSONFloat(g.weights[imetricMap]) - g.avgMetricMap[metricName] += metricValue * weight - g.totalWeights[metricName] += weight - } - } - // Divide weighted sum by counts to get weighted average for metricName := range g.avgMetricMap { if totalWeight, ok := g.totalWeights[metricName]; ok { From 76ee5acd3e8aaf5a02c4883422f8be81e8443134 Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Wed, 24 Jul 2024 09:03:57 +0200 Subject: [PATCH 2/3] perf: Attempt to improve current usage query perf * Set index on columns that are used in current usage query * Improve row scanning logic slightly to gain perf * Misc fixes and typo corrections Signed-off-by: Mahendra Paipuri --- internal/structset/structset.go | 36 +++---- pkg/api/cli/cli_test.go | 8 ++ .../000006_create_index_units_table.down.sql | 1 + .../000006_create_index_units_table.up.sql | 1 + pkg/api/http/docs/docs.go | 14 ++- pkg/api/http/docs/swagger.json | 14 ++- pkg/api/http/docs/swagger.yaml | 11 ++- pkg/api/http/querier.go | 33 +++++-- pkg/api/http/server.go | 73 +++++++++++---- pkg/api/http/server_test.go | 93 ++++++++++++++++--- 10 files changed, 217 insertions(+), 67 deletions(-) create mode 100644 pkg/api/db/migrations/000006_create_index_units_table.down.sql create mode 100644 pkg/api/db/migrations/000006_create_index_units_table.up.sql diff --git a/internal/structset/structset.go b/internal/structset/structset.go index 3ed7d826..07685ae3 100644 --- a/internal/structset/structset.go +++ b/internal/structset/structset.go @@ -3,8 +3,6 @@ package structset import ( "database/sql" - "errors" - "fmt" "reflect" "strings" "sync" @@ -81,34 +79,24 @@ func GetStructFieldTagMap(Struct interface{}, keyTag string, valueTag string) ma // ScanRow is a cut-down version of the proposed Rows.ScanRow method. It // currently only handles dest being a (pointer to) struct, and does not // handle embedded fields. See https://github.com/golang/go/issues/61637 -func ScanRow(rows *sql.Rows, dest any) error { - rv := reflect.ValueOf(dest) - if rv.Kind() != reflect.Pointer || rv.IsNil() { - return errors.New("dest must be a non-nil pointer") - } +func ScanRow(rows *sql.Rows, columns []string, indexes map[string]int, dest any) error { + // elem := reflect.ValueOf(dest).Elem() + // if rv.Kind() != reflect.Pointer || rv.IsNil() { + // return errors.New("dest must be a non-nil pointer") + // } - elem := rv.Elem() - if elem.Kind() != reflect.Struct { - return errors.New("dest must point to a struct") - } - indexes := cachedFieldIndexes(reflect.TypeOf(dest).Elem()) - - columns, err := rows.Columns() - if err != nil { - return fmt.Errorf("cannot fetch columns: %w", err) - } + // elem := rv.Elem() + // if elem.Kind() != reflect.Struct { + // return errors.New("dest must point to a struct") + // } var scanArgs []any for _, column := range columns { index, ok := indexes[column] if ok { // We have a column to field mapping, scan the value. - field := elem.Field(index) + field := reflect.ValueOf(dest).Elem().Field(index) scanArgs = append(scanArgs, field.Addr().Interface()) - } else { - // Unassigned column, throw away the scanned value. - var throwAway any - scanArgs = append(scanArgs, &throwAway) } } return rows.Scan(scanArgs...) @@ -131,8 +119,8 @@ func fieldIndexes(structType reflect.Type) map[string]int { return indexes } -// cachedFieldIndexes is like fieldIndexes, but cached per struct type. -func cachedFieldIndexes(structType reflect.Type) map[string]int { +// CachedFieldIndexes is like fieldIndexes, but cached per struct type. +func CachedFieldIndexes(structType reflect.Type) map[string]int { if f, ok := fieldIndexesCache.Load(structType); ok { return f.(map[string]int) } diff --git a/pkg/api/cli/cli_test.go b/pkg/api/cli/cli_test.go index eaccc6ed..fe1f7d23 100644 --- a/pkg/api/cli/cli_test.go +++ b/pkg/api/cli/cli_test.go @@ -106,6 +106,14 @@ ceems_api_server: configFile := fmt.Sprintf(configFileTmpl, dataDir) configFilePath := makeConfigFile(configFile, tmpDir) + // Create sample DB file + os.MkdirAll(dataDir, os.ModePerm) + f, err := os.Create(filepath.Join(dataDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + f.Close() + // Remove test related args os.Args = append([]string{os.Args[0]}, fmt.Sprintf("--config.file=%s", configFilePath)) os.Args = append(os.Args, "--log.level=debug") diff --git a/pkg/api/db/migrations/000006_create_index_units_table.down.sql b/pkg/api/db/migrations/000006_create_index_units_table.down.sql new file mode 100644 index 00000000..3d6b2c79 --- /dev/null +++ b/pkg/api/db/migrations/000006_create_index_units_table.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_cluster_id_project_user_ended; diff --git a/pkg/api/db/migrations/000006_create_index_units_table.up.sql b/pkg/api/db/migrations/000006_create_index_units_table.up.sql new file mode 100644 index 00000000..0289a774 --- /dev/null +++ b/pkg/api/db/migrations/000006_create_index_units_table.up.sql @@ -0,0 +1 @@ +CREATE INDEX idx_cluster_id_project_user_ended ON units (cluster_id,project,username,ended_at); diff --git a/pkg/api/http/docs/docs.go b/pkg/api/http/docs/docs.go index 09a288db..079a6c53 100644 --- a/pkg/api/http/docs/docs.go +++ b/pkg/api/http/docs/docs.go @@ -214,7 +214,7 @@ const docTemplate = `{ "tags": [ "projects" ], - "summary": "Admin ednpoint to fetch project details", + "summary": "Admin endpoint to fetch project details", "parameters": [ { "type": "string", @@ -706,11 +706,21 @@ const docTemplate = `{ "items": { "type": "string" }, - "collectionFormat": "multi", + "collectionFormat": "csv", "description": "Project", "name": "project", "in": "query" }, + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "Username", + "name": "user", + "in": "query" + }, { "type": "string", "description": "From timestamp", diff --git a/pkg/api/http/docs/swagger.json b/pkg/api/http/docs/swagger.json index a2e3472a..af86eb35 100644 --- a/pkg/api/http/docs/swagger.json +++ b/pkg/api/http/docs/swagger.json @@ -206,7 +206,7 @@ "tags": [ "projects" ], - "summary": "Admin ednpoint to fetch project details", + "summary": "Admin endpoint to fetch project details", "parameters": [ { "type": "string", @@ -698,11 +698,21 @@ "items": { "type": "string" }, - "collectionFormat": "multi", + "collectionFormat": "csv", "description": "Project", "name": "project", "in": "query" }, + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "multi", + "description": "Username", + "name": "user", + "in": "query" + }, { "type": "string", "description": "From timestamp", diff --git a/pkg/api/http/docs/swagger.yaml b/pkg/api/http/docs/swagger.yaml index 8a9a39e1..2895dc10 100644 --- a/pkg/api/http/docs/swagger.yaml +++ b/pkg/api/http/docs/swagger.yaml @@ -608,7 +608,7 @@ paths: $ref: '#/definitions/http.Response-any' security: - BasicAuth: [] - summary: Admin ednpoint to fetch project details + summary: Admin endpoint to fetch project details tags: - projects /units: @@ -995,13 +995,20 @@ paths: type: string name: cluster_id type: array - - collectionFormat: multi + - collectionFormat: csv description: Project in: query items: type: string name: project type: array + - collectionFormat: multi + description: Username + in: query + items: + type: string + name: user + type: array - description: From timestamp in: query name: from diff --git a/pkg/api/http/querier.go b/pkg/api/http/querier.go index 5b544c63..53a547c0 100644 --- a/pkg/api/http/querier.go +++ b/pkg/api/http/querier.go @@ -3,6 +3,7 @@ package http import ( "database/sql" "fmt" + "reflect" "regexp" "strings" @@ -76,13 +77,26 @@ func projectsSubQuery(users []string) Query { // Ref: https://oilbeater.com/en/2024/03/04/golang-slice-performance/ // For the rest of queries, they should return fewer rows and hence, we can live with // dynamic allocation -func scanRows[T any](rows *sql.Rows, numRows int) []T { +func scanRows[T any](rows *sql.Rows, numRows int) ([]T, error) { + var columns []string var values = make([]T, numRows) var value T + var err error + scanErrs := 0 rowIdx := 0 + + // Get indexes + indexes := structset.CachedFieldIndexes(reflect.TypeOf(&value).Elem()) + + // Get columns + if columns, err = rows.Columns(); err != nil { + return nil, fmt.Errorf("cannot fetch columns: %w", err) + } + + // Scan each row for rows.Next() { - if err := structset.ScanRow(rows, &value); err != nil { - continue + if err := structset.ScanRow(rows, columns, indexes, &value); err != nil { + scanErrs++ } if numRows > 0 { values[rowIdx] = value @@ -91,7 +105,13 @@ func scanRows[T any](rows *sql.Rows, numRows int) []T { } rowIdx++ } - return values + + // If we failed to scan any rows, return error which will be included in warnings + // in the response + if scanErrs > 0 { + err = fmt.Errorf("failed to scan %d rows", scanErrs) + } + return values, err } func countRows(dbConn *sql.DB, query Query) (int, error) { @@ -181,10 +201,9 @@ func Querier[T any](dbConn *sql.DB, query Query, logger log.Logger) ([]T, error) defer rows.Close() // Loop through rows, using Scan to assign column data to struct fields. - var values = scanRows[T](rows, numRows) level.Debug(logger).Log( "msg", "Rows", "query", queryString, "queryParams", strings.Join(queryParams, ","), - "num_rows", numRows, + "num_rows", numRows, "error", err, ) - return values, nil + return scanRows[T](rows, numRows) } diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index 37c4feae..9a8d5d03 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -101,7 +101,11 @@ func init() { // Use SQL aggregate functions in query for _, col := range base.UsageDBTableColNames { if strings.HasPrefix(col, "num") { - aggUsageDBCols[col] = "COUNT(id) AS num_units" + if col == "num_units" { + aggUsageDBCols[col] = "COUNT(id) AS num_units" + } else { + aggUsageDBCols[col] = "SUM(num_updates) AS num_updates" + } } else if strings.HasPrefix(col, "total") { aggUsageDBCols[col] = fmt.Sprintf("sum_metric_map_agg(%[1]s) AS %[1]s", col) } else if strings.HasPrefix(col, "avg") { @@ -127,7 +131,6 @@ func getDBStatus(dbConn *sql.DB, logger log.Logger) bool { // NewCEEMSServer creates new CEEMSServer struct instance func NewCEEMSServer(c *Config) (*CEEMSServer, func(), error) { - var dbConn *sql.DB var err error router := mux.NewRouter() @@ -137,7 +140,7 @@ func NewCEEMSServer(c *Config) (*CEEMSServer, func(), error) { Addr: c.Web.Address, Handler: router, ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + WriteTimeout: 300 * time.Second, ReadHeaderTimeout: 2 * time.Second, // slowloris attack: https://app.deepsource.com/directory/analyzers/go/issues/GO-S2112 }, webConfig: &web.FlagConfig{ @@ -219,10 +222,29 @@ func NewCEEMSServer(c *Config) (*CEEMSServer, func(), error) { )).Methods(http.MethodGet) // Open DB connection - if dbConn, err = sql.Open(sqlite3.DriverName, filepath.Join(c.DB.Data.Path, base.CEEMSDBName)); err != nil { - return nil, func() {}, err + dsn := fmt.Sprintf("file:%s?%s", filepath.Join(c.DB.Data.Path, base.CEEMSDBName), "_mutex=no&mode=ro") + if server.db, err = sql.Open(sqlite3.DriverName, dsn); err != nil { + return nil, func() {}, fmt.Errorf("failed to open DB: %s", err) } - server.db = dbConn + + // // Ping DB to establish first connection in the pool + // // Make repetitive connection attempts until timeout as DB might be created with a + // // latency + // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + // defer cancel() + // ticker := time.NewTicker(time.Second) + // defer ticker.Stop() + // for loop := true; loop; { + // err := server.db.Ping() + // select { + // case <-ctx.Done(): + // return nil, func() {}, fmt.Errorf("failed to ping DB: %s", err) + // case <-ticker.C: + // if err == nil { + // loop = false + // } + // } + // } // Add a middleware that verifies headers and pass them in requests // The middleware will fetch admin users from Grafana periodically to update list @@ -230,7 +252,7 @@ func NewCEEMSServer(c *Config) (*CEEMSServer, func(), error) { logger: c.Logger, routerPrefix: routePrefix, whitelistedURLs: regexp.MustCompile(fmt.Sprintf("%s(swagger|health|demo)(.*)", routePrefix)), - db: dbConn, + db: server.db, adminUsers: adminUsers, } router.Use(amw.Middleware) @@ -271,7 +293,7 @@ func (s *CEEMSServer) Start() error { level.Info(s.logger).Log("msg", fmt.Sprintf("Starting %s", base.CEEMSServerAppName)) if err := web.ListenAndServe(s.server, s.webConfig, s.logger); err != nil && err != http.ErrServerClosed { - level.Error(s.logger).Log("msg", "Failed to Listen and Server HTTP server", "err", err) + level.Error(s.logger).Log("msg", "Failed to Listen and Serve HTTP server", "err", err) return err } return nil @@ -495,7 +517,7 @@ queryUnits: // Get all user units in the given time window units, err := s.queriers.unit(s.db, q, s.logger) - if err != nil { + if units == nil && err != nil { level.Error(s.logger).Log("msg", "Failed to fetch units", "loggedUser", loggedUser, "err", err) errorResponse[any](w, &apiError{errorInternal, err}, s.logger, nil) return @@ -507,6 +529,9 @@ queryUnits: Status: "success", Data: units, } + if err != nil { + response.Warnings = append(response.Warnings, err.Error()) + } if err = json.NewEncoder(w).Encode(&response); err != nil { level.Error(s.logger).Log("msg", "Failed to encode response", "err", err) w.Write([]byte("KO")) @@ -727,7 +752,7 @@ func (s *CEEMSServer) clustersAdmin(w http.ResponseWriter, r *http.Request) { // Make query and get list of cluster ids clusterIDs, err := s.queriers.cluster(s.db, q, s.logger) - if err != nil { + if clusterIDs == nil && err != nil { level.Error(s.logger).Log("msg", "Failed to fetch cluster IDs", "user", dashboardUser, "err", err) errorResponse[any](w, &apiError{errorInternal, err}, s.logger, nil) return @@ -739,6 +764,9 @@ func (s *CEEMSServer) clustersAdmin(w http.ResponseWriter, r *http.Request) { Status: "success", Data: clusterIDs, } + if err != nil { + clusterIDsResponse.Warnings = append(clusterIDsResponse.Warnings, err.Error()) + } if err = json.NewEncoder(w).Encode(&clusterIDsResponse); err != nil { level.Error(s.logger).Log("msg", "Failed to encode response", "err", err) w.Write([]byte("KO")) @@ -773,7 +801,7 @@ func (s *CEEMSServer) usersQuerier(users []string, w http.ResponseWriter, r *htt // Make query and check for users returned in usage userModels, err := s.queriers.user(s.db, q, s.logger) - if err != nil { + if userModels == nil && err != nil { level.Error(s.logger).Log("msg", "Failed to fetch user details", "users", strings.Join(users, ","), "err", err) errorResponse[any](w, &apiError{errorInternal, err}, s.logger, nil) return @@ -785,6 +813,9 @@ func (s *CEEMSServer) usersQuerier(users []string, w http.ResponseWriter, r *htt Status: "success", Data: userModels, } + if err != nil { + usersResponse.Warnings = append(usersResponse.Warnings, err.Error()) + } if err = json.NewEncoder(w).Encode(&usersResponse); err != nil { level.Error(s.logger).Log("msg", "Failed to encode response", "err", err) w.Write([]byte("KO")) @@ -892,7 +923,7 @@ func (s *CEEMSServer) projectsQuerier(users []string, w http.ResponseWriter, r * // Make query projectModels, err := s.queriers.project(s.db, q, s.logger) - if err != nil { + if projectModels == nil && err != nil { level.Error(s.logger).Log( "msg", "Failed to fetch project details", "users", strings.Join(users, ","), "err", err, @@ -907,6 +938,9 @@ func (s *CEEMSServer) projectsQuerier(users []string, w http.ResponseWriter, r * Status: "success", Data: projectModels, } + if err != nil { + projectsResponse.Warnings = append(projectsResponse.Warnings, err.Error()) + } if err = json.NewEncoder(w).Encode(&projectsResponse); err != nil { level.Error(s.logger).Log("msg", "Failed to encode response", "err", err) w.Write([]byte("KO")) @@ -950,7 +984,7 @@ func (s *CEEMSServer) projects(w http.ResponseWriter, r *http.Request) { // projectsAdmin godoc // -// @Summary Admin ednpoint to fetch project details +// @Summary Admin endpoint to fetch project details // @Description This endpoint will show details of the queried project. The // @Description current user is always identified by the header `X-Grafana-User` in // @Description the request. @@ -1040,7 +1074,7 @@ func (s *CEEMSServer) currentUsage(users []string, fields []string, w http.Respo // Make query and check for returned number of rows usage, err := s.queriers.usage(s.db, q, s.logger) - if err != nil { + if usage == nil && err != nil { level.Error(s.logger). Log("msg", "Failed to fetch current usage statistics", "users", strings.Join(users, ","), "err", err) errorResponse[any](w, &apiError{errorInternal, err}, s.logger, nil) @@ -1053,6 +1087,9 @@ func (s *CEEMSServer) currentUsage(users []string, fields []string, w http.Respo Status: "success", Data: usage, } + if err != nil { + projectsResponse.Warnings = append(projectsResponse.Warnings, err.Error()) + } if err = json.NewEncoder(w).Encode(&projectsResponse); err != nil { level.Error(s.logger).Log("msg", "Failed to encode response", "err", err) w.Write([]byte("KO")) @@ -1081,7 +1118,7 @@ func (s *CEEMSServer) globalUsage(users []string, queriedFields []string, w http // Make query and check for returned number of rows usage, err := s.queriers.usage(s.db, q, s.logger) - if err != nil { + if usage == nil && err != nil { level.Error(s.logger). Log("msg", "Failed to fetch global usage statistics", "users", strings.Join(users, ","), "err", err) errorResponse[any](w, &apiError{errorInternal, err}, s.logger, nil) @@ -1094,6 +1131,9 @@ func (s *CEEMSServer) globalUsage(users []string, queriedFields []string, w http Status: "success", Data: usage, } + if err != nil { + projectsResponse.Warnings = append(projectsResponse.Warnings, err.Error()) + } if err = json.NewEncoder(w).Encode(&projectsResponse); err != nil { level.Error(s.logger).Log("msg", "Failed to encode response", "err", err) w.Write([]byte("KO")) @@ -1211,7 +1251,8 @@ func (s *CEEMSServer) usage(w http.ResponseWriter, r *http.Request) { // @Param X-Grafana-User header string true "Current user name" // @Param mode path string true "Whether to get usage stats within a period or global" Enums(current, global) // @Param cluster_id query []string false "cluster ID" collectionFormat(multi) -// @Param project query []string false "Project" collectionFormat(multi) +// @Param project query []string false "Project" +// @Param user query []string false "Username" collectionFormat(multi) // @Param from query string false "From timestamp" // @Param to query string false "To timestamp" // @Param field query []string false "Fields to return in response" collectionFormat(multi) diff --git a/pkg/api/http/server_test.go b/pkg/api/http/server_test.go index 27b22cd0..d9d2cbe9 100644 --- a/pkg/api/http/server_test.go +++ b/pkg/api/http/server_test.go @@ -4,10 +4,13 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "strings" "testing" "time" @@ -15,6 +18,7 @@ import ( "github.com/go-kit/log" "github.com/gorilla/mux" "github.com/mahendrapaipuri/ceems/pkg/api/base" + "github.com/mahendrapaipuri/ceems/pkg/api/db" "github.com/mahendrapaipuri/ceems/pkg/api/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -50,11 +54,12 @@ var ( {ID: "slurm-0", Manager: "slurm"}, {ID: "os-0", Manager: "openstack"}, } + errTest = fmt.Errorf("failed to query 10 rows") ) -func setupServer() *CEEMSServer { +func setupServer(d string) *CEEMSServer { logger := log.NewNopLogger() - server, _, _ := NewCEEMSServer(&Config{Logger: logger}) + server, _, _ := NewCEEMSServer(&Config{Logger: logger, DB: db.Config{Data: db.DataConfig{Path: d}}}) server.maxQueryPeriod = time.Duration(time.Hour * 168) server.queriers = queriers{ unit: unitQuerier, @@ -75,7 +80,7 @@ func usageQuerier(db *sql.DB, q Query, logger log.Logger) ([]models.Usage, error } func projectQuerier(db *sql.DB, q Query, logger log.Logger) ([]models.Project, error) { - return mockServerProjects, nil + return mockServerProjects, errTest } func userQuerier(db *sql.DB, q Query, logger log.Logger) ([]models.User, error) { @@ -95,7 +100,13 @@ func getMockUnits( // Test users and users admin handlers func TestUsersHandlers(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Test cases @@ -148,7 +159,13 @@ func TestUsersHandlers(t *testing.T) { // Test projects and projects admin handlers func TestProjectsHandler(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Test cases @@ -196,12 +213,19 @@ func TestProjectsHandler(t *testing.T) { assert.Equal(t, w.Code, test.code) assert.Equal(t, response.Status, "success") assert.Equal(t, response.Data, mockServerProjects) + assert.Equal(t, response.Warnings, []string{errTest.Error()}) } } // Test units and units admin handlers func TestUnitsHandler(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Test cases @@ -254,7 +278,13 @@ func TestUnitsHandler(t *testing.T) { // Test usage and usage admin handlers func TestUsageHandlers(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Test cases @@ -328,7 +358,13 @@ func TestUsageHandlers(t *testing.T) { // Test verify handler func TestVerifyHandler(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) tests := []testCase{ @@ -366,7 +402,13 @@ func TestVerifyHandler(t *testing.T) { // Test demo handlers func TestDemoHandlers(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Test cases @@ -410,7 +452,13 @@ func TestDemoHandlers(t *testing.T) { // Test clusters handlers func TestClustersHandler(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Create request @@ -442,7 +490,13 @@ func TestClustersHandler(t *testing.T) { // Test /units when from/to query parameters are malformed func TestUnitsHandlerWithMalformedQueryParams(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Create request @@ -475,9 +529,14 @@ func TestUnitsHandlerWithMalformedQueryParams(t *testing.T) { // Test /units when from/to query parameters exceed max time window func TestUnitsHandlerWithQueryWindowExceeded(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) - // Create request req := httptest.NewRequest(http.MethodGet, "/api/v1/units", nil) // Add user header @@ -510,7 +569,13 @@ func TestUnitsHandlerWithQueryWindowExceeded(t *testing.T) { // Test /units when from/to query parameters exceed max time window but when unit uuids // are present func TestUnitsHandlerWithUnituuidsQueryParams(t *testing.T) { - server := setupServer() + tmpDir := t.TempDir() + f, err := os.Create(filepath.Join(tmpDir, base.CEEMSDBName)) + if err != nil { + require.NoError(t, err) + } + defer f.Close() + server := setupServer(tmpDir) defer server.Shutdown(context.Background()) // Create request From 4d6482dd0c564fb78f2e92ed3fb8d2b8e272749e Mon Sep 17 00:00:00 2001 From: Mahendra Paipuri Date: Wed, 24 Jul 2024 09:27:19 +0200 Subject: [PATCH 3/3] test: Update e2e test outputs Signed-off-by: Mahendra Paipuri --- .../testdata/output/e2e-test-api-server-current-usage-query.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/testdata/output/e2e-test-api-server-current-usage-query.txt b/pkg/api/testdata/output/e2e-test-api-server-current-usage-query.txt index b9415a2f..3a190a84 100644 --- a/pkg/api/testdata/output/e2e-test-api-server-current-usage-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-server-current-usage-query.txt @@ -1 +1 @@ -{"status":"success","data":[{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":2,"project":"acc1","group":"grp15","user":"usr15","total_time_seconds":{"alloc_cpumemtime":325713920,"alloc_cputime":15904,"alloc_gpumemtime":994,"alloc_gputime":7952,"walltime":994},"avg_cpu_usage":{"global":18.18507953},"avg_cpu_mem_usage":{"global":18.18507953},"total_cpu_energy_usage_kwh":{"total":36.37015906},"total_cpu_emissions_gms":{"emaps_total":36.37015906,"rte_total":36.37015906},"avg_gpu_usage":{"global":18.18507953},"avg_gpu_mem_usage":{"global":18.18507953},"total_gpu_energy_usage_kwh":{"total":36.37015906},"total_gpu_emissions_gms":{"emaps_total":36.37015906,"rte_total":36.37015906}},{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":1,"project":"acc2","group":"grp2","user":"usr2","total_time_seconds":{"alloc_cpumemtime":162856960,"alloc_cputime":7952,"alloc_gpumemtime":0,"alloc_gputime":0,"walltime":497},"avg_cpu_usage":{"global":53.47701540},"avg_cpu_mem_usage":{"global":53.47701540},"total_cpu_energy_usage_kwh":{"total":53.47701540},"total_cpu_emissions_gms":{"emaps_total":53.47701540,"rte_total":53.47701540},"avg_gpu_usage":{"global":0},"avg_gpu_mem_usage":{"global":0},"total_gpu_energy_usage_kwh":{"total":53.47701540},"total_gpu_emissions_gms":{"emaps_total":53.47701540,"rte_total":53.47701540}},{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":1,"project":"acc1","group":"grp8","user":"usr8","total_time_seconds":{"alloc_cpumemtime":970588160,"alloc_cputime":23696,"alloc_gpumemtime":2962,"alloc_gputime":23696,"walltime":2962},"avg_cpu_usage":{"global":20.21483680},"avg_cpu_mem_usage":{"global":20.21483680},"total_cpu_energy_usage_kwh":{"total":20.21483680},"total_cpu_emissions_gms":{"emaps_total":20.21483680,"rte_total":20.21483680},"avg_gpu_usage":{"global":20.21483680},"avg_gpu_mem_usage":{"global":20.21483680},"total_gpu_energy_usage_kwh":{"total":20.21483680},"total_gpu_emissions_gms":{"emaps_total":20.21483680,"rte_total":20.21483680}}]} +{"status":"success","data":[{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":2,"project":"acc1","group":"grp15","user":"usr15","total_time_seconds":{"alloc_cpumemtime":325713920,"alloc_cputime":15904,"alloc_gpumemtime":994,"alloc_gputime":7952,"walltime":994},"avg_cpu_usage":{"global":18.18507953},"avg_cpu_mem_usage":{"global":18.18507953},"total_cpu_energy_usage_kwh":{"total":36.37015906},"total_cpu_emissions_gms":{"emaps_total":36.37015906,"rte_total":36.37015906},"avg_gpu_usage":{"global":18.18507953},"avg_gpu_mem_usage":{"global":18.18507953},"total_gpu_energy_usage_kwh":{"total":36.37015906},"total_gpu_emissions_gms":{"emaps_total":36.37015906,"rte_total":36.37015906}},{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":1,"project":"acc2","group":"grp2","user":"usr2","total_time_seconds":{"alloc_cpumemtime":162856960,"alloc_cputime":7952,"alloc_gpumemtime":0,"alloc_gputime":0,"walltime":497},"avg_cpu_usage":{"global":53.47701540},"avg_cpu_mem_usage":{"global":53.47701540},"total_cpu_energy_usage_kwh":{"total":53.47701540},"total_cpu_emissions_gms":{"emaps_total":53.47701540,"rte_total":53.47701540},"total_gpu_energy_usage_kwh":{"total":53.47701540},"total_gpu_emissions_gms":{"emaps_total":53.47701540,"rte_total":53.47701540}},{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":1,"project":"acc1","group":"grp8","user":"usr8","total_time_seconds":{"alloc_cpumemtime":970588160,"alloc_cputime":23696,"alloc_gpumemtime":2962,"alloc_gputime":23696,"walltime":2962},"avg_cpu_usage":{"global":20.21483680},"avg_cpu_mem_usage":{"global":20.21483680},"total_cpu_energy_usage_kwh":{"total":20.21483680},"total_cpu_emissions_gms":{"emaps_total":20.21483680,"rte_total":20.21483680},"avg_gpu_usage":{"global":20.21483680},"avg_gpu_mem_usage":{"global":20.21483680},"total_gpu_energy_usage_kwh":{"total":20.21483680},"total_gpu_emissions_gms":{"emaps_total":20.21483680,"rte_total":20.21483680}}]}