diff --git a/Makefile b/Makefile index 153fe37d..b6e966dd 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,10 @@ STATICCHECK_IGNORE = CGO_BUILD ?= 0 +# Swagger docs +SWAGGER_DIR ?= pkg/api/http +SWAGGER_MAIN ?= server.go + ifeq ($(GOHOSTOS), linux) test-e2e := test-e2e else @@ -240,7 +244,6 @@ skip-test-docker: .PHONY: promtool promtool: $(PROMTOOL) - $(PROMTOOL): mkdir -p $(FIRST_GOPATH)/bin curl -fsS -L $(PROMTOOL_URL) | tar -xvzf - -C $(FIRST_GOPATH)/bin --strip 1 "prometheus-$(PROMTOOL_VERSION).$(GO_BUILD_PLATFORM)/promtool" diff --git a/Makefile.common b/Makefile.common index 7da873d1..42e1da23 100644 --- a/Makefile.common +++ b/Makefile.common @@ -23,6 +23,7 @@ GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') PROMU := $(FIRST_GOPATH)/bin/promu +SWAG := $(FIRST_GOPATH)/bin/swag pkgs = ./... ifeq (arm, $(GOHOSTARCH)) @@ -181,7 +182,11 @@ common-unused: @git diff --exit-code -- go.sum go.mod .PHONY: common-build -common-build: promu +common-build: promu swag +ifeq ($(CGO_BUILD), 1) + @echo ">> updating swagger docs" + $(SWAG) init -d $(SWAGGER_DIR) -g $(SWAGGER_MAIN) -o $(SWAGGER_DIR)/docs --pd --quiet +endif @echo ">> building binaries" $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) @@ -222,7 +227,6 @@ common-docker-manifest: .PHONY: promu promu: $(PROMU) - $(PROMU): $(eval PROMU_TMP := $(shell mktemp -d)) curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP) @@ -230,6 +234,11 @@ $(PROMU): cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu rm -r $(PROMU_TMP) +.PHONY: swag +swag: $(SWAG) +$(SWAG): + $(GO) install github.com/swaggo/swag/cmd/swag@latest + .PHONY: proto proto: @echo ">> generating code from proto files" diff --git a/pkg/api/cli/cli_test.go b/pkg/api/cli/cli_test.go index afc3c9d7..e8e681fb 100644 --- a/pkg/api/cli/cli_test.go +++ b/pkg/api/cli/cli_test.go @@ -8,17 +8,19 @@ import ( "path/filepath" "testing" "time" + + "github.com/mahendrapaipuri/ceems/pkg/api/base" ) func setCLIArgs() { - os.Args = append(os.Args, "--resource.manager.slurm") - os.Args = append(os.Args, "--slurm.sacct.path=../testdata/sacct") - os.Args = append(os.Args, "--log.level=error") + // os.Args = append(os.Args, "--resource.manager.slurm") + // os.Args = append(os.Args, "--slurm.sacct.path=../testdata/sacct") + os.Args = append(os.Args, "--log.level=debug") } func queryServer(address string) error { client := &http.Client{} - req, _ := http.NewRequest("GET", fmt.Sprintf("http://%s/api/health", address), nil) + req, _ := http.NewRequest("GET", fmt.Sprintf("http://%s/api/%s/health", address, base.APIVersion), nil) req.Header.Set("X-Grafana-User", "usr1") resp, err := client.Do(req) if err != nil { @@ -57,6 +59,8 @@ func TestBatchStatsServerMainNestedDirs(t *testing.T) { for i := 0; i < 10; i++ { if err := queryServer("localhost:9020"); err == nil { break + } else { + fmt.Printf("err %s", err) } time.Sleep(500 * time.Millisecond) if i == 9 { diff --git a/pkg/api/http/middleware_test.go b/pkg/api/http/middleware_test.go index 159f5fe0..e69fb425 100644 --- a/pkg/api/http/middleware_test.go +++ b/pkg/api/http/middleware_test.go @@ -101,8 +101,8 @@ func TestMiddlewareAdminFailure(t *testing.T) { handlerToTest.ServeHTTP(w, req) // Should pass test - if w.Result().StatusCode != 401 { - t.Errorf("expected 401 got %d", w.Result().StatusCode) + if w.Result().StatusCode != 403 { + t.Errorf("expected 403 got %d", w.Result().StatusCode) } } @@ -120,8 +120,8 @@ func TestMiddlewareAdminFailurePresetHeader(t *testing.T) { handlerToTest.ServeHTTP(w, req) // Should pass test - if w.Result().StatusCode != 401 { - t.Errorf("expected 401 got %d", w.Result().StatusCode) + if w.Result().StatusCode != 403 { + t.Errorf("expected 403 got %d", w.Result().StatusCode) } // Should not contain adminHeader if req.Header.Get(adminUserHeader) != "" { diff --git a/pkg/api/testdata/output/e2e-test-api-server-admin--denied-query.txt b/pkg/api/testdata/output/e2e-test-api-server-admin--denied-query.txt index 57c302f8..ccc7aa08 100644 --- a/pkg/api/testdata/output/e2e-test-api-server-admin--denied-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-server-admin--denied-query.txt @@ -1 +1 @@ -{"status":"error","errorType":"unauthorized","error":"current user does not have admin privileges"} +{"status":"error","errorType":"forbidden","error":"current user does not have admin privileges"} diff --git a/pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-denied-query.txt b/pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-denied-query.txt index 57c302f8..ccc7aa08 100644 --- a/pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-denied-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-denied-query.txt @@ -1 +1 @@ -{"status":"error","errorType":"unauthorized","error":"current user does not have admin privileges"} +{"status":"error","errorType":"forbidden","error":"current user does not have admin privileges"} diff --git a/pkg/api/testdata/output/e2e-test-api-verify-fail-query.txt b/pkg/api/testdata/output/e2e-test-api-verify-fail-query.txt index ea766690..9e828293 100644 --- a/pkg/api/testdata/output/e2e-test-api-verify-fail-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-verify-fail-query.txt @@ -1 +1 @@ -{"status":"error","errorType":"unauthorized","error":"user do not have permissions on uuids"} +{"status":"error","errorType":"forbidden","error":"user do not have permissions on uuids"} diff --git a/pkg/api/testdata/output/e2e-test-api-verify-pass-query.txt b/pkg/api/testdata/output/e2e-test-api-verify-pass-query.txt index d63ad864..d7f82459 100644 --- a/pkg/api/testdata/output/e2e-test-api-verify-pass-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-verify-pass-query.txt @@ -1 +1 @@ -{"status":"success","data":{"user":"usr1","uuids":["1479763","1479765"],"verfiy":true}} +{"status":"success","data":[{"user":"usr1","uuids":["1479763","1479765"],"owner":true}]} diff --git a/pkg/lb/frontend/middleware.go b/pkg/lb/frontend/middleware.go index 7e0f2d01..5bc6461d 100644 --- a/pkg/lb/frontend/middleware.go +++ b/pkg/lb/frontend/middleware.go @@ -152,7 +152,7 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler // Write an error and stop the handler chain w.WriteHeader(http.StatusUnauthorized) - response := http_api.Response{ + response := http_api.Response[any]{ Status: "error", ErrorType: "unauthorized", Error: "no user header found", @@ -187,10 +187,10 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler // Check if user is querying for his/her own compute units by looking to DB if !amw.isUserUnit(loggedUser, queryParams.(*QueryParams).uuids) { // Write an error and stop the handler chain - w.WriteHeader(http.StatusUnauthorized) - response := http_api.Response{ + w.WriteHeader(http.StatusForbidden) + response := http_api.Response[any]{ Status: "error", - ErrorType: "unauthorized", + ErrorType: "forbidden", Error: "user do not have permissions to view unit metrics", } if err := json.NewEncoder(w).Encode(&response); err != nil { diff --git a/pkg/lb/frontend/middleware_test.go b/pkg/lb/frontend/middleware_test.go index f9859877..75d59ae2 100644 --- a/pkg/lb/frontend/middleware_test.go +++ b/pkg/lb/frontend/middleware_test.go @@ -136,7 +136,7 @@ func TestMiddlewareWithDB(t *testing.T) { req: "/test?query=foo{uuid=~\"1479765|1481510\"}", user: "usr1", header: true, - code: 401, + code: 403, }, { name: "allow query for admins", @@ -150,7 +150,7 @@ func TestMiddlewareWithDB(t *testing.T) { req: "/test?query=foo{uuid=~\"123|345\"}", user: "usr1", header: true, - code: 401, + code: 403, }, { name: "forbid due to missing header", diff --git a/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-api.txt b/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-api.txt index 75b16ef1..98810b5f 100644 --- a/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-api.txt +++ b/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-api.txt @@ -1 +1 @@ -{"status":"error","errorType":"unauthorized","error":"user do not have permissions to view unit metrics"} +{"status":"error","errorType":"forbidden","error":"user do not have permissions to view unit metrics"} diff --git a/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-db.txt b/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-db.txt index 75b16ef1..98810b5f 100644 --- a/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-db.txt +++ b/pkg/lb/testdata/output/e2e-test-lb-forbid-user-query-db.txt @@ -1 +1 @@ -{"status":"error","errorType":"unauthorized","error":"user do not have permissions to view unit metrics"} +{"status":"error","errorType":"forbidden","error":"user do not have permissions to view unit metrics"} diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 8e2e07b1..c9006792 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -11,6 +11,8 @@ skip_re="^(go_|ceems_exporter_build_info|ceems_scrape_collector_duration_seconds arch="$(uname -m)" +api_version="v1" + scenario="exporter-cgroups-v1"; keep=0; update=0; verbose=0 while getopts 'hs:kuv' opt do @@ -84,67 +86,67 @@ then if [ "${scenario}" = "api-project-query" ] then - desc="/api/projects end point test" + desc="/projects end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-project-query.txt' elif [ "${scenario}" = "api-uuid-query" ] then - desc="/api/units end point test with uuid query param" + desc="/units end point test with uuid query param" fixture='pkg/api/testdata/output/e2e-test-api-server-uuid-query.txt' elif [ "${scenario}" = "api-running-query" ] then - desc="/api/units end point test with running query param" + desc="/units end point test with running query param" fixture='pkg/api/testdata/output/e2e-test-api-server-running-query.txt' elif [ "${scenario}" = "api-admin-query" ] then - desc="/api/units/admin end point test for admin query" + desc="/units/admin end point test for admin query" fixture='pkg/api/testdata/output/e2e-test-api-server-admin-query.txt' elif [ "${scenario}" = "api-admin-query-all" ] then - desc="/api/units/admin end point test for admin query for all jobs" + desc="/units/admin end point test for admin query for all jobs" fixture='pkg/api/testdata/output/e2e-test-api-server-admin-query-all.txt' elif [ "${scenario}" = "api-admin-query-all-selected-fields" ] then - desc="/api/units/admin end point test for admin query for all jobs with selected fields" + desc="/units/admin end point test for admin query for all jobs with selected fields" fixture='pkg/api/testdata/output/e2e-test-api-server-admin-query-all-selected-fields.txt' elif [ "${scenario}" = "api-admin-denied-query" ] then - desc="/api/units/admin end point test for denied request" + desc="/units/admin end point test for denied request" fixture='pkg/api/testdata/output/e2e-test-api-server-admin--denied-query.txt' elif [ "${scenario}" = "api-current-usage-query" ] then - desc="/api/usage/current end point test" + desc="/usage/current end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-current-usage-query.txt' elif [ "${scenario}" = "api-global-usage-query" ] then - desc="/api/usage/global end point test" + desc="/usage/global end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-global-usage-query.txt' elif [ "${scenario}" = "api-current-usage-admin-query" ] then - desc="/api/usage/current/admin end point test" + desc="/usage/current/admin end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-query.txt' elif [ "${scenario}" = "api-global-usage-admin-query" ] then - desc="/api/usage/global/admin end point test" + desc="/usage/global/admin end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-global-usage-admin-query.txt' elif [ "${scenario}" = "api-current-usage-admin-denied-query" ] then - desc="/api/usage/current/admin end point test" + desc="/usage/current/admin end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-denied-query.txt' elif [ "${scenario}" = "api-verify-pass-query" ] then - desc="/api/units/verify end point test with pass request" + desc="/units/verify end point test with pass request" fixture='pkg/api/testdata/output/e2e-test-api-verify-pass-query.txt' elif [ "${scenario}" = "api-verify-fail-query" ] then - desc="/api/units/verify end point test with fail request" + desc="/units/verify end point test with fail request" fixture='pkg/api/testdata/output/e2e-test-api-verify-fail-query.txt' elif [ "${scenario}" = "api-demo-units-query" ] then - desc="/api/units/demo end point test" + desc="/units/demo end point test" fixture='pkg/api/testdata/output/e2e-test-api-demo-units-query.txt' elif [ "${scenario}" = "api-demo-usage-query" ] then - desc="/api/usage/demo end point test" + desc="/usage/demo end point test" fixture='pkg/api/testdata/output/e2e-test-api-demo-usage-query.txt' fi @@ -388,52 +390,52 @@ then if [ "${scenario}" = "api-project-query" ] then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/projects" > "${fixture_output}" + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/projects" > "${fixture_output}" elif [ "${scenario}" = "api-uuid-query" ] then - get -H "X-Grafana-User: usr2" "127.0.0.1:${port}/api/units?uuid=1481508&project=acc2" > "${fixture_output}" + get -H "X-Grafana-User: usr2" "127.0.0.1:${port}/api/${api_version}/units?uuid=1481508&project=acc2" > "${fixture_output}" elif [ "${scenario}" = "api-running-query" ] then - get -H "X-Grafana-User: usr3" "127.0.0.1:${port}/api/units?running&from=1676934000&to=1677538800&field=uuid&field=state&field=started_at&field=allocation&field=tags" > "${fixture_output}" + get -H "X-Grafana-User: usr3" "127.0.0.1:${port}/api/${api_version}/units?running&from=1676934000&to=1677538800&field=uuid&field=state&field=started_at&field=allocation&field=tags" > "${fixture_output}" elif [ "${scenario}" = "api-admin-query" ] then - get -H "X-Grafana-User: grafana" -H "X-Dashboard-User: usr3" "127.0.0.1:${port}/api/units?project=acc3&from=1676934000&to=1677538800" > "${fixture_output}" + get -H "X-Grafana-User: grafana" -H "X-Dashboard-User: usr3" "127.0.0.1:${port}/api/${api_version}/units?project=acc3&from=1676934000&to=1677538800" > "${fixture_output}" elif [ "${scenario}" = "api-admin-query-all" ] then - get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/units/admin?from=1676934000&to=1677538800" > "${fixture_output}" + get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/units/admin?from=1676934000&to=1677538800" > "${fixture_output}" elif [ "${scenario}" = "api-admin-query-all-selected-fields" ] then - get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/units/admin?from=1676934000&to=1677538800&field=uuid&field=started_at&field=ended_at&field=foo" > "${fixture_output}" + get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/units/admin?from=1676934000&to=1677538800&field=uuid&field=started_at&field=ended_at&field=foo" > "${fixture_output}" elif [ "${scenario}" = "api-admin-denied-query" ] then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/units/admin" > "${fixture_output}" + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/units/admin" > "${fixture_output}" elif [ "${scenario}" = "api-current-usage-query" ] then - get -H "X-Grafana-User: usr3" "127.0.0.1:${port}/api/usage/current?from=1676934000&to=1677538800" > "${fixture_output}" + get -H "X-Grafana-User: usr3" "127.0.0.1:${port}/api/${api_version}/usage/current?from=1676934000&to=1677538800" > "${fixture_output}" elif [ "${scenario}" = "api-global-usage-query" ] then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/usage/global?field=usr&field=project&field=num_units" > "${fixture_output}" + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/usage/global?field=usr&field=project&field=num_units" > "${fixture_output}" elif [ "${scenario}" = "api-current-usage-admin-query" ] then - get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/usage/current/admin?user=usr3&from=1676934000&to=1677538800" > "${fixture_output}" + get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/usage/current/admin?user=usr3&from=1676934000&to=1677538800" > "${fixture_output}" elif [ "${scenario}" = "api-global-usage-admin-query" ] then - get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/usage/global/admin?field=usr&field=project&field=num_units" > "${fixture_output}" + get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/usage/global/admin?field=usr&field=project&field=num_units" > "${fixture_output}" elif [ "${scenario}" = "api-current-usage-admin-denied-query" ] then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/usage/global/admin?user=usr2" > "${fixture_output}" + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/usage/global/admin?user=usr2" > "${fixture_output}" elif [ "${scenario}" = "api-verify-pass-query" ] then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/units/verify?uuid=1479763&uuid=1479765" > "${fixture_output}" + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/units/verify?uuid=1479763&uuid=1479765" > "${fixture_output}" elif [ "${scenario}" = "api-verify-fail-query" ] then - get -H "X-Grafana-User: usr2" "127.0.0.1:${port}/api/units/verify?uuid=1479763&uuid=11508" > "${fixture_output}" + get -H "X-Grafana-User: usr2" "127.0.0.1:${port}/api/${api_version}/units/verify?uuid=1479763&uuid=11508" > "${fixture_output}" elif [ "${scenario}" = "api-demo-units-query" ] then - get -s -o /dev/null -w "%{http_code}" "127.0.0.1:${port}/api/units/demo" > "${fixture_output}" + get -s -o /dev/null -w "%{http_code}" "127.0.0.1:${port}/api/${api_version}/units/demo" > "${fixture_output}" elif [ "${scenario}" = "api-demo-usage-query" ] then - get -s -o /dev/null -w "%{http_code}" "127.0.0.1:${port}/api/usage/demo" > "${fixture_output}" + get -s -o /dev/null -w "%{http_code}" "127.0.0.1:${port}/api/${api_version}/usage/demo" > "${fixture_output}" fi elif [[ "${scenario}" =~ ^"lb" ]]