diff --git a/pkg/lb/cli/cli_test.go b/pkg/lb/cli/cli_test.go index c369bcb..24b1797 100644 --- a/pkg/lb/cli/cli_test.go +++ b/pkg/lb/cli/cli_test.go @@ -22,13 +22,14 @@ var mockCEEMSLBApp = *kingpin.New( "Mock Load Balancer App.", ) -func queryLB(address string) error { +func queryLB(address, clusterID string) error { req, err := http.NewRequest(http.MethodGet, "http://"+address, nil) //nolint:noctx if err != nil { return err } req.Header.Add("X-Grafana-User", "usr1") + req.Header.Add("X-Ceems-Cluster-Id", clusterID) client := &http.Client{Timeout: 10 * time.Second} @@ -96,7 +97,7 @@ ceems_lb: // Query LB for i := range 10 { - if err := queryLB("localhost:9030/default"); err == nil { + if err := queryLB("localhost:9030", "default"); err == nil { break } diff --git a/pkg/lb/frontend/frontend.go b/pkg/lb/frontend/frontend.go index 080667b..779ef1b 100644 --- a/pkg/lb/frontend/frontend.go +++ b/pkg/lb/frontend/frontend.go @@ -42,7 +42,7 @@ type QueryParamsContextKey struct{} // QueryParams is the context value. type QueryParams struct { - id string + clusterID string uuids []string queryPeriod time.Duration } @@ -315,7 +315,7 @@ func (lb *loadBalancer) Serve(w http.ResponseWriter, r *http.Request) { if v, ok := queryParams.(*QueryParams); ok { queryPeriod = v.queryPeriod - id = v.id + id = v.clusterID } else { http.Error(w, "Invalid query parameters", http.StatusBadRequest) diff --git a/pkg/lb/frontend/frontend_test.go b/pkg/lb/frontend/frontend_test.go index 533cdef..0c123fd 100644 --- a/pkg/lb/frontend/frontend_test.go +++ b/pkg/lb/frontend/frontend_test.go @@ -140,7 +140,7 @@ func TestNewFrontendSingleGroup(t *testing.T) { newReq = request.WithContext( context.WithValue( request.Context(), QueryParamsContextKey{}, - &QueryParams{queryPeriod: period, id: clusterID}, + &QueryParams{queryPeriod: period, clusterID: clusterID}, ), ) } else { @@ -164,7 +164,7 @@ func TestNewFrontendSingleGroup(t *testing.T) { newReq := request.WithContext( context.WithValue( request.Context(), QueryParamsContextKey{}, - &QueryParams{id: "default"}, + &QueryParams{clusterID: "default"}, ), ) responseRecorder := httptest.NewRecorder() @@ -260,7 +260,7 @@ func TestNewFrontendTwoGroups(t *testing.T) { newReq = request.WithContext( context.WithValue( request.Context(), QueryParamsContextKey{}, - &QueryParams{queryPeriod: period, id: test.clusterID}, + &QueryParams{queryPeriod: period, clusterID: test.clusterID}, ), ) } else { @@ -284,7 +284,7 @@ func TestNewFrontendTwoGroups(t *testing.T) { newReq := request.WithContext( context.WithValue( request.Context(), QueryParamsContextKey{}, - &QueryParams{id: "rm-0"}, + &QueryParams{clusterID: "rm-0"}, ), ) responseRecorder := httptest.NewRecorder() diff --git a/pkg/lb/frontend/helpers.go b/pkg/lb/frontend/helpers.go index 98a66fd..f44b67a 100644 --- a/pkg/lb/frontend/helpers.go +++ b/pkg/lb/frontend/helpers.go @@ -70,10 +70,10 @@ func setQueryParams(r *http.Request, queryParams *QueryParams) *http.Request { } // Parse query in the request after cloning it and add query params to context. -func parseQueryParams(r *http.Request, rmIDs []string, logger log.Logger) *http.Request { +func parseQueryParams(r *http.Request, logger log.Logger) *http.Request { var body []byte - var id string + var clusterID string var uuids []string @@ -81,48 +81,51 @@ func parseQueryParams(r *http.Request, rmIDs []string, logger log.Logger) *http. var err error - // Get id from path parameter. - // Requested paths will be of form /{id}/. Here will strip `id` - // part and proxy the rest to backend - var pathParts []string - - for _, p := range strings.Split(r.URL.Path, "/") { - if strings.TrimSpace(p) == "" { - continue - } - - pathParts = append(pathParts, p) - } - - // First path part must be resource manager ID and check if it is in the valid IDs - if len(pathParts) > 0 { - if slices.Contains(rmIDs, pathParts[0]) { - id = pathParts[0] - - // If there is more than 1 pathParts, make URL or set / as URL - if len(pathParts) > 1 { - r.URL.Path = "/" + strings.Join(pathParts[1:], "/") - r.RequestURI = r.URL.Path - } else { - r.URL.Path = "/" - r.RequestURI = "/" - } - } - } + // Get cluster id from X-Ceems-Cluster-Id header + clusterID = r.Header.Get(ceemsClusterIDHeader) + + // // Get id from path parameter. + // // Requested paths will be of form /{id}/. Here will strip `id` + // // part and proxy the rest to backend + // var pathParts []string + + // for _, p := range strings.Split(r.URL.Path, "/") { + // if strings.TrimSpace(p) == "" { + // continue + // } + + // pathParts = append(pathParts, p) + // } + + // // First path part must be resource manager ID and check if it is in the valid IDs + // if len(pathParts) > 0 { + // if slices.Contains(rmIDs, pathParts[0]) { + // id = pathParts[0] + + // // If there is more than 1 pathParts, make URL or set / as URL + // if len(pathParts) > 1 { + // r.URL.Path = "/" + strings.Join(pathParts[1:], "/") + // r.RequestURI = r.URL.Path + // } else { + // r.URL.Path = "/" + // r.RequestURI = "/" + // } + // } + // } // Make a new request and add newReader to that request body clonedReq := r.Clone(r.Context()) // If request has no body go to proxy directly if r.Body == nil { - return setQueryParams(r, &QueryParams{id, uuids, queryPeriod}) + return setQueryParams(r, &QueryParams{clusterID, uuids, queryPeriod}) } // If failed to read body, skip verification and go to request proxy if body, err = io.ReadAll(r.Body); err != nil { level.Error(logger).Log("msg", "Failed to read request body", "err", err) - return setQueryParams(r, &QueryParams{id, uuids, queryPeriod}) + return setQueryParams(r, &QueryParams{clusterID, uuids, queryPeriod}) } // clone body to existing request and new request @@ -133,7 +136,7 @@ func parseQueryParams(r *http.Request, rmIDs []string, logger log.Logger) *http. if err = clonedReq.ParseForm(); err != nil { level.Error(logger).Log("msg", "Could not parse request body", "err", err) - return setQueryParams(r, &QueryParams{id, uuids, queryPeriod}) + return setQueryParams(r, &QueryParams{clusterID, uuids, queryPeriod}) } // Parse TSDB's query in request query params @@ -159,7 +162,7 @@ func parseQueryParams(r *http.Request, rmIDs []string, logger log.Logger) *http. for _, idMatch := range strings.Split(match[1], "|") { // Ignore empty strings if strings.TrimSpace(idMatch) != "" { - id = strings.TrimSpace(idMatch) + clusterID = strings.TrimSpace(idMatch) } } } @@ -184,7 +187,7 @@ func parseQueryParams(r *http.Request, rmIDs []string, logger log.Logger) *http. } // Set query params to request's context - return setQueryParams(r, &QueryParams{id, uuids, queryPeriod}) + return setQueryParams(r, &QueryParams{clusterID, uuids, queryPeriod}) } // Parse time parameter in request. diff --git a/pkg/lb/frontend/helpers_test.go b/pkg/lb/frontend/helpers_test.go index 458676b..bb9799d 100644 --- a/pkg/lb/frontend/helpers_test.go +++ b/pkg/lb/frontend/helpers_test.go @@ -204,10 +204,10 @@ func TestParseQueryParams(t *testing.T) { req.Header.Add("Content-Type", "application/x-www-form-urlencoded") } - newReq := parseQueryParams(req, test.rmIDs, log.NewNopLogger()) + newReq := parseQueryParams(req, log.NewNopLogger()) queryParams := newReq.Context().Value(QueryParamsContextKey{}).(*QueryParams) //nolint:forcetypeassert assert.Equal(t, queryParams.uuids, test.uuids) - assert.Equal(t, queryParams.id, test.rmID) + assert.Equal(t, queryParams.clusterID, test.rmID) if test.method == "POST" { // Check the new request body can still be parsed diff --git a/pkg/lb/frontend/middleware.go b/pkg/lb/frontend/middleware.go index bf3e8b5..a15e483 100644 --- a/pkg/lb/frontend/middleware.go +++ b/pkg/lb/frontend/middleware.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "regexp" + "slices" "strings" "github.com/go-kit/log" @@ -16,11 +17,12 @@ import ( // Headers. const ( - grafanaUserHeader = "X-Grafana-User" - dashboardUserHeader = "X-Dashboard-User" - loggedUserHeader = "X-Logged-User" - adminUserHeader = "X-Admin-User" - ceemsUserHeader = "X-Ceems-User" + grafanaUserHeader = "X-Grafana-User" + dashboardUserHeader = "X-Dashboard-User" + loggedUserHeader = "X-Logged-User" + adminUserHeader = "X-Admin-User" + ceemsUserHeader = "X-Ceems-User" + ceemsClusterIDHeader = "X-Ceems-Cluster-Id" ) var ( @@ -30,7 +32,7 @@ var ( // Playground: https://goplay.tools/snippet/kq_r_1SOgnG regexpUUID = regexp.MustCompile("(?:.+?)[^gpu]uuid=[~]{0,1}\"(?P[a-zA-Z0-9-|]+)\"(?:.*)") - // Regex that will match unit's ID. + // Regex that will match cluster's ID. regexID = regexp.MustCompile("(?:.+?)ceems_id=[~]{0,1}\"(?P[a-zA-Z0-9-|_]+)\"(?:.*)") ) @@ -124,7 +126,7 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var loggedUser string - var id string + var clusterID string var uuids []string @@ -132,7 +134,7 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler // Clone request, parse query params and set them in request context // This will ensure we set query params in request's context always - r = parseQueryParams(r, amw.clusterIDs, amw.logger) + r = parseQueryParams(r, amw.logger) // Apply middleware only for following endpoints: // - query @@ -140,7 +142,8 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler // - labels // - labels values // - series - if !strings.HasSuffix(r.URL.Path, "query") && !strings.HasSuffix(r.URL.Path, "query_range") && + if !strings.HasSuffix(r.URL.Path, "query") && + !strings.HasSuffix(r.URL.Path, "query_range") && !strings.HasSuffix(r.URL.Path, "values") && !strings.HasSuffix(r.URL.Path, "labels") && !strings.HasSuffix(r.URL.Path, "series") { @@ -206,9 +209,30 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler // Check type assertions if v, ok := queryParams.(*QueryParams); ok { - id = v.id + clusterID = v.clusterID uuids = v.uuids + + // Verify clusterID is in list of valid cluster IDs + if !slices.Contains(amw.clusterIDs, clusterID) { + // Write an error and stop the handler chain + w.WriteHeader(http.StatusBadRequest) + + response := ceems_api.Response[any]{ + Status: "error", + ErrorType: "bad_request", + Error: "invalid cluster ID", + } + if err := json.NewEncoder(w).Encode(&response); err != nil { + level.Error(amw.logger).Log("msg", "Failed to encode response", "err", err) + w.Write([]byte("KO")) + } + + return + } } else { + // Write an error and stop the handler chain + w.WriteHeader(http.StatusBadRequest) + response := ceems_api.Response[any]{ Status: "error", ErrorType: "bad_request", @@ -223,7 +247,7 @@ func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler } // Check if user is querying for his/her own compute units by looking to DB - if !amw.isUserUnit(r.Context(), loggedUser, []string{id}, uuids) { //nolint:contextcheck // False positive + if !amw.isUserUnit(r.Context(), loggedUser, []string{clusterID}, uuids) { //nolint:contextcheck // False positive // Write an error and stop the handler chain w.WriteHeader(http.StatusForbidden) diff --git a/pkg/lb/frontend/middleware_test.go b/pkg/lb/frontend/middleware_test.go index 7767c4a..dd234a5 100644 --- a/pkg/lb/frontend/middleware_test.go +++ b/pkg/lb/frontend/middleware_test.go @@ -185,13 +185,15 @@ func TestMiddlewareWithDB(t *testing.T) { tests := []struct { name string req string + id string user string header bool code int }{ { name: "forbid due to mismatch uuid", - req: "/rm-0/query?query=foo{uuid=~\"1479765|1481510\"}", + req: "/query?query=foo{uuid=~\"1479765|1481510\"}", + id: "rm-0", user: "usr1", header: true, code: 403, @@ -201,59 +203,67 @@ func TestMiddlewareWithDB(t *testing.T) { req: "/query?query=foo{uuid=~\"1481508|1479765\"}", user: "usr2", header: true, - code: 403, + code: 400, }, { name: "allow query for admins", - req: "/rm-0/query_range?query=foo{uuid=~\"1479765|1481510\"}", + req: "/query_range?query=foo{uuid=~\"1479765|1481510\"}", + id: "rm-0", user: "adm1", header: true, code: 200, }, { name: "forbid due to missing project", - req: "/rm-1/query_range?query=foo{uuid=~\"123|345\"}", + req: "/query_range?query=foo{uuid=~\"123|345\"}", + id: "rm-1", user: "usr1", header: true, code: 403, }, { name: "forbid due to missing header", - req: "/rm-0/query?query=foo{uuid=~\"123|345\"}", + req: "/query?query=foo{uuid=~\"123|345\"}", + id: "rm-0", header: false, code: 401, }, { name: "pass due to correct uuid", - req: "/rm-0/query_range?query=foo{uuid=\"1479763\"}", + req: "/query_range?query=foo{uuid=\"1479763\"}", + id: "rm-0", user: "usr1", header: true, code: 200, }, { name: "pass due to correct uuid with gpuuuid in query", - req: "/rm-1/query?query=foo{uuid=\"1479763\",gpuuuid=\"GPU-01234\"}", + req: "/query?query=foo{uuid=\"1479763\",gpuuuid=\"GPU-01234\"}", + id: "rm-1", user: "usr1", header: true, code: 200, }, { name: "pass due to uuid from same project", - req: "/rm-0/query?query=foo{uuid=\"1481508\"}", + req: "/query?query=foo{uuid=\"1481508\"}", + id: "rm-0", user: "usr1", header: true, code: 200, }, { name: "pass due to no uuid", - req: "/rm-0/query_range?query=foo{uuid=\"\"}", + req: "/query_range?query=foo{uuid=\"\"}", + id: "rm-0", header: true, user: "usr3", code: 200, }, { name: "pass due to no uuid and non-empty gpuuuid", - req: "/rm-0/query?query=foo{uuid=\"\",gpuuuid=\"GPU-01234\"}", + req: "/query?query=foo{uuid=\"\",gpuuuid=\"GPU-01234\"}", + id: "rm-0", header: true, user: "usr2", code: 200, @@ -263,7 +273,11 @@ func TestMiddlewareWithDB(t *testing.T) { for _, test := range tests { request := httptest.NewRequest(http.MethodGet, test.req, nil) if test.header { - request.Header.Set("X-Grafana-User", test.user) + request.Header.Set(grafanaUserHeader, test.user) + } + + if test.id != "" { + request.Header.Set(ceemsClusterIDHeader, test.id) } // Tests with CEEMS DB @@ -273,7 +287,7 @@ func TestMiddlewareWithDB(t *testing.T) { resDB := responseRecorderDB.Result() defer resDB.Body.Close() - assert.Equal(t, resDB.StatusCode, test.code, "DB") + assert.Equal(t, test.code, resDB.StatusCode, "%s with DB", test.name) // Tests with CEEMS API apiRequest := request.Clone(request.Context()) @@ -282,6 +296,6 @@ func TestMiddlewareWithDB(t *testing.T) { resAPI := responseRecorderAPI.Result() defer resAPI.Body.Close() - assert.Equal(t, resAPI.StatusCode, test.code, "API") + assert.Equal(t, test.code, resAPI.StatusCode, "%s with API", test.name) } } diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 4d3119b..41646bd 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -717,7 +717,7 @@ then waitport "${port}" - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/slurm-0/api/v1/status/config" > "${fixture_output}" + get -H "X-Grafana-User: usr1" -H "X-Ceems-Cluster-Id: slurm-0" "127.0.0.1:${port}/api/v1/status/config" > "${fixture_output}" elif [[ "${scenario}" = "lb-forbid-user-query-db" ]] then @@ -746,7 +746,7 @@ then waitport "${port}" - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/slurm-1/api/v1/query?query=foo\{uuid=\"1481510\"\}&time=1713032179.506" > "${fixture_output}" + get -H "X-Grafana-User: usr1" -H "X-Ceems-Cluster-Id: slurm-1" "127.0.0.1:${port}/api/v1/query?query=foo\{uuid=\"1481510\"\}&time=1713032179.506" > "${fixture_output}" elif [[ "${scenario}" = "lb-allow-user-query-db" ]] then @@ -775,7 +775,7 @@ then waitport "${port}" - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/slurm-0/api/v1/query?query=foo\{uuid=\"1479763\"\}&time=${timestamp}" > "${fixture_output}" + get -H "X-Grafana-User: usr1" -H "X-Ceems-Cluster-Id: slurm-0" "127.0.0.1:${port}/api/v1/query?query=foo\{uuid=\"1479763\"\}&time=${timestamp}" > "${fixture_output}" elif [[ "${scenario}" = "lb-forbid-user-query-api" ]] then @@ -825,7 +825,7 @@ then waitport "${port}" - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/slurm-1/api/v1/query?query=foo\{uuid=\"1481510\"\}&time=${timestamp}" > "${fixture_output}" + get -H "X-Grafana-User: usr1" -H "X-Ceems-Cluster-Id: slurm-1" "127.0.0.1:${port}/api/v1/query?query=foo\{uuid=\"1481510\"\}&time=${timestamp}" > "${fixture_output}" elif [[ "${scenario}" = "lb-allow-user-query-api" ]] then @@ -875,7 +875,7 @@ then waitport "${port}" - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/slurm-0/api/v1/query?query=foo\{uuid=\"1479763\"\}&time=${timestamp}" > "${fixture_output}" + get -H "X-Grafana-User: usr1" -H "X-Ceems-Cluster-Id: slurm-0" "127.0.0.1:${port}/api/v1/query?query=foo\{uuid=\"1479763\"\}&time=${timestamp}" > "${fixture_output}" elif [[ "${scenario}" = "lb-allow-admin-query" ]] then @@ -904,7 +904,7 @@ then waitport "${port}" - get -H "X-Grafana-User: grafana" -H "Content-Type: application/x-www-form-urlencoded" -X POST -d "query=foo{uuid=\"1479765\"}&time=${timestamp}" "127.0.0.1:${port}/slurm-1/api/v1/query" > "${fixture_output}" + get -H "X-Grafana-User: grafana" -H "X-Ceems-Cluster-Id: slurm-1" -H "Content-Type: application/x-www-form-urlencoded" -X POST -d "query=foo{uuid=\"1479765\"}&time=${timestamp}" "127.0.0.1:${port}/api/v1/query" > "${fixture_output}" elif [[ "${scenario}" = "lb-auth" ]] then @@ -934,7 +934,7 @@ then waitport "${port}" - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/slurm-1/api/v1/status/config" > "${fixture_output}" + get -H "X-Grafana-User: usr1" -H "X-Ceems-Cluster-Id: slurm-1" "127.0.0.1:${port}/api/v1/status/config" > "${fixture_output}" fi fi diff --git a/website/docs/components/ceems-lb.md b/website/docs/components/ceems-lb.md index fbd754d..9f35de0 100644 --- a/website/docs/components/ceems-lb.md +++ b/website/docs/components/ceems-lb.md @@ -6,33 +6,33 @@ sidebar_position: 3 ## Background -The motivation behind creating CEEMS load balancer component is that Prometheus TSDB -do not enforce any sort of access control over its metrics querying. This means once -a user has been given the permissions to query a Prometheus TSDB, they can query _any_ -metrics stored in the TSDB. - -Generally, it is not necessary to expose TSDB to end users directly and it is done -using Grafana as Prometheus datasource. Dashboards that are exposed to the end users -need to have query access on the underlying -datasource that the dashboard uses. Although a regular user with -[`Viewer`](https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/#basic-roles) -role cannot add more panels to an existing dashboard, in order to _view_ the metrics the +The motivation behind creating CEEMS load balancer component is that Prometheus TSDB +do not enforce any sort of access control over its metrics querying. This means once +a user has been given the permissions to query a Prometheus TSDB, they can query _any_ +metrics stored in the TSDB. + +Generally, it is not necessary to expose TSDB to end users directly and it is done +using Grafana as Prometheus datasource. Dashboards that are exposed to the end users +need to have query access on the underlying +datasource that the dashboard uses. Although a regular user with +[`Viewer`](https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/#basic-roles) +role cannot add more panels to an existing dashboard, in order to _view_ the metrics the user has effectively `query` permissions on the datasource. -This effectively means, the user can make _any_ query to the underlying datasource, _e.g.,_ -Prometheus, using the browser cookie that is set by Grafana auth. The consequence is that -the user can query the metrics of _any_ user or _any_ compute unit. Straight forward -solutions to this problem is to create a Prometheus instance for each project/namespace. -However, this is not a scalable solution when they are thousands of projects/namespaces -exist. +This effectively means, the user can make _any_ query to the underlying datasource, _e.g.,_ +Prometheus, using the browser cookie that is set by Grafana auth. The consequence is that +the user can query the metrics of _any_ user or _any_ compute unit. Straight forward +solutions to this problem is to create a Prometheus instance for each project/namespace. +However, this is not a scalable solution when they are thousands of projects/namespaces +exist. -This can pose few issues in multi tenant systems like HPC and cloud computing platforms. -Ideally, we do not want one user to be able to access the compute unit metrics of +This can pose few issues in multi tenant systems like HPC and cloud computing platforms. +Ideally, we do not want one user to be able to access the compute unit metrics of other users. CEEMS load balancer component has been created to address this issue. -CEEMS Load Balancer addresses this issue by acting as a gate keeper to introspect the -query before deciding whether to proxy the request to TSDB or not. It means when a user -makes a TSDB query for a given compute unit, CEEMS load balancer will check if the user +CEEMS Load Balancer addresses this issue by acting as a gate keeper to introspect the +query before deciding whether to proxy the request to TSDB or not. It means when a user +makes a TSDB query for a given compute unit, CEEMS load balancer will check if the user owns that compute unit by verifying with CEEMS API server. ## Objectives @@ -43,51 +43,51 @@ The main objectives of the CEEMS load balancer are two-fold: are only accessible to the members of that project/namespace - To provide basic load balancing for replicated TSDB instances. -Thus, CEEMS load balancer can be configured as Prometheus data source in Grafana and -the load balancer will take care of routing traffic to backend TSDB instances and at +Thus, CEEMS load balancer can be configured as Prometheus data source in Grafana and +the load balancer will take care of routing traffic to backend TSDB instances and at the same time enforcing access control. ## Load balancing -CEEMS load balancer supports classic load balancing strategies like round-robin and least -connection methods. Besides these two, it supports resource based strategy that is +CEEMS load balancer supports classic load balancing strategies like round-robin and least +connection methods. Besides these two, it supports resource based strategy that is based on retention time. Let's take a look at this strategy in-detail. -Taking Prometheus TSDB as an example, Prometheus advises to use local file system to store -the data. This ensure performance and data integrity. However, storing data on local -disk is not fault tolerant unless data is replicated elsewhere. There are cloud native -projects like [Thanos](https://thanos.io/), [Cortex](https://cortexmetrics.io/) to -address this issue. This load balancer is meant +Taking Prometheus TSDB as an example, Prometheus advises to use local file system to store +the data. This ensure performance and data integrity. However, storing data on local +disk is not fault tolerant unless data is replicated elsewhere. There are cloud native +projects like [Thanos](https://thanos.io/), [Cortex](https://cortexmetrics.io/) to +address this issue. This load balancer is meant to provide the basic functionality proposed by Thanos, Cortex, _etc_. -The core idea is to replicate the Prometheus data using -[Prometheus' remote write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) -functionality onto a remote storage which -is fault tolerant and have higher storage capacity but with a degraded query performance. +The core idea is to replicate the Prometheus data using +[Prometheus' remote write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) +functionality onto a remote storage which +is fault tolerant and have higher storage capacity but with a degraded query performance. In this scenario, we have two TSDBs with following characteristics: - TSDB using local disk: faster query performance with limited storage space - TSDB using remote storage: slower query performance with bigger storage space -TSDB using local disk ("hot" instance) will have shorter retention period and the +TSDB using local disk ("hot" instance) will have shorter retention period and the one using remote storage ("cold" instance) can have longer retention. CEEMS load balancer is capable of introspecting the query and then routing the request to either "hot" or "cold" instances of TSDB. ## Multi cluster support -A single deployment of CEEMS load balancer is capable of loading balancing traffic between -different replicated TSDB instances of multiple clusters. Imagine there are two different -clusters, one for SLURM and one for Openstack, in a DC. Slurm cluster has two dedicated -TSDB instances where data is replicated between them and the same for Openstack cluster. -Thus, in total, there are four TSDB instances, two for SLURM cluster and two for -Openstack cluster. A single instance of CEEMS load balancer can route the traffic +A single deployment of CEEMS load balancer is capable of loading balancing traffic between +different replicated TSDB instances of multiple clusters. Imagine there are two different +clusters, one for SLURM and one for Openstack, in a DC. Slurm cluster has two dedicated +TSDB instances where data is replicated between them and the same for Openstack cluster. +Thus, in total, there are four TSDB instances, two for SLURM cluster and two for +Openstack cluster. A single instance of CEEMS load balancer can route the traffic between these four different TSDB instances by targeting the correct cluster. -However, in the production with heavy traffic a single instance of CEEMS load balancer -might not be a optimal solution. In that case, it is however possible to deploy a dedicated +However, in the production with heavy traffic a single instance of CEEMS load balancer +might not be a optimal solution. In that case, it is however possible to deploy a dedicated CEEMS load balancer for each cluster. -More details on how to configuration of multi-clusters can be found in [Configuration](../configuration/ceems-lb.md) -section and some example scenarios are discussed in [Advanced](../advanced/multi-cluster.md) +More details on how to configuration of multi-clusters can be found in [Configuration](../configuration/ceems-lb.md) +section and some example scenarios are discussed in [Advanced](../advanced/multi-cluster.md) section. diff --git a/website/docs/configuration/ceems-lb.md b/website/docs/configuration/ceems-lb.md index f4e0180..fb3abb6 100644 --- a/website/docs/configuration/ceems-lb.md +++ b/website/docs/configuration/ceems-lb.md @@ -67,6 +67,8 @@ does not support TLS for the backends. ### Matching `backends.id` with `clusters.id` +#### Using custom header + This is the tricky part of the configuration which can be better explained with an example. Consider we are running CEEMS API server with the following configuration: @@ -137,15 +139,68 @@ As metrics data of `slurm-0` only exists in either `tsdb-0` or these TSDB backends. Effectively we will use CEEMS LB as a Prometheus datasource in -Grafana and while doing so, we need to target correct cluster -using path parameter. For instance, for `slurm-0` cluster the -datasource URL must be configured as `http://ceems-lb:9030/slurm-0` -assuming `ceems_lb` is running on port `9030`. Now, CEEMS LB will -know which cluster to target (in this case `slurm-0`), strips -the path parameter `slurm-0` from the path and proxies the request -to one of the configured backends. This allows a single instance +Grafana and while doing so, we need to target correct cluster. +This is done using a custom header `X-Ceems-Cluster-Id`. When +configuring the datasource in Grafana, we need to add `X-Ceems-Cluster-Id` +to the custom headers section and set the value to cluster ID. + +For instance, for `slurm-0` cluster the provisioned datasource +config for Grafana will look as follows: + +```yaml +- name: CEEMS-LB + type: prometheus + access: proxy + url: http://localhost:9030 + basicAuth: true + basicAuthUser: ceems + jsonData: + prometheusVersion: 2.51 + prometheusType: Prometheus + timeInterval: 30s + incrementalQuerying: true + cacheLevel: Medium + httpHeaderName1: X-Ceems-Cluster-Id + secureJsonData: + basicAuthPassword: + httpHeaderValue1: slurm-0 + isDefault: true +``` + +assuming CEEMS LB is running at port 9030 on the same host as Grafana. +Notice that we set the header and value in `jsonData` and `secureJsonData`, +respectively. This ensures that datasource will send the header with +every request to CEEMS LB and then LB will redirect the query request +to correct backend. This allows a single instance of CEEMS to load balance across different clusters. +#### Using query label + +If for any reason, the above strategy does not work for a given deployment, +it is also possible to identify target clusters using query labels. However, +for this strategy to work, it is needed to inject labels to Prometheus metrics. +For example in the above case, using [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) +we can set a custom label as follows: + +```yaml +- job_name: ceems + static_configs: + - targets: + - compute-0:9100 + labels: + ceems_id: slurm-0 +``` + +CEEMS LB will read value of `ceems_id` label and then redirects the query +to the appropriate backend. + +:::important[IMPORTANT] + +If both custom header and label `ceems_id` are present in the request to +CEEMS LB, the query label will take the precedence. + +::: + ## CEEMS API Server Configuration This is an optional config when provided will enforce access diff --git a/website/docs/configuration/config-reference.md b/website/docs/configuration/config-reference.md index 8b2147f..1170525 100644 --- a/website/docs/configuration/config-reference.md +++ b/website/docs/configuration/config-reference.md @@ -1,5 +1,5 @@ --- -sidebar_position: 7 +sidebar_position: 9 --- # Configuration Reference