feat: Support automatic token rotation for Openstack

* By default Openstack's API tokens are only valid for 1 hour and AFAIK there is no way to generate tokens with long duration. This PR modifies the config accepted by Openstack manager and takes auth object as input. This auth object will be used to generate and rotate API tokens for every 1 hour.

*We accept the same auth object as Openstack so we pass this object to keystone transparently which allows us to support all forms of auth object without much code. It is the user's responsibility to pass correct auth object to clusters config.

* Add new handlers for mock servers and modify test resources appropriately

* Update docs

Signed-off-by: Mahendra Paipuri <[email protected]>

build/config/ceems_api_server/ceems_api_server.yml

@@ -299,15 +299,8 @@ clusters: []
   #     # An object of environment variables that will be injected while executing the 
   #     # CLI utilities to fetch compute unit data. 
   #     #
-  #     # This is handy when executing CLI tools like `keystone` for openstack or `kubectl` 
-  #     # for k8s needs to source admin credentials. Those credentials can be set manually
-  #     # here in this section. 
-  #     #
   #     environment_variables: {}
-  #       # OS_USERNAME: username  # Openstack RC File
-  #       # OS_PASSWORD: password  # Openstack RC File
-  #       # OS_TENANT_NAME: projectName  # Openstack RC File
-  #       # OS_AUTH_URL: https://identityHost:portNumber/v2.0  # Openstack RC File
+  #       # NAME: value  # Environment variable name value pair
 
   #   # If the resource manager supports API server, configure the REST API
   #   # server details here.
@@ -459,23 +452,27 @@ clusters: []
   #   # Any other configuration needed to reach API server of the resource manager
   #   # can be configured in this section.
   #   #
-  #   # Currently this section is used for both SLURM and Openstack resource managers
+  #   # Currently this section is used for Openstack resource manager
   #   # to configure API versions
   #   #
-  #   # For example, for SLURM if your API endpoints are of form `/slurm/v0.0.40/diag`, 
-  #   # the version is `v0.0.40`.
-  #   # Docs: https://slurm.schedmd.com/rest_api.html
-  #   # SLURM's REST API version can be set as `slurm: v0.0.40`
-  #   #
-  #   # In the case of Openstack, we need to fetch from different sources like identity,
-  #   # compute and they use different versioning of API. They can be configured using
-  #   # this section as well
+  #   # In the case of Openstack, this section must have two keys `api_service_endpoints`
+  #   # and `auth`. Both of these are compulsory.
+  #   # `api_service_endpoints` must provide API endpoints for compute and identity
+  #   # services as provided in service catalog of Openstack cluster. `auth` must be the
+  #   # same `auth` object that must be sent in POST request to keystone to get a API token.
   #   #
-  #   extra_config:
-  #     api_versions: {}
-  #       # slurm: v0.0.40  # SLURM
-  #       # identity: v3  # Openstack
-  #       # compute: v2.1  # Openstack
+  #   extra_config: {}
+  #      # api_service_endpoints:
+  #      #   compute: https://openstack-nova.example.com/v2.1
+  #      #   identity: https://openstack-keystone.example.com
+  #      # auth:
+  #      #   identity:
+  #      #     methods:
+  #      #       - password
+  #      #     password:
+  #      #       user:
+  #      #         name: admin
+  #      #         password: supersecret
 
 # A list of Updaters that will be used to update the compute unit metrics. This update 
 # step can be used to update the aggregate metrics of each compute unit in real time

build/package/ceems_api_server/ceems_api_server.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Prometheus CEEMS API Server
+Description=CEEMS API Server
 After=network-online.target
 
 [Service]

cmd/mock_servers/main.go

@@ -239,6 +239,22 @@ func ServersHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("KO"))
 }
 
+// TokensHandler handles OS tokens.
+func TokensHandler(w http.ResponseWriter, r *http.Request) {
+	decoder := json.NewDecoder(r.Body)
+
+	var t map[string]interface{}
+
+	if err := decoder.Decode(&t); err != nil {
+		w.Write([]byte("KO"))
+
+		return
+	}
+
+	w.Header().Add("X-Subject-Token", "apitokensecret")
+	w.WriteHeader(http.StatusCreated)
+}
+
 // UsersHandler handles OS users.
 func UsersHandler(w http.ResponseWriter, r *http.Request) {
 	if data, err := os.ReadFile("pkg/api/testdata/openstack/identity/users.json"); err == nil {
@@ -328,10 +344,11 @@ func osKSServer(ctx context.Context) {
 
 	// Registering our handler functions, and creating paths.
 	osKSMux := http.NewServeMux()
+	osKSMux.HandleFunc("/v3/auth/tokens", TokensHandler)
 	osKSMux.HandleFunc("/v3/users", UsersHandler)
 	osKSMux.HandleFunc("/v3/users/{id}/projects", ProjectsHandler)
 
-	log.Println("Started Prometheus on port", osKSPortNum)
+	log.Println("Started Openstack identity API server on port", osKSPortNum)
 	log.Println("To close connection CTRL+C :-)")
 
 	// Start server

internal/common/helpers.go

@@ -57,6 +57,51 @@ func GetUUIDFromString(stringSlice []string) (string, error) {
 	return uuid.String(), err
 }
 
+// ConvertMapI2MapS walks the given dynamic object recursively, and
+// converts maps with interface{} key type to maps with string key type.
+// This function comes handy if you want to marshal a dynamic object into
+// JSON where maps with interface{} key type are not allowed.
+//
+// Recursion is implemented into values of the following types:
+//
+//	-map[interface{}]interface{}
+//	-map[string]interface{}
+//	-[]interface{}
+//
+// When converting map[interface{}]interface{} to map[string]interface{},
+// fmt.Sprint() with default formatting is used to convert the key to a string key.
+//
+// Nicked from https://github.com/icza/dyno
+func ConvertMapI2MapS(v interface{}) interface{} {
+	switch x := v.(type) {
+	case map[interface{}]interface{}:
+		m := map[string]interface{}{}
+
+		for k, v2 := range x {
+			switch k2 := k.(type) {
+			case string: // Fast check if it's already a string
+				m[k2] = ConvertMapI2MapS(v2)
+			default:
+				m[fmt.Sprint(k)] = ConvertMapI2MapS(v2)
+			}
+		}
+
+		v = m
+
+	case []interface{}:
+		for i, v2 := range x {
+			x[i] = ConvertMapI2MapS(v2)
+		}
+
+	case map[string]interface{}:
+		for k, v2 := range x {
+			x[k] = ConvertMapI2MapS(v2)
+		}
+	}
+
+	return v
+}
+
 // MakeConfig reads config file, merges with passed default config and returns updated
 // config instance.
 func MakeConfig[T any](filePath string) (*T, error) {

internal/common/helpers_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/mahendrapaipuri/ceems/pkg/grafana"
@@ -110,6 +111,78 @@ func TestGetUuid(t *testing.T) {
 	assert.Equal(t, expected, got, "mismatched UUIDs")
 }
 
+func TestConvertMapI2MapS(t *testing.T) {
+	cases := []struct {
+		title string      // Title of the test case
+		v     interface{} // Input dynamic object
+		exp   interface{} // Expected result
+	}{
+		{
+			title: "nil value",
+			v:     nil,
+			exp:   nil,
+		},
+		{
+			title: "string value",
+			v:     "a",
+			exp:   "a",
+		},
+		{
+			title: "map[interfac{}]interface{} value",
+			v: map[interface{}]interface{}{
+				"s": "s",
+				1:   1,
+			},
+			exp: map[string]interface{}{
+				"s": "s",
+				"1": 1,
+			},
+		},
+		{
+			title: "nested maps and slices",
+			v: map[interface{}]interface{}{
+				"s": "s",
+				1:   1,
+				float64(0): []interface{}{
+					1,
+					"x",
+					map[interface{}]interface{}{
+						"s": "s",
+						2.0: 2,
+					},
+					map[string]interface{}{
+						"s": "s",
+						"1": 1,
+					},
+				},
+			},
+			exp: map[string]interface{}{
+				"s": "s",
+				"1": 1,
+				"0": []interface{}{
+					1,
+					"x",
+					map[string]interface{}{
+						"s": "s",
+						"2": 2,
+					},
+					map[string]interface{}{
+						"s": "s",
+						"1": 1,
+					},
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		v := ConvertMapI2MapS(c.v)
+		if !reflect.DeepEqual(v, c.exp) {
+			t.Errorf("[title: %s] Expected value: %v, got: %v", c.title, c.exp, c.v)
+		}
+	}
+}
+
 func TestMakeConfig(t *testing.T) {
 	tmpDir := t.TempDir()
 	configFile := `

pkg/api/resource/openstack/compute.go

@@ -108,7 +108,7 @@ func (o *openstackManager) activeInstances(ctx context.Context, start time.Time,
 	wg.Wait()
 
 	// If no servers found, return error(s)
-	if len(allServers) == 0 {
+	if allErrs != nil {
 		return nil, allErrs
 	}
 
@@ -257,14 +257,20 @@ func (o *openstackManager) fetchInstances(ctx context.Context, start time.Time,
 		return nil, fmt.Errorf("failed to create request to fetch Openstack instances: %w", err)
 	}
 
+	// Add token to request headers
+	req, err = o.addTokenHeader(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to rotate api token for openstack cluster: %w", err)
+	}
+
 	// Add query parameters
 	q := req.URL.Query()
 	q.Add("all_tenants", "true")
 
 	if deleted {
 		q.Add("deleted", "true")
 		q.Add("changes-since", start.Format(osTimeFormat))
-		q.Add("changes-until", end.Format(osTimeFormat))
+		q.Add("changes-before", end.Format(osTimeFormat))
 	}
 
 	req.URL.RawQuery = q.Encode()

pkg/api/resource/openstack/identity.go

@@ -1,6 +1,7 @@
 package openstack
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -17,6 +18,32 @@ const (
 	chunkSize = 256
 )
 
+// rotateToken requests new API token from keystone.
+func (o *openstackManager) rotateToken(ctx context.Context) error {
+	// Create a new GET request
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		o.tokens().String(),
+		bytes.NewBuffer(o.auth),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create request to rotate API token for openstack cluster: %w", err)
+	}
+
+	// Get Token
+	o.apiToken, err = apiTokenRequest(req, o.client)
+	if err != nil {
+		return fmt.Errorf("failed to complete request to rotate token for openstack cluster: %w", err)
+	}
+
+	// Set token expiry. By default Openstack tokens are 1 hour and we use a tolerance
+	// of 5 minutes just to account for clock skew to avoid failed requests
+	o.apiTokenExpiry = time.Now().Add(tokenExpiryDuration - 5*time.Minute)
+
+	return nil
+}
+
 // updateUsersProjects updates users and projects of a given Openstack cluster.
 func (o *openstackManager) updateUsersProjects(ctx context.Context, current time.Time) error {
 	// Fetch current users and projects
@@ -43,6 +70,12 @@ func (o *openstackManager) fetchUsers(ctx context.Context) ([]User, error) {
 		return nil, fmt.Errorf("failed to create request to fetch users for openstack cluster: %w", err)
 	}
 
+	// Add token to request headers
+	req, err = o.addTokenHeader(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to rotate api token for openstack cluster: %w", err)
+	}
+
 	// Get response
 	resp, err := apiRequest[UsersResponse](req, o.client)
 	if err != nil {
@@ -65,6 +98,12 @@ func (o *openstackManager) fetchUserProjects(ctx context.Context, userID string)
 		return nil, fmt.Errorf("failed to create request to fetch user projects for openstack cluster: %w", err)
 	}
 
+	// Add token to request headers
+	req, err = o.addTokenHeader(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to rotate api token for openstack cluster: %w", err)
+	}
+
 	// Get response
 	resp, err := apiRequest[ProjectsResponse](req, o.client)
 	if err != nil {