wip

.gitignore

@@ -1,2 +1,4 @@
 grafana-organizations-operator
 env
+.idea/
+control.kubeconfig

README.md

@@ -2,6 +2,42 @@
 
 Automatically set up Grafana organizations based on information from the [APPUiO Control API](https://kb.vshn.ch/appuio-cloud/references/architecture/control-api.html).
 
+## Design Considerations
+
+There are a bunch of issues this operator needs to overcome. These issues have a large impact on how the operator is implemented.
+
+### Systematic Issues
+
+* Changes can happen both in the APPUiO Control API and on Grafana. Hence it is not sufficient to observe the APPUiO Control API and only sync when its data changes; instead in order to fix misconfigurations in Grafana this operator runs near continuously.
+
+### Issues with the APPUiO Control API
+
+* The Control API has no notion of "admin" users, i.e. users which should have access to everything. The configuration option `CONTROL_API_ADMIN_ORG` allows to specify a Control API organization, and all users who have access to that will be treated as admins.
+* The `Organization` and `User` types use different GroupVersions. It seems that this cannot be implemented using a single k8s.io client; instead we need two client instances for the Control API.
+* The `OrganizationMember` field `spec.userRefs` contains a lot of invalid values, including even values with incorrect syntax. These must be filtered out.
+
+### Issues with Grafana
+
+* When a new user is created in Grafana (either by logging in via Keycloak or explicitly created via API), Grafana's `auto_assign_org` "feature" automatically gives the user permission to some organization (whichever is configured). This is almost never what we want. To work around this:
+  * It would be possible to disable `auto_assign_org`, but then Grafana would create a new organization for every new user, which would be even worse.
+  * We can't let Grafana create users on demand when they first log ina, thus we create a Grafana users for all known user, even if they don't have permission to do anything, avoiding any situations where Grafana would create a user on demand.
+  * When we create a Grafana user via API we can specify which organization it should be assigned to. We set this to an organization that the user is allowed to access whenever possible.
+  * We manage the users of org ID 1, which is the one where users might get assigned to by accident, thereby fixing any wrong permissions which might have happened due to race conditions or other problems.
+* Because of the `auto_assign_org` "feature" we have to create the organizations before we can create users.
+* Because the `grafana-api-golang-client` implementation is incomplete we are wrapping it in the GrafanaClient type and add some functionality.
+* The Grafana API often ignores the OrgID JSON field. The only workaround for this is to set the HTTP header `x-grafana-org-id`.
+
+## Development Environment Setup
+
+In order to develop the operator you need:
+
+* Access to the [APPUiO Control API](https://kb.vshn.ch/appuio-cloud/references/architecture/control-api.html)
+* A [Grafana test instance](https://operator-dev-grafana.apps.cloudscale-lpg-2.appuio.cloud/) to write to
+
+You can run the `gen-dev-env.sh` to set up an environment file (`env`) with the required configuration.
+
+Once that's done you can source the env file (`. ./env`) and run the operator on your local machine.
+
 ## License
 
 [BSD-3-Clause](LICENSE)

gen-dev-env.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+if [ ! -e control.kubeconfig ]; then
+	echo "1. Install kubelogin from https://github.com/int128/kubelogin (you must rename the 'kubelogin' binary to 'kubectl-oidc_login' and copy it e.g. to '/usr/local/bin/')"
+	echo "2. Visit https://portal.appuio.cloud/kubeconfig and put the displayed kubeconfig into control.kubeconfig"
+	echo "3. Re-run this command"
+	echo "4. Remove the control.kubeconfig file"
+	exit 1
+fi
+echo -n "" > env
+echo "export GRAFANA_URL=https://operator-dev-grafana.apps.cloudscale-lpg-2.appuio.cloud" >> env
+echo "export GRAFANA_USERNAME=admin" >> env
+echo -n "export GRAFANA_PASSWORD=" >> env
+kubectl --as cluster-admin -n vshn-grafana-organizations-operator-dev get secret grafana-env -ojsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d >> env
+echo "" >> env
+echo "export CONTROL_API_URL=https://api.appuio.cloud" >> env
+echo -n "export CONTROL_API_TOKEN=" >> env
+kubectl --kubeconfig control.kubeconfig get secret grafana-organizations-operator -ojsonpath='{.data.token}' | base64 -d >> env
+echo "" >> env
+echo "export CONTROL_API_ADMIN_ORG=vshn" >> env
+echo "done"

go.mod

@@ -4,7 +4,8 @@ go 1.20
 
 require (
 	github.com/appuio/control-api v0.26.0
-	github.com/grafana/grafana-api-golang-client v0.21.0
+	github.com/grafana/grafana-api-golang-client v0.23.0
+	github.com/hashicorp/go-cleanhttp v0.5.2
 	k8s.io/apimachinery v0.26.2
 	k8s.io/client-go v0.26.2
 	k8s.io/klog/v2 v2.90.1
@@ -20,7 +21,6 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect

go.sum

@@ -18,6 +18,7 @@ github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKf
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -47,6 +48,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grafana/grafana-api-golang-client v0.21.0 h1:PQ2Wfo9jMMiftC4VRMlJxbUNvYCXMV1YFDKm7Ny3SaM=
 github.com/grafana/grafana-api-golang-client v0.21.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
+github.com/grafana/grafana-api-golang-client v0.23.0 h1:Uta0dSkxWYf1D83/E7MRLCG69387FiUc+k9U/35nMhY=
+github.com/grafana/grafana-api-golang-client v0.23.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 h1:1JYBfzqrWPcCclBwxFCPAou9n+q86mfnu7NAeHfte7A=
@@ -56,8 +59,10 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -71,9 +76,11 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc=
 github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=

main.go

@@ -20,29 +20,44 @@ import (
 )
 
 var (
-	ControlApiToken string
-	ControlApiUrl   string
-	GrafanaUrl      string
-	GrafanaUsername string
-	GrafanaPassword string
+	ControlApiToken    string
+	ControlApiUrl      string
+	ControlApiAdminOrg string
+	GrafanaUrl         string
+	GrafanaUsername    string
+	GrafanaPassword    string
+	AdminExtraOrgs     = [1]int{1}
 )
 
 func main() {
 	ControlApiUrl = os.Getenv("CONTROL_API_URL")
+	ControlApiAdminOrg = os.Getenv("CONTROL_API_ADMIN_ORG")
 	ControlApiToken = os.Getenv("CONTROL_API_TOKEN")
+	ControlApiTokenHidden := ""
+	if ControlApiToken != "" {
+		ControlApiTokenHidden = "***hidden***"
+	}
+
 	GrafanaUrl = os.Getenv("GRAFANA_URL")
 	GrafanaUsername = os.Getenv("GRAFANA_USERNAME")
 	if GrafanaUsername == "" {
 		GrafanaUsername = os.Getenv("admin-user") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
 	}
 	GrafanaPassword = os.Getenv("GRAFANA_PASSWORD")
+	GrafanaPasswordHidden := ""
 	if GrafanaPassword == "" {
 		GrafanaPassword = os.Getenv("admin-password") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
 	}
+	if GrafanaPassword != "" {
+		GrafanaPasswordHidden = "***hidden***"
+	}
 
 	klog.Infof("CONTROL_API_URL: %s\n", ControlApiUrl)
+	klog.Infof("CONTROL_API_ADMIN_ORG: %s\n", ControlApiAdminOrg)
+	klog.Infof("CONTROL_API_TOKEN: %s\n", ControlApiTokenHidden)
 	klog.Infof("GRAFANA_URL: %s\n", GrafanaUrl)
 	klog.Infof("GRAFANA_USERNAME: %s\n", GrafanaUsername)
+	klog.Infof("GRAFANA_PASSWORD: %s\n", GrafanaPasswordHidden)
 
 	// Because of the strange design of the k8s client we actually need two client objects, which both internally use the same httpClient.
 	// To make this work we also need three (!) config objects, a common one for the httpClient and one for each k8s client.
@@ -100,7 +115,7 @@ func main() {
 	json.Unmarshal(db, &dashboard)
 
 	klog.Info("Starting initial sync...")
-	err = controller.ReconcileAllOrgs(ctx, organizationAppuioIoClient, appuioIoClient, grafanaConfig, GrafanaUrl, dashboard)
+	err = controller.Reconcile(ctx, organizationAppuioIoClient, appuioIoClient, ControlApiAdminOrg, grafanaConfig, GrafanaUrl, dashboard)
 	if err != nil {
 		klog.Errorf("Could not do initial reconciliation: %v\n", err)
 		os.Exit(1)
@@ -112,7 +127,7 @@ func main() {
 		case <-ctx.Done():
 			os.Exit(0)
 		}
-		err = controller.ReconcileAllOrgs(ctx, organizationAppuioIoClient, appuioIoClient, grafanaConfig, GrafanaUrl, dashboard)
+		err = controller.Reconcile(ctx, organizationAppuioIoClient, appuioIoClient, ControlApiAdminOrg, grafanaConfig, GrafanaUrl, dashboard)
 		if err != nil {
 			klog.Errorf("Could not reconcile (will retry): %v\n", err)
 		}

pkg/controlApi.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	orgs "github.com/appuio/control-api/apis/organization/v1"
 	controlapi "github.com/appuio/control-api/apis/v1"
-	grafana "github.com/grafana/grafana-api-golang-client"
 	"k8s.io/client-go/rest"
-	"k8s.io/klog/v2"
 )
 
 // Generate list of control API organizations
@@ -36,25 +34,3 @@ func getControlApiUsersMap(ctx context.Context, appuioIoClient *rest.RESTClient)
 	}
 	return appuioControlApiUsersMap, nil
 }
-
-// Generate map containing all Grafana users grouped by organization. Key is the organization ID, value is an array of users.
-func getControlApiOrganizationUsersMap(ctx context.Context, grafanaUsersMap map[string]grafana.User, appuioIoClient *rest.RESTClient) (map[string][]grafana.User, error) {
-	appuioControlApiOrganizationMembers := controlapi.OrganizationMembersList{}
-	err := appuioIoClient.Get().Resource("OrganizationMembers").Do(ctx).Into(&appuioControlApiOrganizationMembers)
-	if err != nil {
-		return nil, err
-	}
-	controlApiOrganizationUsersMap := make(map[string][]grafana.User)
-	for _, memberlist := range appuioControlApiOrganizationMembers.Items {
-		users := []grafana.User{}
-		for _, userRef := range memberlist.Spec.UserRefs {
-			if grafanaUser, ok := grafanaUsersMap[userRef.Name]; ok {
-				users = append(users, grafanaUser)
-			} else {
-				klog.Warningf("Organization '%s' should have user %s but the user wasn't synced to Grafana, ignoring", memberlist.Namespace, userRef.Name)
-			}
-		}
-		controlApiOrganizationUsersMap[memberlist.Namespace] = users
-	}
-	return controlApiOrganizationUsersMap, nil
-}