feat: Datasource URL configurable

gen-dev-env.sh

@@ -12,3 +12,6 @@ echo "export GRAFANA_USERNAME=\"admin\"" >> env
 echo -n "export GRAFANA_PASSWORD=\"" >> env
 kubectl --as cluster-admin -n vshn-grafana-organizations-operator-dev get secret grafana-env -ojsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d >> env
 echo "\"" >> env
+echo "export GRAFANA_DATASOURCE_URL=\"http://vshn-appuio-mimir-nginx.vshn-appuio-mimir.svc.cluster.local/prometheus\"" >> env
+echo "export GRAFANA_DATASOURCE_USERNAME=\"dummyuser\"" >> env
+echo "export GRAFANA_DATASOURCE_PASSWORD=\"dummypass\"" >> env

main.go

@@ -18,66 +18,64 @@ import (
 	"time"
 )
 
-var (
-	GrafanaUrl                     string
-	GrafanaUsername                string
-	GrafanaPassword                string
-	KeycloakUrl                    string
-	KeycloakRealm                  string
-	KeycloakUsername               string
-	KeycloakPassword               string
-	KeycloakClientId               string
-	KeycloakAdminGroupPath         string
-	KeycloakAutoAssignOrgGroupPath string
-)
-
 func main() {
-	GrafanaUrl = os.Getenv("GRAFANA_URL")
-	GrafanaUsername = os.Getenv("GRAFANA_USERNAME")
-	if GrafanaUsername == "" {
-		GrafanaUsername = os.Getenv("admin-user") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
-	}
-	GrafanaPassword = os.Getenv("GRAFANA_PASSWORD")
-	GrafanaPasswordHidden := ""
-	if GrafanaPassword == "" {
-		GrafanaPassword = os.Getenv("admin-password") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
-	}
-	if GrafanaPassword != "" {
-		GrafanaPasswordHidden = "***hidden***"
-	}
-
-	KeycloakUrl = os.Getenv("KEYCLOAK_URL")
-	KeycloakRealm = os.Getenv("KEYCLOAK_REALM")
-	KeycloakUsername = os.Getenv("KEYCLOAK_USERNAME")
-	KeycloakPassword = os.Getenv("KEYCLOAK_PASSWORD")
-	KeycloakClientId = os.Getenv("KEYCLOAK_CLIENT_ID")
-	KeycloakPasswordHidden := ""
-	if KeycloakPassword != "" {
-		KeycloakPasswordHidden = "***hidden***"
-	}
-	KeycloakAdminGroupPath = os.Getenv("KEYCLOAK_ADMIN_GROUP_PATH")
-	KeycloakAutoAssignOrgGroupPath = os.Getenv("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH")
-
-	klog.Infof("GRAFANA_URL:                         %s\n", GrafanaUrl)
-	klog.Infof("GRAFANA_USERNAME:                    %s\n", GrafanaUsername)
-	klog.Infof("GRAFANA_PASSWORD:                    %s\n", GrafanaPasswordHidden)
-	klog.Infof("KEYCLOAK_URL:                        %s\n", KeycloakUrl)
-	klog.Infof("KEYCLOAK_REALM:                      %s\n", KeycloakRealm)
-	klog.Infof("KEYCLOAK_USERNAME:                   %s\n", KeycloakUsername)
-	klog.Infof("KEYCLOAK_PASSWORD:                   %s\n", KeycloakPasswordHidden)
-	klog.Infof("KEYCLOAK_CLIENT_ID:                  %s\n", KeycloakClientId)
-	klog.Infof("KEYCLOAK_ADMIN_GROUP_PATH:           %s\n", KeycloakAdminGroupPath)
-	klog.Infof("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH: %s\n", KeycloakAutoAssignOrgGroupPath)
-
-	keycloakClient, err := controller.NewKeycloakClient(KeycloakUrl, KeycloakRealm, KeycloakUsername, KeycloakPassword, KeycloakClientId, KeycloakAdminGroupPath, KeycloakAutoAssignOrgGroupPath)
+	config := controller.Config{}
+	grafanaUrl := os.Getenv("GRAFANA_URL")
+	grafanaUsername := os.Getenv("GRAFANA_USERNAME")
+	if grafanaUsername == "" {
+		grafanaUsername = os.Getenv("admin-user") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
+	}
+	grafanaPassword := os.Getenv("GRAFANA_PASSWORD")
+	grafanaPasswordHidden := ""
+	if grafanaPassword == "" {
+		grafanaPassword = os.Getenv("admin-password") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
+	}
+	if grafanaPassword != "" {
+		grafanaPasswordHidden = "***hidden***"
+	}
+	config.GrafanaDatasourceUrl = os.Getenv("GRAFANA_DATASOURCE_URL")
+	config.GrafanaDatasourceUsername = os.Getenv("GRAFANA_DATASOURCE_USERNAME")
+	config.GrafanaDatasourcePassword = os.Getenv("GRAFANA_DATASOURCE_PASSWORD")
+	grafanaDatasourcePasswordHidden := ""
+	if config.GrafanaDatasourcePassword != "" {
+		grafanaDatasourcePasswordHidden = "***hidden***"
+	}
+
+	keycloakUrl := os.Getenv("KEYCLOAK_URL")
+	keycloakRealm := os.Getenv("KEYCLOAK_REALM")
+	keycloakUsername := os.Getenv("KEYCLOAK_USERNAME")
+	keycloakPassword := os.Getenv("KEYCLOAK_PASSWORD")
+	keycloakClientId := os.Getenv("KEYCLOAK_CLIENT_ID")
+	keycloakPasswordHidden := ""
+	if keycloakPassword != "" {
+		keycloakPasswordHidden = "***hidden***"
+	}
+	keycloakAdminGroupPath := os.Getenv("KEYCLOAK_ADMIN_GROUP_PATH")
+	keycloakAutoAssignOrgGroupPath := os.Getenv("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH")
+
+	klog.Infof("GRAFANA_URL:                         %s\n", grafanaUrl)
+	klog.Infof("GRAFANA_USERNAME:                    %s\n", grafanaUsername)
+	klog.Infof("GRAFANA_PASSWORD:                    %s\n", grafanaPasswordHidden)
+	klog.Infof("GRAFANA_DATASOURCE_URL:              %s\n", config.GrafanaDatasourceUrl)
+	klog.Infof("GRAFANA_DATASOURCE_USERNAME:         %s\n", config.GrafanaDatasourceUsername)
+	klog.Infof("GRAFANA_DATASOURCE_PASSWORD:         %s\n", grafanaDatasourcePasswordHidden)
+	klog.Infof("KEYCLOAK_URL:                        %s\n", keycloakUrl)
+	klog.Infof("KEYCLOAK_REALM:                      %s\n", keycloakRealm)
+	klog.Infof("KEYCLOAK_USERNAME:                   %s\n", keycloakUsername)
+	klog.Infof("KEYCLOAK_PASSWORD:                   %s\n", keycloakPasswordHidden)
+	klog.Infof("KEYCLOAK_CLIENT_ID:                  %s\n", keycloakClientId)
+	klog.Infof("KEYCLOAK_ADMIN_GROUP_PATH:           %s\n", keycloakAdminGroupPath)
+	klog.Infof("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH: %s\n", keycloakAutoAssignOrgGroupPath)
+
+	keycloakClient, err := controller.NewKeycloakClient(keycloakUrl, keycloakRealm, keycloakUsername, keycloakPassword, keycloakClientId, keycloakAdminGroupPath, keycloakAutoAssignOrgGroupPath)
 	if err != nil {
 		klog.Errorf("Could not create keycloakClient client: %v\n", err)
 		os.Exit(1)
 	}
 	defer keycloakClient.CloseIdleConnections()
 
-	grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(GrafanaUsername, GrafanaPassword)}
-	grafanaClient, err := controller.NewGrafanaClient(GrafanaUrl, grafanaConfig)
+	grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(grafanaUsername, grafanaPassword)}
+	grafanaClient, err := controller.NewGrafanaClient(grafanaUrl, grafanaConfig)
 	if err != nil {
 		klog.Errorf("Could not create Grafana client: %v\n", err)
 		os.Exit(1)
@@ -104,14 +102,14 @@ func main() {
 	}
 
 	klog.Info("Starting initial sync...")
-	err = controller.Reconcile(ctx, keycloakClient, grafanaClient, dashboards)
+	err = controller.Reconcile(ctx, config, keycloakClient, grafanaClient, dashboards)
 	if err != nil {
 		klog.Errorf("Could not do initial reconciliation: %v\n", err)
 		os.Exit(1)
 	}
 
 	for {
-		err = controller.Reconcile(ctx, keycloakClient, grafanaClient, dashboards)
+		err = controller.Reconcile(ctx, config, keycloakClient, grafanaClient, dashboards)
 		select {
 		case <-time.After(2 * time.Second):
 		case <-ctx.Done():

pkg/reconcile.go

@@ -6,11 +6,17 @@ import (
 	"k8s.io/klog/v2"
 )
 
+type Config struct {
+	GrafanaDatasourceUrl      string
+	GrafanaDatasourceUsername string
+	GrafanaDatasourcePassword string
+}
+
 var (
 	interruptedError = errors.New("interrupted")
 )
 
-func Reconcile(ctx context.Context, keycloakClient *KeycloakClient, grafanaClient *GrafanaClient, dashboards []Dashboard) error {
+func Reconcile(ctx context.Context, config Config, keycloakClient *KeycloakClient, grafanaClient *GrafanaClient, dashboards []Dashboard) error {
 	klog.Infof("Fetching Keycloak access token...")
 	keycloakToken, err := keycloakClient.GetToken()
 	if err != nil {
@@ -74,7 +80,7 @@ outAutoAssignOrgUsers:
 	}
 	klog.Infof("Found %d auto_assign_org users", len(keycloakAutoAssignOrgUsers))
 
-	grafanaOrgsMap, err := reconcileAllOrgs(ctx, keycloakOrganizations, grafanaClient, dashboards)
+	grafanaOrgsMap, err := reconcileAllOrgs(ctx, config, keycloakOrganizations, grafanaClient, dashboards)
 	if err != nil {
 		return err
 	}

pkg/reconcileOrg.go

@@ -38,8 +38,8 @@ func reconcileOrgBasic(grafanaOrgLookup map[string]grafana.Org, grafanaClient *G
 	return &grafanaOrg, nil
 }
 
-func reconcileOrgSettings(org *grafana.Org, orgName string, grafanaClient *GrafanaClient, dashboards []Dashboard) error {
-	dataSource, err := reconcileOrgDataSource(org, orgName, grafanaClient)
+func reconcileOrgSettings(config Config, org *grafana.Org, orgName string, grafanaClient *GrafanaClient, dashboards []Dashboard) error {
+	dataSource, err := reconcileOrgDataSource(config, org, orgName, grafanaClient)
 	if err != nil {
 		return err
 	}
@@ -53,23 +53,31 @@ func reconcileOrgSettings(org *grafana.Org, orgName string, grafanaClient *Grafa
 	return nil
 }
 
-func reconcileOrgDataSource(org *grafana.Org, orgName string, grafanaClient *GrafanaClient) (*grafana.DataSource, error) {
+func reconcileOrgDataSource(config Config, org *grafana.Org, orgName string, grafanaClient *GrafanaClient) (*grafana.DataSource, error) {
+	secureJSONData := map[string]interface{}{
+		"httpHeaderValue1": orgName,
+	}
+	basicAuth := config.GrafanaDatasourceUsername != ""
+	if basicAuth {
+		secureJSONData["basicAuthPassword"] = config.GrafanaDatasourcePassword
+	}
+
 	// If you add/remove fields here you must also adjust the 'if' statement further down
 	desiredDataSource := &grafana.DataSource{
-		Name:      "Mimir",
-		URL:       "http://vshn-appuio-mimir-query-frontend.vshn-appuio-mimir.svc.cluster.local:8080/prometheus",
-		OrgID:     org.ID, // doesn't actually do anything, we just keep it here in case it becomes relevant with some never version of the client library. The actual orgId is taken from the 'X-Grafana-Org-Id' HTTP header which is set up via grafanaConfig.OrgID
-		Type:      "prometheus",
-		IsDefault: true,
+		Name:          "Mimir",
+		URL:           config.GrafanaDatasourceUrl,
+		BasicAuth:     basicAuth,
+		BasicAuthUser: config.GrafanaDatasourceUsername,
+		OrgID:         org.ID, // doesn't actually do anything, we just keep it here in case it becomes relevant with some never version of the client library. The actual orgId is taken from the 'X-Grafana-Org-Id' HTTP header which is set up via grafanaConfig.OrgID
+		Type:          "prometheus",
+		IsDefault:     true,
 		JSONData: map[string]interface{}{
 			"httpHeaderName1": "X-Scope-OrgID",
 			"httpMethod":      "POST",
 			"prometheusType":  "Mimir",
 		},
-		SecureJSONData: map[string]interface{}{
-			"httpHeaderValue1": orgName,
-		},
-		Access: "proxy",
+		SecureJSONData: secureJSONData,
+		Access:         "proxy",
 	}
 
 	var configuredDataSource *grafana.DataSource
@@ -82,10 +90,12 @@ func reconcileOrgDataSource(org *grafana.Org, orgName string, grafanaClient *Gra
 		for _, dataSource := range dataSources {
 			if dataSource.Name == desiredDataSource.Name {
 				if dataSource.URL != desiredDataSource.URL ||
+					dataSource.BasicAuth != desiredDataSource.BasicAuth ||
 					dataSource.Type != desiredDataSource.Type ||
 					dataSource.IsDefault != desiredDataSource.IsDefault ||
 					!reflect.DeepEqual(dataSource.JSONData, desiredDataSource.JSONData) ||
 					dataSource.Access != desiredDataSource.Access {
+					// note that we can't detect changed basic auth credentials (BasicAuthUser, secureJSONData) because the API does not give us the current settings
 					klog.Infof("Organization %d has misconfigured data source, fixing", org.ID)
 					desiredDataSource.ID = dataSource.ID
 					desiredDataSource.UID = dataSource.UID
@@ -99,7 +109,7 @@ func reconcileOrgDataSource(org *grafana.Org, orgName string, grafanaClient *Gra
 				}
 			} else {
 				klog.Infof("Organization %d has invalid data source %d %s, removing", org.ID, dataSource.ID, dataSource.Name)
-				grafanaClient.DeleteDataSource(org, dataSource.ID)
+				err = grafanaClient.DeleteDataSource(org, dataSource.ID)
 				if err != nil {
 					return nil, err
 				}
@@ -128,7 +138,7 @@ func reconcileOrgDashboard(org *grafana.Org, dataSource *grafana.DataSource, gra
 
 	dashboardTitle, ok := dashboard.Data["title"]
 	if !ok {
-		errors.New("Invalid dashboard format: 'title' key not found")
+		return errors.New("Invalid dashboard format: 'title' key not found")
 	}
 
 	dashboards, err := grafanaClient.Dashboards(org)
@@ -147,12 +157,6 @@ func reconcileOrgDashboard(org *grafana.Org, dataSource *grafana.DataSource, gra
 		}
 	}
 
-	// FIXME not required?
-	/*err = configureDashboard(dashboard.Data, dataSource)
-	if err != nil {
-		return err
-	}*/
-
 	db := grafana.Dashboard{
 		Model:     dashboard.Data,
 		Overwrite: true,

pkg/reconcileOrgs.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-func reconcileAllOrgs(ctx context.Context, keycloakOrganizations []*KeycloakGroup, grafanaClient *GrafanaClient, dashboards []Dashboard) (map[string]*grafana.Org, error) {
+func reconcileAllOrgs(ctx context.Context, config Config, keycloakOrganizations []*KeycloakGroup, grafanaClient *GrafanaClient, dashboards []Dashboard) (map[string]*grafana.Org, error) {
 	grafanaOrgLookupFinal := make(map[string]*grafana.Org)
 
 	// Get all orgs from Grafana
@@ -34,7 +34,7 @@ func reconcileAllOrgs(ctx context.Context, keycloakOrganizations []*KeycloakGrou
 		}
 		delete(grafanaOrgLookup, keycloakOrganization.Name)
 
-		err = reconcileOrgSettings(grafanaOrg, keycloakOrganization.Name, grafanaClient, dashboards)
+		err = reconcileOrgSettings(config, grafanaOrg, keycloakOrganization.Name, grafanaClient, dashboards)
 		if err != nil {
 			return nil, err
 		}