diff --git a/gen-dev-env.sh b/gen-dev-env.sh index 04f2f99..825b486 100755 --- a/gen-dev-env.sh +++ b/gen-dev-env.sh @@ -12,3 +12,6 @@ echo "export GRAFANA_USERNAME=\"admin\"" >> env echo -n "export GRAFANA_PASSWORD=\"" >> env kubectl --as cluster-admin -n vshn-grafana-organizations-operator-dev get secret grafana-env -ojsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d >> env echo "\"" >> env +echo "export GRAFANA_DATASOURCE_URL=\"http://vshn-appuio-mimir-nginx.vshn-appuio-mimir.svc.cluster.local/prometheus\"" >> env +echo "export GRAFANA_DATASOURCE_USERNAME=\"dummyuser\"" >> env +echo "export GRAFANA_DATASOURCE_PASSWORD=\"dummypass\"" >> env diff --git a/main.go b/main.go index 300c8e7..6114557 100644 --- a/main.go +++ b/main.go @@ -18,66 +18,64 @@ import ( "time" ) -var ( - GrafanaUrl string - GrafanaUsername string - GrafanaPassword string - KeycloakUrl string - KeycloakRealm string - KeycloakUsername string - KeycloakPassword string - KeycloakClientId string - KeycloakAdminGroupPath string - KeycloakAutoAssignOrgGroupPath string -) - func main() { - GrafanaUrl = os.Getenv("GRAFANA_URL") - GrafanaUsername = os.Getenv("GRAFANA_USERNAME") - if GrafanaUsername == "" { - GrafanaUsername = os.Getenv("admin-user") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells. - } - GrafanaPassword = os.Getenv("GRAFANA_PASSWORD") - GrafanaPasswordHidden := "" - if GrafanaPassword == "" { - GrafanaPassword = os.Getenv("admin-password") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells. - } - if GrafanaPassword != "" { - GrafanaPasswordHidden = "***hidden***" - } - - KeycloakUrl = os.Getenv("KEYCLOAK_URL") - KeycloakRealm = os.Getenv("KEYCLOAK_REALM") - KeycloakUsername = os.Getenv("KEYCLOAK_USERNAME") - KeycloakPassword = os.Getenv("KEYCLOAK_PASSWORD") - KeycloakClientId = os.Getenv("KEYCLOAK_CLIENT_ID") - KeycloakPasswordHidden := "" - if KeycloakPassword != "" { - KeycloakPasswordHidden = "***hidden***" - } - KeycloakAdminGroupPath = os.Getenv("KEYCLOAK_ADMIN_GROUP_PATH") - KeycloakAutoAssignOrgGroupPath = os.Getenv("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH") - - klog.Infof("GRAFANA_URL: %s\n", GrafanaUrl) - klog.Infof("GRAFANA_USERNAME: %s\n", GrafanaUsername) - klog.Infof("GRAFANA_PASSWORD: %s\n", GrafanaPasswordHidden) - klog.Infof("KEYCLOAK_URL: %s\n", KeycloakUrl) - klog.Infof("KEYCLOAK_REALM: %s\n", KeycloakRealm) - klog.Infof("KEYCLOAK_USERNAME: %s\n", KeycloakUsername) - klog.Infof("KEYCLOAK_PASSWORD: %s\n", KeycloakPasswordHidden) - klog.Infof("KEYCLOAK_CLIENT_ID: %s\n", KeycloakClientId) - klog.Infof("KEYCLOAK_ADMIN_GROUP_PATH: %s\n", KeycloakAdminGroupPath) - klog.Infof("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH: %s\n", KeycloakAutoAssignOrgGroupPath) - - keycloakClient, err := controller.NewKeycloakClient(KeycloakUrl, KeycloakRealm, KeycloakUsername, KeycloakPassword, KeycloakClientId, KeycloakAdminGroupPath, KeycloakAutoAssignOrgGroupPath) + config := controller.Config{} + grafanaUrl := os.Getenv("GRAFANA_URL") + grafanaUsername := os.Getenv("GRAFANA_USERNAME") + if grafanaUsername == "" { + grafanaUsername = os.Getenv("admin-user") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells. + } + grafanaPassword := os.Getenv("GRAFANA_PASSWORD") + grafanaPasswordHidden := "" + if grafanaPassword == "" { + grafanaPassword = os.Getenv("admin-password") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells. + } + if grafanaPassword != "" { + grafanaPasswordHidden = "***hidden***" + } + config.GrafanaDatasourceUrl = os.Getenv("GRAFANA_DATASOURCE_URL") + config.GrafanaDatasourceUsername = os.Getenv("GRAFANA_DATASOURCE_USERNAME") + config.GrafanaDatasourcePassword = os.Getenv("GRAFANA_DATASOURCE_PASSWORD") + grafanaDatasourcePasswordHidden := "" + if config.GrafanaDatasourcePassword != "" { + grafanaDatasourcePasswordHidden = "***hidden***" + } + + keycloakUrl := os.Getenv("KEYCLOAK_URL") + keycloakRealm := os.Getenv("KEYCLOAK_REALM") + keycloakUsername := os.Getenv("KEYCLOAK_USERNAME") + keycloakPassword := os.Getenv("KEYCLOAK_PASSWORD") + keycloakClientId := os.Getenv("KEYCLOAK_CLIENT_ID") + keycloakPasswordHidden := "" + if keycloakPassword != "" { + keycloakPasswordHidden = "***hidden***" + } + keycloakAdminGroupPath := os.Getenv("KEYCLOAK_ADMIN_GROUP_PATH") + keycloakAutoAssignOrgGroupPath := os.Getenv("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH") + + klog.Infof("GRAFANA_URL: %s\n", grafanaUrl) + klog.Infof("GRAFANA_USERNAME: %s\n", grafanaUsername) + klog.Infof("GRAFANA_PASSWORD: %s\n", grafanaPasswordHidden) + klog.Infof("GRAFANA_DATASOURCE_URL: %s\n", config.GrafanaDatasourceUrl) + klog.Infof("GRAFANA_DATASOURCE_USERNAME: %s\n", config.GrafanaDatasourceUsername) + klog.Infof("GRAFANA_DATASOURCE_PASSWORD: %s\n", grafanaDatasourcePasswordHidden) + klog.Infof("KEYCLOAK_URL: %s\n", keycloakUrl) + klog.Infof("KEYCLOAK_REALM: %s\n", keycloakRealm) + klog.Infof("KEYCLOAK_USERNAME: %s\n", keycloakUsername) + klog.Infof("KEYCLOAK_PASSWORD: %s\n", keycloakPasswordHidden) + klog.Infof("KEYCLOAK_CLIENT_ID: %s\n", keycloakClientId) + klog.Infof("KEYCLOAK_ADMIN_GROUP_PATH: %s\n", keycloakAdminGroupPath) + klog.Infof("KEYCLOAK_AUTO_ASSIGN_ORG_GROUP_PATH: %s\n", keycloakAutoAssignOrgGroupPath) + + keycloakClient, err := controller.NewKeycloakClient(keycloakUrl, keycloakRealm, keycloakUsername, keycloakPassword, keycloakClientId, keycloakAdminGroupPath, keycloakAutoAssignOrgGroupPath) if err != nil { klog.Errorf("Could not create keycloakClient client: %v\n", err) os.Exit(1) } defer keycloakClient.CloseIdleConnections() - grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(GrafanaUsername, GrafanaPassword)} - grafanaClient, err := controller.NewGrafanaClient(GrafanaUrl, grafanaConfig) + grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(grafanaUsername, grafanaPassword)} + grafanaClient, err := controller.NewGrafanaClient(grafanaUrl, grafanaConfig) if err != nil { klog.Errorf("Could not create Grafana client: %v\n", err) os.Exit(1) @@ -104,14 +102,14 @@ func main() { } klog.Info("Starting initial sync...") - err = controller.Reconcile(ctx, keycloakClient, grafanaClient, dashboards) + err = controller.Reconcile(ctx, config, keycloakClient, grafanaClient, dashboards) if err != nil { klog.Errorf("Could not do initial reconciliation: %v\n", err) os.Exit(1) } for { - err = controller.Reconcile(ctx, keycloakClient, grafanaClient, dashboards) + err = controller.Reconcile(ctx, config, keycloakClient, grafanaClient, dashboards) select { case <-time.After(2 * time.Second): case <-ctx.Done(): diff --git a/pkg/reconcile.go b/pkg/reconcile.go index 7d1e2fa..bcc631c 100644 --- a/pkg/reconcile.go +++ b/pkg/reconcile.go @@ -6,11 +6,17 @@ import ( "k8s.io/klog/v2" ) +type Config struct { + GrafanaDatasourceUrl string + GrafanaDatasourceUsername string + GrafanaDatasourcePassword string +} + var ( interruptedError = errors.New("interrupted") ) -func Reconcile(ctx context.Context, keycloakClient *KeycloakClient, grafanaClient *GrafanaClient, dashboards []Dashboard) error { +func Reconcile(ctx context.Context, config Config, keycloakClient *KeycloakClient, grafanaClient *GrafanaClient, dashboards []Dashboard) error { klog.Infof("Fetching Keycloak access token...") keycloakToken, err := keycloakClient.GetToken() if err != nil { @@ -74,7 +80,7 @@ outAutoAssignOrgUsers: } klog.Infof("Found %d auto_assign_org users", len(keycloakAutoAssignOrgUsers)) - grafanaOrgsMap, err := reconcileAllOrgs(ctx, keycloakOrganizations, grafanaClient, dashboards) + grafanaOrgsMap, err := reconcileAllOrgs(ctx, config, keycloakOrganizations, grafanaClient, dashboards) if err != nil { return err } diff --git a/pkg/reconcileOrg.go b/pkg/reconcileOrg.go index 092a288..b2c311d 100644 --- a/pkg/reconcileOrg.go +++ b/pkg/reconcileOrg.go @@ -38,8 +38,8 @@ func reconcileOrgBasic(grafanaOrgLookup map[string]grafana.Org, grafanaClient *G return &grafanaOrg, nil } -func reconcileOrgSettings(org *grafana.Org, orgName string, grafanaClient *GrafanaClient, dashboards []Dashboard) error { - dataSource, err := reconcileOrgDataSource(org, orgName, grafanaClient) +func reconcileOrgSettings(config Config, org *grafana.Org, orgName string, grafanaClient *GrafanaClient, dashboards []Dashboard) error { + dataSource, err := reconcileOrgDataSource(config, org, orgName, grafanaClient) if err != nil { return err } @@ -53,23 +53,31 @@ func reconcileOrgSettings(org *grafana.Org, orgName string, grafanaClient *Grafa return nil } -func reconcileOrgDataSource(org *grafana.Org, orgName string, grafanaClient *GrafanaClient) (*grafana.DataSource, error) { +func reconcileOrgDataSource(config Config, org *grafana.Org, orgName string, grafanaClient *GrafanaClient) (*grafana.DataSource, error) { + secureJSONData := map[string]interface{}{ + "httpHeaderValue1": orgName, + } + basicAuth := config.GrafanaDatasourceUsername != "" + if basicAuth { + secureJSONData["basicAuthPassword"] = config.GrafanaDatasourcePassword + } + // If you add/remove fields here you must also adjust the 'if' statement further down desiredDataSource := &grafana.DataSource{ - Name: "Mimir", - URL: "http://vshn-appuio-mimir-query-frontend.vshn-appuio-mimir.svc.cluster.local:8080/prometheus", - OrgID: org.ID, // doesn't actually do anything, we just keep it here in case it becomes relevant with some never version of the client library. The actual orgId is taken from the 'X-Grafana-Org-Id' HTTP header which is set up via grafanaConfig.OrgID - Type: "prometheus", - IsDefault: true, + Name: "Mimir", + URL: config.GrafanaDatasourceUrl, + BasicAuth: basicAuth, + BasicAuthUser: config.GrafanaDatasourceUsername, + OrgID: org.ID, // doesn't actually do anything, we just keep it here in case it becomes relevant with some never version of the client library. The actual orgId is taken from the 'X-Grafana-Org-Id' HTTP header which is set up via grafanaConfig.OrgID + Type: "prometheus", + IsDefault: true, JSONData: map[string]interface{}{ "httpHeaderName1": "X-Scope-OrgID", "httpMethod": "POST", "prometheusType": "Mimir", }, - SecureJSONData: map[string]interface{}{ - "httpHeaderValue1": orgName, - }, - Access: "proxy", + SecureJSONData: secureJSONData, + Access: "proxy", } var configuredDataSource *grafana.DataSource @@ -82,10 +90,12 @@ func reconcileOrgDataSource(org *grafana.Org, orgName string, grafanaClient *Gra for _, dataSource := range dataSources { if dataSource.Name == desiredDataSource.Name { if dataSource.URL != desiredDataSource.URL || + dataSource.BasicAuth != desiredDataSource.BasicAuth || dataSource.Type != desiredDataSource.Type || dataSource.IsDefault != desiredDataSource.IsDefault || !reflect.DeepEqual(dataSource.JSONData, desiredDataSource.JSONData) || dataSource.Access != desiredDataSource.Access { + // note that we can't detect changed basic auth credentials (BasicAuthUser, secureJSONData) because the API does not give us the current settings klog.Infof("Organization %d has misconfigured data source, fixing", org.ID) desiredDataSource.ID = dataSource.ID desiredDataSource.UID = dataSource.UID @@ -99,7 +109,7 @@ func reconcileOrgDataSource(org *grafana.Org, orgName string, grafanaClient *Gra } } else { klog.Infof("Organization %d has invalid data source %d %s, removing", org.ID, dataSource.ID, dataSource.Name) - grafanaClient.DeleteDataSource(org, dataSource.ID) + err = grafanaClient.DeleteDataSource(org, dataSource.ID) if err != nil { return nil, err } @@ -128,7 +138,7 @@ func reconcileOrgDashboard(org *grafana.Org, dataSource *grafana.DataSource, gra dashboardTitle, ok := dashboard.Data["title"] if !ok { - errors.New("Invalid dashboard format: 'title' key not found") + return errors.New("Invalid dashboard format: 'title' key not found") } dashboards, err := grafanaClient.Dashboards(org) @@ -147,12 +157,6 @@ func reconcileOrgDashboard(org *grafana.Org, dataSource *grafana.DataSource, gra } } - // FIXME not required? - /*err = configureDashboard(dashboard.Data, dataSource) - if err != nil { - return err - }*/ - db := grafana.Dashboard{ Model: dashboard.Data, Overwrite: true, diff --git a/pkg/reconcileOrgs.go b/pkg/reconcileOrgs.go index 20993cc..052a72c 100644 --- a/pkg/reconcileOrgs.go +++ b/pkg/reconcileOrgs.go @@ -7,7 +7,7 @@ import ( "strings" ) -func reconcileAllOrgs(ctx context.Context, keycloakOrganizations []*KeycloakGroup, grafanaClient *GrafanaClient, dashboards []Dashboard) (map[string]*grafana.Org, error) { +func reconcileAllOrgs(ctx context.Context, config Config, keycloakOrganizations []*KeycloakGroup, grafanaClient *GrafanaClient, dashboards []Dashboard) (map[string]*grafana.Org, error) { grafanaOrgLookupFinal := make(map[string]*grafana.Org) // Get all orgs from Grafana @@ -34,7 +34,7 @@ func reconcileAllOrgs(ctx context.Context, keycloakOrganizations []*KeycloakGrou } delete(grafanaOrgLookup, keycloakOrganization.Name) - err = reconcileOrgSettings(grafanaOrg, keycloakOrganization.Name, grafanaClient, dashboards) + err = reconcileOrgSettings(config, grafanaOrg, keycloakOrganization.Name, grafanaClient, dashboards) if err != nil { return nil, err }