Addressing Org Membership Inconsistency (#263)

* Addressing Organizational Potential Inconsistencies.

ChangeLog:
  - When Listing Orgs, making preferences an optional parameter.
  - If Orgs exist that the configured Grafana Admin does not have access to, prompts user to fix it or abort.
  - Adds Retry Logic to Grafana OpenAPI client as well as Orgs Listing.
  - Addressing code review comments

cli/backup/organizations.go

@@ -30,7 +30,6 @@ func newOrganizationsCommand() simplecobra.Commander {
 		},
 
 		InitCFunc: func(cd *simplecobra.Commandeer, r *support.RootCommand) error {
-			r.GrafanaSvc().InitOrganizations()
 			return nil
 		},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
@@ -53,26 +52,38 @@ func newOrganizationsListCmd() simplecobra.Commander {
 		Long:  description,
 		WithCFunc: func(cmd *cobra.Command, r *support.RootCommand) {
 			cmd.Aliases = []string{"l"}
+			cmd.PersistentFlags().BoolP("with-preferences", "", false, "when set to true, Attempts to retrieve Orgs Preferences (Warning, this is slow due to Grafana current API design)")
 		},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			filter := service.NewOrganizationFilter(parseOrganizationGlobalFlags(cd.CobraCommand)...)
-			slog.Info("Listing organizations for context", "context", config.Config().GetGDGConfig().GetContext())
-			rootCmd.TableObj.AppendHeader(table.Row{"id", "organization Name", "org slug ID", "HomeDashboardUID", "Theme", "WeekStart"})
-			listOrganizations := rootCmd.GrafanaSvc().ListOrganizations(filter)
+			includePreferences, _ := cd.CobraCommand.Flags().GetBool("with-preferences")
+
+			headerRow := table.Row{"id", "organization Name", "org slug ID"}
+			if includePreferences {
+				headerRow = append(headerRow, table.Row{"HomeDashboardUID", "Theme", "WeekStart"}...)
+			}
+			rootCmd.TableObj.AppendHeader(headerRow)
+			listOrganizations := rootCmd.GrafanaSvc().ListOrganizations(filter, includePreferences)
+			slog.Info("Listing organizations for context",
+				slog.Any("count", len(listOrganizations)),
+				slog.Any("context", config.Config().GetGDGConfig().GetContext()))
 			sort.Slice(listOrganizations, func(a, b int) bool {
 				return listOrganizations[a].Organization.ID < listOrganizations[b].Organization.ID
 			})
 			if len(listOrganizations) == 0 {
 				slog.Info("No organizations found")
 			} else {
 				for _, org := range listOrganizations {
-					rootCmd.TableObj.AppendRow(table.Row{org.Organization.ID,
+					data := table.Row{org.Organization.ID,
 						org.Organization.Name,
-						slug.Make(org.Organization.Name),
-						org.Preferences.HomeDashboardUID,
-						org.Preferences.Theme,
-						org.Preferences.WeekStart,
-					})
+						slug.Make(org.Organization.Name)}
+					if includePreferences {
+						data = append(data, table.Row{org.Preferences.HomeDashboardUID,
+							org.Preferences.Theme,
+							org.Preferences.WeekStart}...)
+					}
+					rootCmd.TableObj.AppendRow(data)
+
 				}
 				rootCmd.Render(cd.CobraCommand, listOrganizations)
 			}

cli/tools/organizations.go

@@ -208,7 +208,7 @@ func newAddUserRoleCmd() simplecobra.Commander {
 			if err != nil {
 				slog.Error("Unable to add user to Org")
 			} else {
-				slog.Info("User has been add to Org")
+				slog.Info("User has been add to Org", slog.Any("userId", userId), slog.String("organization", orgSlug))
 			}
 			return nil
 		},

config/importer-example.yml

@@ -112,3 +112,7 @@ contexts:
 global:
   debug: true
   ignore_ssl_errors: false ##when set to true will ignore invalid SSL errors
+  retry_count: 3 ## Will retry any failed API request up to 3 times.
+  retry_delay: 5s  ## will wait for specified duration before trying again.
+## Keep in mind longer the dealy and higher the count the slower GDG will be in performing certain tasks.
+## A failing endpoint that has 10s * 6 = 60 seconds minimum for each failing endpoint.  Use this carefully

go.mod

@@ -5,6 +5,7 @@ go 1.22.1
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.50.0
 	github.com/bep/simplecobra v0.4.0
 	github.com/carlmjohnson/requests v0.23.5
@@ -29,6 +30,7 @@ require (
 	github.com/zeitlinger/conflate v0.0.0-20230622100834-279724abda8c
 	gocloud.dev v0.36.0
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
+	golang.org/x/mod v0.15.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -177,7 +179,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4
 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.50.0 h1:HBtrLeO+QyDKnc3t1+5DR1RxodOHCGr8ZcrHudpv7jI=
 github.com/aws/aws-sdk-go v1.50.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
@@ -616,8 +618,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/api/orgs.go

@@ -3,19 +3,48 @@ package api
 import (
 	"context"
 	"errors"
+	"github.com/avast/retry-go"
+	"github.com/esnet/gdg/internal/config"
 	"github.com/grafana/grafana-openapi-client-go/models"
+	"log/slog"
 	"net/http"
+	"time"
 )
 
 // GetConfiguredOrgId needed to call grafana API in order to configure the Grafana API correctly.  Invoking
 // this endpoint manually to avoid a circular dependency.
 func (extended *ExtendedApi) GetConfiguredOrgId(orgName string) (int64, error) {
 	var result []*models.UserOrgDTO
-	err := extended.getRequestBuilder().
-		Path("api/user/orgs").
-		ToJSON(&result).
-		Method(http.MethodGet).
-		Fetch(context.Background())
+	fetch := func() error {
+		return extended.getRequestBuilder().
+			Path("api/user/orgs").
+			ToJSON(&result).
+			Method(http.MethodGet).
+			Fetch(context.Background())
+	}
+
+	/* There's something goofy here.  This seems to fail sporadically in grafana if we keep swapping orgs too fast.
+	   This is a safety check that should ideally never be triggered, but if the URL fails, then we retry a few times
+		before finally giving up.
+	*/
+	delay := time.Second * 5
+	var count uint = 5
+	//Giving user configured value preference over defaults
+	if config.Config().GetGDGConfig().GetAppGlobals().RetryCount != 0 {
+		count = uint(config.Config().GetGDGConfig().GetAppGlobals().RetryCount)
+	}
+	if config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout() != time.Millisecond*100 {
+		delay = config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout()
+	}
+	err := retry.Do(fetch,
+		retry.Attempts(count),
+		retry.Delay(delay),
+		retry.OnRetry(func(n uint, err error) {
+			slog.Info("Retrying request after error",
+				slog.String("orgName", orgName),
+				slog.Any("err", err))
+		}))
+
 	if err != nil {
 		return 0, err
 	}

internal/config/globals.go

@@ -1,7 +1,39 @@
 package config
 
+import (
+	"log/slog"
+	"time"
+)
+
 // AppGlobals is the global configuration for the application
 type AppGlobals struct {
-	Debug           bool `mapstructure:"debug" yaml:"debug"`
-	IgnoreSSLErrors bool `mapstructure:"ignore_ssl_errors" yaml:"ignore_ssl_errors"`
+	Debug           bool           `mapstructure:"debug" yaml:"debug"`
+	IgnoreSSLErrors bool           `mapstructure:"ignore_ssl_errors" yaml:"ignore_ssl_errors"`
+	RetryCount      int            `mapstructure:"retry_count" yaml:"retry_count"`
+	RetryDelay      string         `mapstructure:"retry_delay" yaml:"retry_delay"`
+	retryTimeout    *time.Duration `mapstructure:"-" yaml:"-"`
+}
+
+// GetRetryTimeout returns 100ms, by default otherwise the parsed value
+func (app *AppGlobals) GetRetryTimeout() time.Duration {
+	defaultBehavior := func() {
+		d := time.Millisecond * 100
+		app.retryTimeout = &d
+	}
+	if app.RetryDelay == "" {
+		defaultBehavior()
+	}
+	if app.retryTimeout != nil {
+		return *app.retryTimeout
+	}
+	d, err := time.ParseDuration(app.RetryDelay)
+	if err != nil {
+		slog.Warn("Unable to parse the retry_delay value.  Falling back on default to 100ms")
+		defaultBehavior()
+	} else {
+		app.retryTimeout = &d
+	}
+
+	return *app.retryTimeout
+
 }

internal/service/contracts.go

@@ -85,7 +85,7 @@ type OrgPreferencesApi interface {
 }
 
 type organizationCrudApi interface {
-	ListOrganizations(filter filters.Filter) []*gdgType.OrgsDTOWithPreferences
+	ListOrganizations(filter filters.Filter, withPreferences bool) []*gdgType.OrgsDTOWithPreferences
 	DownloadOrganizations(filter filters.Filter) []string
 	UploadOrganizations(filter filters.Filter) []string
 }

internal/service/dashboards.go

@@ -25,6 +25,62 @@ import (
 	"github.com/thoas/go-funk"
 )
 
+func NewDashboardFilter(entries ...string) filters.Filter {
+	if len(entries) != 3 {
+		log.Fatalf("Unable to create a valid Dashboard Filter, aborting.")
+	}
+	folderFilter := entries[0]
+	dashboardFilter := entries[1]
+	tagsFilter := entries[2]
+	if tagsFilter == "" {
+		tagsFilter = "[]"
+	}
+
+	filterObj := filters.NewBaseFilter()
+	filterObj.AddFilter(filters.FolderFilter, folderFilter)
+	filterObj.AddFilter(filters.DashFilter, dashboardFilter)
+	filterObj.AddFilter(filters.TagsFilter, tagsFilter)
+	quoteRegex, _ := regexp.Compile("['\"]+")
+	filterObj.AddRegex(filters.FolderFilter, quoteRegex)
+	//Add Folder Validation
+	filterObj.AddValidation(filters.FolderFilter, func(i interface{}) bool {
+		val, ok := i.(map[filters.FilterType]string)
+		if !ok {
+			return ok
+		}
+		//Check folder
+		if folderFilter, ok = val[filters.FolderFilter]; ok {
+			if filterObj.GetFilter(filters.FolderFilter) == "" {
+				return true
+			} else {
+				return folderFilter == filterObj.GetFilter(filters.FolderFilter)
+			}
+		} else {
+			return true
+		}
+	})
+
+	//Add DashValidation
+	filterObj.AddValidation(filters.DashFilter, func(i interface{}) bool {
+		val, ok := i.(map[filters.FilterType]string)
+		if !ok {
+			return ok
+		}
+
+		if dashboardFilter, ok = val[filters.DashFilter]; ok {
+			if filterObj.GetFilter(filters.DashFilter) == "" {
+				return true
+			}
+			return dashboardFilter == filterObj.GetFilter(filters.DashFilter)
+		} else {
+			return true
+		}
+
+	})
+
+	return filterObj
+}
+
 func (s *DashNGoImpl) LintDashboards(req types.LintRequest) []string {
 	var (
 		rawBoard []byte
@@ -131,62 +187,6 @@ func (s *DashNGoImpl) getDashboardByUid(uid string) (*models.DashboardFullWithMe
 
 }
 
-func NewDashboardFilter(entries ...string) filters.Filter {
-	if len(entries) != 3 {
-		log.Fatalf("Unable to create a valid Dashboard Filter, aborting.")
-	}
-	folderFilter := entries[0]
-	dashboardFilter := entries[1]
-	tagsFilter := entries[2]
-	if tagsFilter == "" {
-		tagsFilter = "[]"
-	}
-
-	filterObj := filters.NewBaseFilter()
-	filterObj.AddFilter(filters.FolderFilter, folderFilter)
-	filterObj.AddFilter(filters.DashFilter, dashboardFilter)
-	filterObj.AddFilter(filters.TagsFilter, tagsFilter)
-	quoteRegex, _ := regexp.Compile("['\"]+")
-	filterObj.AddRegex(filters.FolderFilter, quoteRegex)
-	//Add Folder Validation
-	filterObj.AddValidation(filters.FolderFilter, func(i interface{}) bool {
-		val, ok := i.(map[filters.FilterType]string)
-		if !ok {
-			return ok
-		}
-		//Check folder
-		if folderFilter, ok = val[filters.FolderFilter]; ok {
-			if filterObj.GetFilter(filters.FolderFilter) == "" {
-				return true
-			} else {
-				return folderFilter == filterObj.GetFilter(filters.FolderFilter)
-			}
-		} else {
-			return true
-		}
-	})
-
-	//Add DashValidation
-	filterObj.AddValidation(filters.DashFilter, func(i interface{}) bool {
-		val, ok := i.(map[filters.FilterType]string)
-		if !ok {
-			return ok
-		}
-
-		if dashboardFilter, ok = val[filters.DashFilter]; ok {
-			if filterObj.GetFilter(filters.DashFilter) == "" {
-				return true
-			}
-			return dashboardFilter == filterObj.GetFilter(filters.DashFilter)
-		} else {
-			return true
-		}
-
-	})
-
-	return filterObj
-}
-
 // ListDashboards List all dashboards optionally filtered by folder name. If folderFilters
 // is blank, defaults to the configured Monitored folders
 func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {

internal/service/login.go

@@ -45,10 +45,11 @@ func (s *DashNGoImpl) getNewClient(opts ...NewClientOpts) (*client.GrafanaHTTPAP
 	}
 
 	httpConfig := &client.TransportConfig{
-		Host:     u.Host,
-		BasePath: path,
-		Schemes:  []string{u.Scheme},
-		// NumRetries: 3,
+		Host:         u.Host,
+		BasePath:     path,
+		Schemes:      []string{u.Scheme},
+		NumRetries:   config.Config().GetGDGConfig().GetAppGlobals().RetryCount,
+		RetryTimeout: config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout(),
 	}
 
 	if s.grafanaConf.OrganizationName != "" {