Saving things

cli/backup/organizations.go

@@ -30,7 +30,6 @@ func newOrganizationsCommand() simplecobra.Commander {
 		},
 
 		InitCFunc: func(cd *simplecobra.Commandeer, r *support.RootCommand) error {
-			r.GrafanaSvc().InitOrganizations()
 			return nil
 		},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
@@ -53,26 +52,42 @@ func newOrganizationsListCmd() simplecobra.Commander {
 		Long:  description,
 		WithCFunc: func(cmd *cobra.Command, r *support.RootCommand) {
 			cmd.Aliases = []string{"l"}
+			cmd.PersistentFlags().BoolP("with-preferences", "", false, "when set to true, Attempts to retrieve Orgs Preferences (Warning, this is slow due to Grafana current API design)")
 		},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			filter := service.NewOrganizationFilter(parseOrganizationGlobalFlags(cd.CobraCommand)...)
-			slog.Info("Listing organizations for context", "context", config.Config().GetGDGConfig().GetContext())
-			rootCmd.TableObj.AppendHeader(table.Row{"id", "organization Name", "org slug ID", "HomeDashboardUID", "Theme", "WeekStart"})
-			listOrganizations := rootCmd.GrafanaSvc().ListOrganizations(filter)
+			includePreferences, _ := cd.CobraCommand.Flags().GetBool("with-preferences")
+
+			if includePreferences {
+				rootCmd.TableObj.AppendHeader(table.Row{"id", "organization Name", "org slug ID", "HomeDashboardUID", "Theme", "WeekStart"})
+			} else {
+				rootCmd.TableObj.AppendHeader(table.Row{"id", "organization Name", "org slug ID"})
+			}
+			listOrganizations := rootCmd.GrafanaSvc().ListOrganizations(filter, includePreferences)
+			slog.Info("Listing organizations for context",
+				slog.Any("count", len(listOrganizations)),
+				slog.Any("context", config.Config().GetGDGConfig().GetContext()))
 			sort.Slice(listOrganizations, func(a, b int) bool {
 				return listOrganizations[a].Organization.ID < listOrganizations[b].Organization.ID
 			})
 			if len(listOrganizations) == 0 {
 				slog.Info("No organizations found")
 			} else {
 				for _, org := range listOrganizations {
-					rootCmd.TableObj.AppendRow(table.Row{org.Organization.ID,
-						org.Organization.Name,
-						slug.Make(org.Organization.Name),
-						org.Preferences.HomeDashboardUID,
-						org.Preferences.Theme,
-						org.Preferences.WeekStart,
-					})
+					if includePreferences {
+						rootCmd.TableObj.AppendRow(table.Row{org.Organization.ID,
+							org.Organization.Name,
+							slug.Make(org.Organization.Name),
+							org.Preferences.HomeDashboardUID,
+							org.Preferences.Theme,
+							org.Preferences.WeekStart,
+						})
+					} else {
+						rootCmd.TableObj.AppendRow(table.Row{org.Organization.ID,
+							org.Organization.Name,
+							slug.Make(org.Organization.Name)})
+					}
+
 				}
 				rootCmd.Render(cd.CobraCommand, listOrganizations)
 			}

cli/tools/dashboards.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"github.com/bep/simplecobra"
 	"github.com/esnet/gdg/cli/support"
-	"github.com/esnet/gdg/internal/service"
+	"github.com/esnet/gdg/internal/service/types"
 	"github.com/spf13/cobra"
 	"log/slog"
 )
@@ -45,7 +45,7 @@ func newDashboardLintCmd() simplecobra.Commander {
 			slog.Info("This is mainly provided as a convenience GDG, if you find yourself relying on this tool heavily, please have a look at: github.com/grafana/dashboard-linter/")
 			dashboard, _ := cd.CobraCommand.Flags().GetString("dashboard")
 			folder, _ := cd.CobraCommand.Flags().GetString("folder")
-			filterReq := service.LintRequest{
+			filterReq := types.LintRequest{
 				StrictFlag:    lintStrictFlag,
 				VerboseFlag:   lintVerboseFlag,
 				AutoFix:       lintAutofixFlag,

config/importer-example.yml

@@ -112,3 +112,7 @@ contexts:
 global:
   debug: true
   ignore_ssl_errors: false ##when set to true will ignore invalid SSL errors
+  retry_count: 3 ## Will retry any failed API request up to 3 times.
+  retry_delay: 5s  ## will wait for specified duration before trying again.
+## Keep in mind longer the dealy and higher the count the slower GDG will be in performing certain tasks.
+## A failing endpoint that has 10s * 6 = 60 seconds minimum for each failing endpoint.  Use this carefully

go.mod

@@ -5,6 +5,7 @@ go 1.22.1
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.50.0
 	github.com/bep/simplecobra v0.4.0
 	github.com/carlmjohnson/requests v0.23.5
@@ -29,6 +30,7 @@ require (
 	github.com/zeitlinger/conflate v0.0.0-20230622100834-279724abda8c
 	gocloud.dev v0.36.0
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
+	golang.org/x/mod v0.16.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -177,7 +179,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4
 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.50.0 h1:HBtrLeO+QyDKnc3t1+5DR1RxodOHCGr8ZcrHudpv7jI=
 github.com/aws/aws-sdk-go v1.50.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
@@ -498,8 +500,8 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -616,8 +618,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/api/orgs.go

@@ -3,19 +3,48 @@ package api
 import (
 	"context"
 	"errors"
+	"github.com/avast/retry-go"
+	"github.com/esnet/gdg/internal/config"
 	"github.com/grafana/grafana-openapi-client-go/models"
+	"log/slog"
 	"net/http"
+	"time"
 )
 
 // GetConfiguredOrgId needed to call grafana API in order to configure the Grafana API correctly.  Invoking
 // this endpoint manually to avoid a circular dependency.
 func (extended *ExtendedApi) GetConfiguredOrgId(orgName string) (int64, error) {
 	var result []*models.UserOrgDTO
-	err := extended.getRequestBuilder().
-		Path("api/user/orgs").
-		ToJSON(&result).
-		Method(http.MethodGet).
-		Fetch(context.Background())
+	fetch := func() error {
+		return extended.getRequestBuilder().
+			Path("api/user/orgs").
+			ToJSON(&result).
+			Method(http.MethodGet).
+			Fetch(context.Background())
+	}
+
+	/* There's something goofy here.  This seems to fail sporadically in grafana if we keep swapping orgs too fast.
+	   This is a safety check that should ideally never be triggered, but if the URL fails, then we retry a few times
+		before finally giving up.
+	*/
+	delay := time.Second * 5
+	var count uint = 5
+	//Giving user configured value preference over defaults
+	if config.Config().GetGDGConfig().GetAppGlobals().RetryCount != 0 {
+		count = uint(config.Config().GetGDGConfig().GetAppGlobals().RetryCount)
+	}
+	if config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout() != time.Millisecond*100 {
+		delay = config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout()
+	}
+	err := retry.Do(fetch,
+		retry.Attempts(count),
+		retry.Delay(delay),
+		retry.OnRetry(func(n uint, err error) {
+			slog.Info("Retrying request after error",
+				slog.String("orgName", orgName),
+				slog.Any("err", err))
+		}))
+
 	if err != nil {
 		return 0, err
 	}