From 1646298b1f04f978ccfc55b91b2160f3b2e429e4 Mon Sep 17 00:00:00 2001 From: malcolmholmes <42545407+malcolmholmes@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:38:00 +0000 Subject: [PATCH] Support output formats (#309) * Support configurable output formats * Remove legacy, support --only-spec --- cmd/grr/workflow.go | 31 +++++--- pkg/config/config.go | 6 ++ pkg/config/model.go | 32 ++++---- pkg/grafana/alertgroup-handler.go | 5 -- pkg/grafana/contactpoint-handler.go | 5 -- pkg/grafana/dashboard-handler.go | 5 -- pkg/grafana/datasource-handler.go | 5 -- pkg/grafana/folder-handler.go | 5 -- pkg/grafana/library-elements-handler.go | 5 -- pkg/grafana/notificationpolicy-handler.go | 5 -- pkg/grafana/rules-handler.go | 5 -- pkg/grafana/synthetic-monitoring-handler.go | 5 -- pkg/grizzly/config.go | 6 +- pkg/grizzly/formatting.go | 65 ++++++++++++++++ pkg/grizzly/parsing.go | 34 +-------- pkg/grizzly/providers.go | 12 ++- pkg/grizzly/workflow.go | 84 +++++++++++++++------ 17 files changed, 186 insertions(+), 129 deletions(-) create mode 100644 pkg/grizzly/formatting.go diff --git a/cmd/grr/workflow.go b/cmd/grr/workflow.go index 9c458652..e7d220e5 100644 --- a/cmd/grr/workflow.go +++ b/cmd/grr/workflow.go @@ -20,13 +20,14 @@ func getCmd() *cli.Command { Short: "retrieve resource", Args: cli.ArgsExact(1), } - var opts grizzly.LoggingOpts + var opts grizzly.Opts cmd.Run = func(cmd *cli.Command, args []string) error { uid := args[0] - return grizzly.Get(uid) + return grizzly.Get(uid, opts) } - return initialiseLogging(cmd, &opts) + cmd = initialiseOnlySpec(cmd, &opts) + return initialiseCmd(cmd, &opts) } func listCmd() *cli.Command { @@ -77,7 +78,7 @@ func pullCmd() *cli.Command { return grizzly.Pull(args[0], opts) } - cmd.Flags().BoolVarP(&opts.JSONSpec, "only-spec", "s", false, "this flag is only used for dashboards to output the spec") + cmd = initialiseOnlySpec(cmd, &opts) return initialiseCmd(cmd, &opts) } @@ -94,7 +95,7 @@ func showCmd() *cli.Command { if err != nil { return err } - return grizzly.Show(resources) + return grizzly.Show(resources, opts) } return initialiseCmd(cmd, &opts) } @@ -112,7 +113,7 @@ func diffCmd() *cli.Command { if err != nil { return err } - return grizzly.Diff(resources) + return grizzly.Diff(resources, opts) } return initialiseCmd(cmd, &opts) } @@ -139,7 +140,7 @@ func applyCmd() *cli.Command { } cmd.Flags().StringVarP(&opts.FolderUID, "folder", "f", generalFolderUID, "folder to push dashboards to") - cmd.Flags().BoolVarP(&opts.JSONSpec, "only-spec", "s", false, "this flag is only used for dashboards to output the spec") + cmd = initialiseOnlySpec(cmd, &opts) return initialiseCmd(cmd, &opts) } @@ -157,7 +158,7 @@ func targetsOfKind(kind string, opts grizzly.Opts) bool { // checkDashboardTarget ensures that the specified targets are of dashboards kind func checkDashboardTarget(opts grizzly.Opts) error { ok := targetsOfKind("Dashboard", opts) - if opts.JSONSpec && !ok { + if opts.OnlySpec && !ok { return fmt.Errorf("-s flag is only supported for dashboards") } @@ -236,7 +237,7 @@ func exportCmd() *cli.Command { if err != nil { return err } - return grizzly.Export(dashboardDir, resources) + return grizzly.Export(dashboardDir, resources, opts) } return initialiseCmd(cmd, &opts) } @@ -287,10 +288,22 @@ func initialiseCmd(cmd *cli.Command, opts *grizzly.Opts) *cli.Command { cmd.Flags().StringSliceVarP(&opts.Targets, "target", "t", nil, "resources to target") cmd.Flags().StringSliceVarP(&opts.JsonnetPaths, "jpath", "J", getDefaultJsonnetFolders(), "Specify an additional library search dir (right-most wins)") + cmd.Flags().StringVarP(&opts.OutputFormat, "output", "o", "", "Output format") return initialiseLogging(cmd, &opts.LoggingOpts) } +func initialiseOnlySpec(cmd *cli.Command, opts *grizzly.Opts) *cli.Command { + cmd.Flags().BoolVarP(&opts.OnlySpec, "only-spec", "s", false, "this flag is only used for dashboards to output the spec") + cmdRun := cmd.Run + cmd.Run = func(cmd *cli.Command, args []string) error { + opts.HasOnlySpec = cmd.Flags().Changed("only-spec") + return cmdRun(cmd, args) + } + + return cmd +} + func initialiseLogging(cmd *cli.Command, loggingOpts *grizzly.LoggingOpts) *cli.Command { cmd.Flags().StringVarP(&loggingOpts.LogLevel, "log-level", "l", log.InfoLevel.String(), "info, debug, warning, error") cmdRun := cmd.Run diff --git a/pkg/config/config.go b/pkg/config/config.go index 988486cb..ac7f7728 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -155,6 +155,8 @@ var acceptableKeys = map[string]string{ "synthetic-monitoring.metrics-id": "string", "synthetic-monitoring.logs-id": "string", "targets": "[]string", + "output-format": "string", + "only-spec": "bool", } func Set(path string, value string) error { @@ -168,6 +170,10 @@ func Set(path string, value string) error { val = value case "[]string": val = strings.Split(value, ",") + case "bool": + val = strings.ToLower(value) == "true" + default: + return fmt.Errorf("Unknown config key type %s for key %s", typ, key) } viper.Set(fullPath, val) return Write() diff --git a/pkg/config/model.go b/pkg/config/model.go index 70f5cc93..4f25bb46 100644 --- a/pkg/config/model.go +++ b/pkg/config/model.go @@ -1,28 +1,30 @@ package config type GrafanaConfig struct { - URL string `yaml:"url"` - User string `yaml:"user"` - Token string `yaml:"token"` + URL string `yaml:"url" mapstructure:"url"` + User string `yaml:"user" mapstructure:"user"` + Token string `yaml:"token" mapstructure:"token"` } type MimirConfig struct { - Address string `yaml:"address"` - TenantID int64 `yaml:"tenant-id"` - ApiKey string `yaml:"api-key"` + Address string `yaml:"address" mapstructure:"address"` + TenantID int64 `yaml:"tenant-id" mapstructure:"tenant-id"` + ApiKey string `yaml:"api-key" mapstructure:"api-key"` } type SyntheticMonitoringConfig struct { - Token string `yaml:"token"` - StackID int64 `yaml:"stack-id"` - LogsID int64 `yaml:"logs-id"` - MetricsID int64 `yaml:"metrics-id"` + Token string `yaml:"token" mapstructure:"token"` + StackID int64 `yaml:"stack-id" mapstructure:"stack-id"` + LogsID int64 `yaml:"logs-id" mapstructure:"logs-id"` + MetricsID int64 `yaml:"metrics-id" mapstructure:"metrics-id"` } type Context struct { - Name string `yaml:"name"` - Grafana GrafanaConfig `yaml:"grafana"` - Mimir MimirConfig `yaml:"mimir"` - SyntheticMonitoring SyntheticMonitoringConfig `yaml:"synthetic-monitoring"` - Targets []string `yaml:"targets"` + Name string `yaml:"name" mapstructure:"name"` + Grafana GrafanaConfig `yaml:"grafana" mapstructure:"grafana"` + Mimir MimirConfig `yaml:"mimir" mapstructure:"mimir"` + SyntheticMonitoring SyntheticMonitoringConfig `yaml:"synthetic-monitoring" mapstructure:"synthetic-monitoring"` + Targets []string `yaml:"targets" mapstructure:"targets"` + OutputFormat string `yaml:"output-format" mapstructure:"output-format"` + OnlySpec bool `yaml:"only-spec" mapstructure:"only-spec"` } diff --git a/pkg/grafana/alertgroup-handler.go b/pkg/grafana/alertgroup-handler.go index 479ace7e..0b7033c0 100644 --- a/pkg/grafana/alertgroup-handler.go +++ b/pkg/grafana/alertgroup-handler.go @@ -55,11 +55,6 @@ func (h *AlertRuleGroupHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a alertRuleGroup -func (h *AlertRuleGroupHandler) GetExtension() string { - return "json" -} - const ( alertRuleGroupGlob = "alert-rules/alertRuleGroup-*" alertRuleGroupPattern = "alert-rules/alertRuleGroup-%s.%s" diff --git a/pkg/grafana/contactpoint-handler.go b/pkg/grafana/contactpoint-handler.go index 8dc105f7..65081f2b 100644 --- a/pkg/grafana/contactpoint-handler.go +++ b/pkg/grafana/contactpoint-handler.go @@ -46,11 +46,6 @@ func (h *AlertContactPointHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a contactPoint -func (h *AlertContactPointHandler) GetExtension() string { - return "json" -} - const ( contactPointGlob = "alert-contact-points/contactPoint-*" contactPointPattern = "alert-contact-points/contactPoint-%s.%s" diff --git a/pkg/grafana/dashboard-handler.go b/pkg/grafana/dashboard-handler.go index fcd931cd..cc59c961 100644 --- a/pkg/grafana/dashboard-handler.go +++ b/pkg/grafana/dashboard-handler.go @@ -52,11 +52,6 @@ func (h *DashboardHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a dashboard -func (h *DashboardHandler) GetExtension() string { - return "json" -} - const ( dashboardGlob = "dashboards/*/dashboard-*" dashboardPattern = "dashboards/%s/dashboard-%s.%s" diff --git a/pkg/grafana/datasource-handler.go b/pkg/grafana/datasource-handler.go index 773beb24..05e27e26 100644 --- a/pkg/grafana/datasource-handler.go +++ b/pkg/grafana/datasource-handler.go @@ -50,11 +50,6 @@ func (h *DatasourceHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a datasource -func (h *DatasourceHandler) GetExtension() string { - return "json" -} - const ( datasourceGlob = "datasources/datasource-*" datasourcePattern = "datasources/datasource-%s.%s" diff --git a/pkg/grafana/folder-handler.go b/pkg/grafana/folder-handler.go index cc6d5866..0e4df07c 100644 --- a/pkg/grafana/folder-handler.go +++ b/pkg/grafana/folder-handler.go @@ -49,11 +49,6 @@ func (h *FolderHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a dashboard -func (h *FolderHandler) GetExtension() string { - return "json" -} - const ( folderGlob = "folders/folder-*" folderPattern = "folders/folder-%s.%s" diff --git a/pkg/grafana/library-elements-handler.go b/pkg/grafana/library-elements-handler.go index dadef038..a9dd9fa6 100644 --- a/pkg/grafana/library-elements-handler.go +++ b/pkg/grafana/library-elements-handler.go @@ -50,11 +50,6 @@ func (h *LibraryElementHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a library element -func (h *LibraryElementHandler) GetExtension() string { - return "json" -} - const ( libraryElementGlob = "library-elements/*-*" libraryElementPattern = "library-elements/%s-%s.%s" diff --git a/pkg/grafana/notificationpolicy-handler.go b/pkg/grafana/notificationpolicy-handler.go index 4f0f1c90..05b586fc 100644 --- a/pkg/grafana/notificationpolicy-handler.go +++ b/pkg/grafana/notificationpolicy-handler.go @@ -48,11 +48,6 @@ func (h *AlertNotificationPolicyHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a alertNotificationPolicy -func (h *AlertNotificationPolicyHandler) GetExtension() string { - return "json" -} - const ( alertNotificationPolicyFile = "alertNotificationPolicy.yaml" ) diff --git a/pkg/grafana/rules-handler.go b/pkg/grafana/rules-handler.go index cc837cc9..0d929454 100644 --- a/pkg/grafana/rules-handler.go +++ b/pkg/grafana/rules-handler.go @@ -48,11 +48,6 @@ func (h *RuleHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a rule grouping -func (h *RuleHandler) GetExtension() string { - return "yaml" -} - const ( prometheusRuleGroupGlob = "prometheus/rules-*" prometheusRuleGroupPattern = "prometheus/rules-%s.%s" diff --git a/pkg/grafana/synthetic-monitoring-handler.go b/pkg/grafana/synthetic-monitoring-handler.go index b4d1bc57..78071427 100644 --- a/pkg/grafana/synthetic-monitoring-handler.go +++ b/pkg/grafana/synthetic-monitoring-handler.go @@ -60,11 +60,6 @@ func (h *SyntheticMonitoringHandler) APIVersion() string { return h.Provider.APIVersion() } -// GetExtension returns the file name extension for a check -func (h *SyntheticMonitoringHandler) GetExtension() string { - return "json" -} - const ( syntheticMonitoringCheckGlob = "synthetic-monitoring/check-*" syntheticMonitoringPattern = "synthetic-monitoring/check-%s.%s" diff --git a/pkg/grizzly/config.go b/pkg/grizzly/config.go index b6d6560f..acbab7f4 100644 --- a/pkg/grizzly/config.go +++ b/pkg/grizzly/config.go @@ -11,10 +11,12 @@ type Opts struct { Directory bool // Deprecated: now is gathered with os.Stat() JsonnetPaths []string Targets []string + OutputFormat string // Used for supporting commands that output dashboard JSON - FolderUID string - JSONSpec bool + FolderUID string + OnlySpec bool + HasOnlySpec bool } // PreviewOpts contains options to configure a preview diff --git a/pkg/grizzly/formatting.go b/pkg/grizzly/formatting.go new file mode 100644 index 00000000..89a25b98 --- /dev/null +++ b/pkg/grizzly/formatting.go @@ -0,0 +1,65 @@ +package grizzly + +import ( + "os" + "path/filepath" +) + +func Format(resourcePath string, resource *Resource, format string, onlySpec bool) ([]byte, string, string, error) { + var content string + var filename string + var extension string + var err error + + spec := resource + if onlySpec { + s := Resource(resource.Spec()) + spec = &s + } + + switch format { + case "yaml": + extension = "yaml" + filename, err = getFilename(resourcePath, resource, extension) + if err != nil { + return nil, "", "", err + } + content, err = spec.YAML() + case "json": + extension = "json" + filename, err = getFilename(resourcePath, resource, extension) + if err != nil { + return nil, "", "", err + } + content, err = spec.JSON() + default: + extension = "yaml" + filename, err = getFilename(resourcePath, resource, extension) + if err != nil { + return nil, "", "", err + } + content, err = spec.YAML() + } + return []byte(content), filename, extension, err +} + +func getFilename(resourcePath string, resource *Resource, extension string) (string, error) { + handler, err := Registry.GetHandler(resource.Kind()) + if err != nil { + return "", err + } + return filepath.Join(resourcePath, handler.ResourceFilePath(*resource, extension)), nil +} + +func WriteFile(filename string, content []byte) error { + dir := filepath.Dir(filename) + err := os.MkdirAll(dir, 0755) + if err != nil { + return err + } + err = os.WriteFile(filename, content, 0644) + if err != nil { + return err + } + return nil +} diff --git a/pkg/grizzly/parsing.go b/pkg/grizzly/parsing.go index 4f647273..c0e75bdb 100644 --- a/pkg/grizzly/parsing.go +++ b/pkg/grizzly/parsing.go @@ -60,7 +60,7 @@ func FindResourceFiles(resourcePath string) ([]string, error) { } func ParseFile(opts Opts, resourceFile string) (Resources, error) { - if opts.JSONSpec && filepath.Ext(resourceFile) != ".json" { + if opts.OnlySpec && filepath.Ext(resourceFile) != ".json" { return nil, fmt.Errorf("when -s flag is passed, command expects only json files as resources") } @@ -102,7 +102,7 @@ func manifestFile(resourceFile string) (bool, error) { // ParseJSON evaluates a JSON file and parses it into resources func ParseJSON(resourceFile string, opts Opts) (Resources, error) { - if opts.JSONSpec { + if opts.OnlySpec { return ParseDashboardJSON(resourceFile, opts) } @@ -257,33 +257,3 @@ func ParseJsonnet(jsonnetFile string, opts Opts) (Resources, error) { sort.Sort(resources) return resources, nil } - -// MarshalYAML takes a resource and renders it to a source file as a YAML string -func MarshalYAML(resource Resource, filename string) error { - y, err := resource.YAML() - if err != nil { - return err - } - return writeFile(filename, []byte(y)) -} - -func MarshalSpecToJSON(resource Resource, filename string) error { - j, err := json.MarshalIndent(resource.Spec(), "", " ") - if err != nil { - return err - } - return writeFile(filename, j) -} - -func writeFile(filename string, content []byte) error { - dir := filepath.Dir(filename) - err := os.MkdirAll(dir, 0755) - if err != nil { - return err - } - err = os.WriteFile(filename, content, 0644) - if err != nil { - return err - } - return nil -} diff --git a/pkg/grizzly/providers.go b/pkg/grizzly/providers.go index 162564de..598f788e 100644 --- a/pkg/grizzly/providers.go +++ b/pkg/grizzly/providers.go @@ -113,7 +113,7 @@ func (r *Resource) Spec() map[string]interface{} { } func (r *Resource) SpecAsJSON() (string, error) { - j, err := json.Marshal(r.Spec()) + j, err := json.MarshalIndent(r.Spec(), "", " ") if err != nil { return "", err } @@ -130,6 +130,15 @@ func (r *Resource) YAML() (string, error) { return string(y), nil } +// JSON Gets the string representation for this resource +func (r *Resource) JSON() (string, error) { + j, err := json.MarshalIndent(*r, "", " ") + if err != nil { + return "", err + } + return string(j), nil +} + // Resources represents a set of resources type Resources []Resource @@ -153,7 +162,6 @@ func (r Resources) Swap(i, j int) { type Handler interface { APIVersion() string Kind() string - GetExtension() string // FindResourceFiles identifies files within a directory that this handler can process FindResourceFiles(dir string) ([]string, error) diff --git a/pkg/grizzly/workflow.go b/pkg/grizzly/workflow.go index e039b227..873b3ef9 100644 --- a/pkg/grizzly/workflow.go +++ b/pkg/grizzly/workflow.go @@ -22,7 +22,7 @@ import ( var interactive = terminal.IsTerminal(int(os.Stdout.Fd())) // Get retrieves a resource from a remote endpoint using its UID -func Get(UID string) error { +func Get(UID string, opts Opts) error { log.Info("Getting ", UID) count := strings.Count(UID, ".") @@ -51,12 +51,16 @@ func Get(UID string) error { } resource = handler.Unprepare(*resource) - rep, err := resource.YAML() + format, onlySpec, err := getOutputFormat(opts) + if err != nil { + return err + } + content, _, _, err := Format("", resource, format, onlySpec) if err != nil { return err } - fmt.Println(rep) + fmt.Println(string(content)) return nil } @@ -134,6 +138,11 @@ func Pull(resourcePath string, opts Opts) error { notifier.Info(notifier.SimpleString(handler.Kind()), "skipped") continue } + + format, onlySpec, err := getOutputFormat(opts) + if err != nil { + return err + } log.Debugf("Listing remote values for handler %s", name) UIDs, err := handler.ListRemote() if err != nil { @@ -156,14 +165,11 @@ func Pull(resourcePath string, opts Opts) error { return err } - if opts.JSONSpec { - path := filepath.Join(resourcePath, handler.ResourceFilePath(*resource, "json")) - err = MarshalSpecToJSON(*resource, path) - } else { - path := filepath.Join(resourcePath, handler.ResourceFilePath(*resource, "yaml")) - err = MarshalYAML(*resource, path) + content, filename, _, err := Format(resourcePath, resource, format, onlySpec) + if err != nil { + return err } - + err = WriteFile(filename, content) if err != nil { return err } @@ -174,7 +180,7 @@ func Pull(resourcePath string, opts Opts) error { } // Show displays resources -func Show(resources Resources) error { +func Show(resources Resources, opts Opts) error { log.Infof("Showing %d resources", resources.Len()) var items []term.PageItem @@ -185,18 +191,23 @@ func Show(resources Resources) error { } resource = *(handler.Unprepare(resource)) - rep, err := resource.YAML() + format, onlySpec, err := getOutputFormat(opts) + if err != nil { + return err + } + content, _, _, err := Format("", &resource, format, onlySpec) if err != nil { return err } + if interactive { items = append(items, term.PageItem{ Name: fmt.Sprintf("%s.%s", resource.Kind(), resource.Name()), - Content: rep, + Content: string(content), }) } else { fmt.Printf("%s.%s:\n", resource.Kind(), resource.Name()) - fmt.Println(rep) + fmt.Println(string(content)) } } if interactive { @@ -206,7 +217,7 @@ func Show(resources Resources) error { } // Diff compares resources to those at the endpoints -func Diff(resources Resources) error { +func Diff(resources Resources, opts Opts) error { log.Infof("Diff-ing %d resources", resources.Len()) for _, resource := range resources { @@ -215,7 +226,11 @@ func Diff(resources Resources) error { return err } - local, err := resource.YAML() + format, onlySpec, err := getOutputFormat(opts) + if err != nil { + return err + } + local, _, _, err := Format("", &resource, format, onlySpec) if err != nil { return err } @@ -235,17 +250,18 @@ func Diff(resources Resources) error { } remote = handler.Unprepare(*remote) - remoteRepresentation, err := (*remote).YAML() + + remoteRepresentation, _, _, err := Format("", &resource, format, onlySpec) if err != nil { return err } - if local == remoteRepresentation { + if string(local) == string(remoteRepresentation) { notifier.NoChanges(resource) } else { diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(remoteRepresentation), - B: difflib.SplitLines(local), + A: difflib.SplitLines(string(remoteRepresentation)), + B: difflib.SplitLines(string(local)), FromFile: "Remote", ToFile: "Local", Context: 3, @@ -431,7 +447,7 @@ func Listen(UID, filename string) error { } // Export renders Jsonnet resources then saves them to a directory -func Export(exportDir string, resources Resources) error { +func Export(exportDir string, resources Resources, opts Opts) error { if _, err := os.Stat(exportDir); os.IsNotExist(err) { err = os.Mkdir(exportDir, 0755) if err != nil { @@ -440,15 +456,15 @@ func Export(exportDir string, resources Resources) error { } for _, resource := range resources { - handler, err := Registry.GetHandler(resource.Kind()) + format, onlySpec, err := getOutputFormat(opts) if err != nil { return err } - updatedResource, err := resource.YAML() + updatedResourceBytes, _, extension, err := Format("", &resource, format, onlySpec) if err != nil { return err } - extension := handler.GetExtension() + dir := fmt.Sprintf("%s/%s", exportDir, resource.Kind()) if _, err := os.Stat(dir); os.IsNotExist(err) { err = os.Mkdir(dir, 0755) @@ -463,6 +479,7 @@ func Export(exportDir string, resources Resources) error { if err != nil && !isNotExist { return err } + updatedResource := string(updatedResourceBytes) existingResource := string(existingResourceBytes) if existingResource == updatedResource { notifier.NoChanges(resource) @@ -492,3 +509,22 @@ func isFile(resourcePath string) (bool, error) { return !stat.IsDir(), nil } + +func getOutputFormat(opts Opts) (string, bool, error) { + var onlySpec bool + context, err := config.CurrentContext() + if err != nil { + return "", false, err + } + if opts.HasOnlySpec { + onlySpec = opts.OnlySpec + } else { + onlySpec = context.OnlySpec + } + if opts.OutputFormat != "" { + return opts.OutputFormat, onlySpec, nil + } else if context.OutputFormat != "" { + return context.OutputFormat, onlySpec, nil + } + return "yaml", onlySpec, nil +}