feat: add gitops.source functions

query/gitops.go

@@ -0,0 +1,183 @@
+package query
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/flanksource/duty/context"
+	"github.com/flanksource/duty/models"
+	"github.com/flanksource/gomplate/v3/conv"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/uuid"
+)
+
+type Kustomize struct {
+	Path string `json:"path"`
+	File string `json:"file"`
+}
+
+func (t *Kustomize) AsMap() map[string]any {
+	return map[string]any{
+		"path": t.Path,
+		"file": t.File,
+	}
+}
+
+type Git struct {
+	File   string `json:"file"`
+	Dir    string `json:"dir"`
+	URL    string `json:"url"`
+	Branch string `json:"branch"`
+}
+
+func (t *Git) AsMap() map[string]any {
+	return map[string]any{
+		"file":   t.File,
+		"dir":    t.Dir,
+		"url":    t.URL,
+		"branch": t.Branch,
+	}
+}
+
+type GitOpsSource struct {
+	Git       Git       `json:"git"`
+	Kustomize Kustomize `json:"kustomize"`
+}
+
+func (t *GitOpsSource) AsMap() map[string]any {
+	return map[string]any{
+		"git":       t.Git.AsMap(),
+		"kustomize": t.Kustomize.AsMap(),
+	}
+}
+
+func getOrigin(ci *models.ConfigItem) (map[string]any, error) {
+	origin := make(map[string]any)
+	_origin := ci.NestedString("metadata", "annotations", "config.kubernetes.io/origin")
+	if _origin != "" {
+		if err := yaml.Unmarshal([]byte(_origin), &origin); err != nil {
+			return origin, err
+		}
+	}
+	return origin, nil
+}
+
+func GetGitOpsSource(ctx context.Context, id uuid.UUID) (GitOpsSource, error) {
+	var source GitOpsSource
+	if id == uuid.Nil {
+		return source, nil
+	}
+
+	ci, err := GetCachedConfig(ctx, id.String())
+	if err != nil {
+		return source, err
+	}
+
+	gitRepos := TraverseConfig(ctx, id.String(), "Kubernetes::Kustomization/Kubernetes::GitRepository", string(models.RelatedConfigTypeIncoming))
+	if len(gitRepos) > 0 && gitRepos[0].Config != nil {
+		source.Git.URL = gitRepos[0].NestedString("spec", "url")
+		source.Git.Branch = gitRepos[0].NestedString("spec", "ref", "branch")
+	}
+
+	kustomization := TraverseConfig(ctx, id.String(), "Kubernetes::Kustomization", string(models.RelatedConfigTypeIncoming))
+	if len(kustomization) > 0 && kustomization[0].Config != nil {
+		source.Kustomize.Path = kustomization[0].NestedString("spec", "path")
+		source.Kustomize.File = filepath.Join(source.Kustomize.Path, "kustomization.yaml")
+	}
+
+	origin, _ := getOrigin(ci)
+	if path, ok := origin["path"]; ok {
+		source.Git.File = filepath.Join(source.Kustomize.Path, path.(string))
+		source.Git.Dir = filepath.Dir(source.Git.File)
+	}
+
+	return source, nil
+}
+
+func gitopsSourceCELFunction() func(ctx context.Context) cel.EnvOption {
+	return func(ctx context.Context) cel.EnvOption {
+		return cel.Function("gitops.source",
+			cel.Overload("gitops.source_interface{}",
+				[]*cel.Type{cel.DynType},
+				cel.DynType,
+				cel.UnaryBinding(func(arg ref.Val) ref.Val {
+
+					id, err := getConfigId(arg.Value())
+					if err != nil {
+						ctx.Errorf("could not find id: %v", err)
+						return types.DefaultTypeAdapter.NativeToValue((&GitOpsSource{}).AsMap())
+					}
+
+					source, err := GetGitOpsSource(ctx, id)
+					if err != nil {
+						return types.WrapErr(err)
+					}
+
+					return types.DefaultTypeAdapter.NativeToValue(source.AsMap())
+				}),
+			),
+		)
+	}
+}
+
+func getConfigId(id any) (uuid.UUID, error) {
+	switch v := id.(type) {
+	case string:
+		return uuid.Parse(v)
+	case uuid.UUID:
+		return v, nil
+	case models.ConfigItem:
+		return v.ID, nil
+	case map[string]string:
+		if v, ok := v["id"]; ok {
+			return uuid.Parse(v)
+		}
+	case map[string]any:
+		if v, ok := v["id"]; ok {
+			switch v2 := v.(type) {
+			case uuid.UUID:
+				return v2, nil
+			case []byte:
+				return uuid.UUID(v2), nil
+			case string:
+				return uuid.Parse(v2)
+			default:
+				return uuid.Parse(conv.ToString(v2))
+			}
+		}
+	}
+	return uuid.Nil, fmt.Errorf("unknown uuid type: %t", id)
+}
+
+func gitopsSourceTemplateFunction() func(ctx context.Context) any {
+	return func(ctx context.Context) any {
+		return func(args ...any) map[string]any {
+
+			var source GitOpsSource
+			if len(args) < 1 {
+				return source.AsMap()
+			}
+
+			id, err := getConfigId(args[0])
+			if err != nil {
+				ctx.Errorf("could not find id '%s' from %v: %v", id, args[0], err)
+				return source.AsMap()
+			}
+
+			source, err = GetGitOpsSource(ctx, id)
+			if err != nil {
+				ctx.Errorf(err.Error())
+			}
+			return source.AsMap()
+		}
+	}
+}
+
+func init() {
+	context.CelEnvFuncs["gitops.source"] = gitopsSourceCELFunction()
+	context.TemplateFuncs["gitops_source"] = gitopsSourceTemplateFunction()
+}

tests/config_gitops_test.go

@@ -0,0 +1,61 @@
+package tests
+
+import (
+	"github.com/flanksource/duty/query"
+	"github.com/flanksource/duty/tests/fixtures/dummy"
+	"github.com/flanksource/gomplate/v3"
+	ginkgo "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var gitopsPath = "aws-demo/spec/namespaces/flux/namespace.yaml"
+
+var gitopsFixtures = []struct {
+	id       any
+	expected string
+}{
+	{dummy.Namespace, gitopsPath},
+	{dummy.Namespace.ID, gitopsPath},
+	{dummy.Namespace.ID.String(), gitopsPath},
+	{dummy.Namespace.AsMap(), gitopsPath},
+}
+var _ = ginkgo.Describe("Config Gitops Source", ginkgo.Ordered, func() {
+	ginkgo.It("should resolve kustomize references", func() {
+		Expect(dummy.Kustomization.ID.String()).NotTo(BeEmpty())
+		Expect(dummy.GitRepository.ID.String()).NotTo(BeEmpty())
+		Expect(dummy.Namespace.ID.String()).NotTo(BeEmpty())
+
+		source, err := query.GetGitOpsSource(DefaultContext, dummy.Namespace.ID)
+		Expect(err).To(BeNil())
+		Expect(source.Kustomize.Path).To(Equal("./aws-demo/spec"))
+		Expect(source.Git.File).To(Equal("aws-demo/spec/namespaces/flux/namespace.yaml"))
+		Expect(source.Git.Dir).To(Equal("aws-demo/spec/namespaces/flux"))
+		Expect(source.Git.URL).To(Equal("ssh://[email protected]/flanksource/sandbox.git"))
+		Expect(source.Git.Branch).To(Equal("main"))
+	})
+
+	ginkgo.It("should resolve references using CEL", func() {
+		for _, fixture := range gitopsFixtures {
+			out, err := DefaultContext.RunTemplate(gomplate.Template{
+				Expression: "gitops.source(id).git.file",
+			}, map[string]any{
+				"id": fixture.id,
+			})
+			Expect(err).To(BeNil())
+			Expect(out).To(Equal(fixture.expected))
+
+		}
+	})
+
+	ginkgo.It("should resolve references using gomplate", func() {
+		for _, fixture := range gitopsFixtures {
+			out, err := DefaultContext.RunTemplate(gomplate.Template{
+				Template: "{{ ( .id | gitops_source ).git.file }}",
+			}, map[string]any{
+				"id": fixture.id,
+			})
+			Expect(err).To(BeNil())
+			Expect(out).To(Equal(fixture.expected))
+		}
+	})
+})

tests/fixtures/dummy/config.go

@@ -1,12 +1,74 @@
 package dummy
 
 import (
+	"embed"
+	"path/filepath"
+	"strings"
+
+	"github.com/flanksource/commons/logger"
+	"github.com/flanksource/duty/kubernetes"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"github.com/google/uuid"
 	"github.com/samber/lo"
 )
 
+//go:embed config/*.yaml
+var yamls embed.FS
+
+func ImportConfigs(data []byte) (configs []models.ConfigItem, relationships []models.ConfigRelationship, err error) {
+	objects, err := kubernetes.GetUnstructuredObjects(data)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, object := range objects {
+		json, _ := object.MarshalJSON()
+		labels := types.JSONStringMap{}
+		for k, v := range object.GetLabels() {
+			labels[k] = v
+		}
+		ci := models.ConfigItem{
+			Config:      lo.ToPtr(string(json)),
+			ID:          uuid.MustParse(string(object.GetUID())),
+			Name:        lo.ToPtr(object.GetName()),
+			ConfigClass: object.GetKind(),
+			Type:        lo.ToPtr("Kubernetes::" + object.GetKind()),
+			Labels:      lo.ToPtr(labels),
+			CreatedAt:   object.GetCreationTimestamp().Time,
+			Tags: types.JSONStringMap{
+				"namespace": object.GetNamespace(),
+			},
+		}
+
+		if parent, ok := object.GetAnnotations()["config-db.flanksource.com/parent"]; ok {
+			id, err := uuid.Parse(parent)
+			if err == nil {
+				ci.ParentID = lo.ToPtr(id)
+				relationships = append(relationships, models.ConfigRelationship{
+					ConfigID:  id.String(),
+					RelatedID: ci.ID.String(),
+				})
+			}
+		}
+
+		if related, ok := object.GetAnnotations()["config-db.flanksource.com/related"]; ok {
+			for _, relation := range strings.Split(related, ",") {
+				id, err := uuid.Parse(relation)
+				if err == nil {
+					relationships = append(relationships, models.ConfigRelationship{
+						ConfigID:  ci.ID.String(),
+						RelatedID: id.String(),
+					})
+				}
+
+			}
+		}
+		configs = append(configs, ci)
+	}
+	return configs, relationships, nil
+}
+
 var EKSCluster = models.ConfigItem{
 	ID:          uuid.New(),
 	Name:        lo.ToPtr("Production EKS"),
@@ -257,3 +319,42 @@ var ClusterNodeBRelationship = models.ConfigRelationship{
 }
 
 var AllConfigRelationships = []models.ConfigRelationship{ClusterNodeARelationship, ClusterNodeBRelationship}
+
+func GetConfig(configType, namespace, name string) models.ConfigItem {
+	for _, config := range AllDummyConfigs {
+		if *config.Type == configType &&
+			*config.Name == name &&
+			config.Tags["namespace"] == namespace {
+			return config
+		}
+	}
+	return models.ConfigItem{}
+}
+
+var GitRepository models.ConfigItem
+var Kustomization models.ConfigItem
+var Namespace models.ConfigItem
+
+func init() {
+	files, _ := yamls.ReadDir("config")
+	for _, file := range files {
+		data, err := yamls.ReadFile(filepath.Join("config", file.Name()))
+		if err != nil {
+			logger.Errorf("Failed to read %s: %v", file.Name(), err)
+			continue
+		}
+		configs, relationships, err := ImportConfigs(data)
+		if err != nil {
+			logger.Errorf("Failed to import configs %v", err)
+			continue
+		}
+
+		AllConfigRelationships = append(AllConfigRelationships, relationships...)
+		AllDummyConfigs = append(AllDummyConfigs, configs...)
+	}
+
+	GitRepository = GetConfig("Kubernetes::GitRepository", "flux-system", "sandbox")
+	Kustomization = GetConfig("Kubernetes::Kustomization", "flux-system", "infra")
+	Namespace = GetConfig("Kubernetes::Namespace", "", "flux")
+
+}