WIP

cmd/dashboard/config/config.go

@@ -9,6 +9,7 @@ import (
 )
 
 const DEFAULT_CHARTS = "title=Web application template,description=Deploy any web application. Multiple container image build options available.,name=onechart,repo=https://chart.onechart.dev,version=0.70.0;title=static website template,description=If your build generates static files%2C we will host it in an Nginx container.,name=static-site,repo=https://chart.onechart.dev,version=0.70.0"
+const DEFAULT_PLAIN_MODULES_URL = "https://github.com/gimlet-io/plain-modules.git"
 
 // LoadConfig returns the static config from the environment.
 func LoadConfig() (*Config, error) {
@@ -35,6 +36,9 @@ func defaults(c *Config) {
 	if c.DefaultCharts == nil {
 		c.DefaultCharts.Decode(DEFAULT_CHARTS)
 	}
+	if c.PlainModulesURL == "" {
+		c.PlainModulesURL = DEFAULT_PLAIN_MODULES_URL
+	}
 	if c.GitSSHAddressFormat == "" {
 		c.GitSSHAddressFormat = "[email protected]:%s.git"
 	}
@@ -118,6 +122,8 @@ type Config struct {
 
 	Instance string `envconfig:"INSTANCE"`
 	License  string `envconfig:"LICENSE"`
+
+	PlainModulesURL string `envconfig:"PLAIN_MODULES_URL"`
 }
 
 // Logging provides the logging configuration.

pkg/dashboard/server/api.go

@@ -316,6 +316,74 @@ func stackConfig(w http.ResponseWriter, r *http.Request) {
 	w.Write(gitopsEnvString)
 }
 
+func plainModules(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	config := ctx.Value("config").(*config.Config)
+
+	repo, err := nativeGit.FlexibleURLCloneToMemory(config.PlainModulesURL)
+	if err != nil {
+		logrus.Errorf("cannot get plain modules: %s", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	headBranch, err := helper.HeadBranch(repo)
+	if err != nil {
+		logrus.Errorf("cannot get plain modules: %s", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	folders, err := helper.RemoteFoldersOnBranchWithoutCheckout(repo, headBranch, "")
+	if err != nil {
+		logrus.Errorf("cannot get plain modules: %s", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	plainModules := []*dx.PlainModule{}
+	for _, folder := range folders {
+		fmt.Println(folder)
+		files, err := helper.RemoteFolderOnBranchWithoutCheckout(repo, headBranch, folder)
+		if err != nil {
+			logrus.Errorf("cannot get plain modules: %s", err)
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			return
+		}
+		pm := &dx.PlainModule{URL: config.PlainModulesURL + "?path=" + folder}
+		for path, content := range files {
+			if path == "schema.json" {
+				var schema map[string]interface{}
+				err := json.Unmarshal([]byte(content), &schema)
+				if err != nil {
+					logrus.Warn(err)
+				}
+				pm.Schema = schema
+			} else if path == "uiSchema.json" {
+				var schema []map[string]interface{}
+				err := json.Unmarshal([]byte(content), &schema)
+				if err != nil {
+					logrus.Warn(err)
+				}
+				pm.UISchema = schema
+			} else if path == "template.yaml" {
+				pm.Template = content
+			}
+		}
+		plainModules = append(plainModules, pm)
+	}
+
+	modulesString, err := json.Marshal(plainModules)
+	if err != nil {
+		logrus.Errorf("cannot serialize plain modules: %s", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	w.WriteHeader(200)
+	w.Write([]byte(modulesString))
+}
+
 func StackConfig(gitRepoCache *nativeGit.RepoCache, stackYamlPath, infraRepo string) (*dx.StackConfig, error) {
 	var stackConfig *dx.StackConfig
 	err := gitRepoCache.PerformAction(infraRepo, func(repo *git.Repository) error {

pkg/dashboard/server/router.go

@@ -188,6 +188,7 @@ func userRoutes(r *chi.Mux, clientHub *streaming.ClientHub) {
 		r.Get(("/api/env/{env}/stackConfig"), stackConfig)
 		r.Post("/api/silenceAlert", silenceAlert)
 		r.Post("/api/restartDeployment", restartDeployment)
+		r.Get(("/api/plainModules"), plainModules)
 
 		r.Get("/ws/", func(w http.ResponseWriter, r *http.Request) {
 			streaming.ServeWs(clientHub, w, r)

pkg/dx/stack.go

@@ -9,6 +9,13 @@ type StackConfig struct {
 	Config map[string]interface{} `yaml:"config" json:"config"`
 }
 
+type PlainModule struct {
+	URL      string                   `json:"url"`
+	Schema   map[string]interface{}   `json:"schema"`
+	UISchema []map[string]interface{} `json:"uiSchema"`
+	Template string                   `json:"-"`
+}
+
 type Component struct {
 	Name        string `json:"name,omitempty" yaml:"name"`
 	Description string `json:"description,omitempty" yaml:"description"`

pkg/git/nativeGit/helper.go

@@ -4,19 +4,23 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	giturl "github.com/whilp/git-urls"
 )
 
 const File_RW_RW_R = 0664
@@ -572,3 +576,53 @@ func StageFile(worktree *git.Worktree, content string, path string) error {
 	_, err = worktree.Add(path)
 	return err
 }
+
+func FlexibleURLCloneToMemory(repoURL string) (*git.Repository, error) {
+	gitAddress, err := giturl.ParseScp(repoURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse git address: %s", err)
+	}
+	gitUrl := strings.ReplaceAll(repoURL, gitAddress.RawQuery, "")
+	gitUrl = strings.ReplaceAll(gitUrl, "?", "")
+
+	fs := memfs.New()
+	opts := &git.CloneOptions{
+		URL: gitUrl,
+	}
+	repo, err := git.Clone(memory.NewStorage(), fs, opts)
+	if err != nil {
+		return nil, fmt.Errorf("cannot clone: %s", err)
+	}
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return nil, fmt.Errorf("cannot get worktree: %s", err)
+	}
+
+	params, _ := url.ParseQuery(gitAddress.RawQuery)
+	if v, found := params["sha"]; found {
+		err = worktree.Checkout(&git.CheckoutOptions{
+			Hash: plumbing.NewHash(v[0]),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("cannot checkout sha: %s", err)
+		}
+	}
+	if v, found := params["tag"]; found {
+		err = worktree.Checkout(&git.CheckoutOptions{
+			Branch: plumbing.NewTagReferenceName(v[0]),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("cannot checkout tag: %s", err)
+		}
+	}
+	if v, found := params["branch"]; found {
+		err = worktree.Checkout(&git.CheckoutOptions{
+			Branch: plumbing.NewRemoteReferenceName("origin", v[0]),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("cannot checkout branch: %s", err)
+		}
+	}
+
+	return repo, nil
+}

pkg/stack/template.go

@@ -15,6 +15,7 @@ import (
 	"github.com/blang/semver/v4"
 	"github.com/fluxcd/pkg/sourceignore"
 	"github.com/gimlet-io/gimlet/pkg/dx"
+	"github.com/gimlet-io/gimlet/pkg/git/nativeGit"
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/go-git/go-billy/v5/util"
 	"github.com/go-git/go-git/v5"
@@ -75,7 +76,7 @@ func StackDefinitionFromRepo(repoUrl string) (string, error) {
 // cloneStackFromRepo takes a git repo url, and returns the files of the git reference
 // if the repoUrl is a local filesystem location, it loads the files from there
 func cloneStackFromRepo(repoURL string) (map[string]string, error) {
-	gitAddress, err := giturl.ParseScp(repoURL)
+	repo, err := nativeGit.FlexibleURLCloneToMemory(repoURL)
 	if err != nil {
 		_, err2 := os.Stat(repoURL)
 		if err2 != nil {
@@ -84,48 +85,12 @@ func cloneStackFromRepo(repoURL string) (map[string]string, error) {
 			return loadStackFromFS(repoURL)
 		}
 	}
-	gitUrl := strings.ReplaceAll(repoURL, gitAddress.RawQuery, "")
-	gitUrl = strings.ReplaceAll(gitUrl, "?", "")
 
-	fs := memfs.New()
-	opts := &git.CloneOptions{
-		URL: gitUrl,
-	}
-	repo, err := git.Clone(memory.NewStorage(), fs, opts)
-	if err != nil {
-		return nil, fmt.Errorf("cannot clone: %s", err)
-	}
 	worktree, err := repo.Worktree()
 	if err != nil {
 		return nil, fmt.Errorf("cannot get worktree: %s", err)
 	}
 
-	params, _ := url.ParseQuery(gitAddress.RawQuery)
-	if v, found := params["sha"]; found {
-		err = worktree.Checkout(&git.CheckoutOptions{
-			Hash: plumbing.NewHash(v[0]),
-		})
-		if err != nil {
-			return nil, fmt.Errorf("cannot checkout sha: %s", err)
-		}
-	}
-	if v, found := params["tag"]; found {
-		err = worktree.Checkout(&git.CheckoutOptions{
-			Branch: plumbing.NewTagReferenceName(v[0]),
-		})
-		if err != nil {
-			return nil, fmt.Errorf("cannot checkout tag: %s", err)
-		}
-	}
-	if v, found := params["branch"]; found {
-		err = worktree.Checkout(&git.CheckoutOptions{
-			Branch: plumbing.NewRemoteReferenceName("origin", v[0]),
-		})
-		if err != nil {
-			return nil, fmt.Errorf("cannot checkout branch: %s", err)
-		}
-	}
-
 	paths, err := util.Glob(worktree.Filesystem, "*/*")
 	if err != nil {
 		return nil, fmt.Errorf("cannot list files: %s", err)
@@ -146,7 +111,7 @@ func cloneStackFromRepo(repoURL string) (map[string]string, error) {
 	}
 	paths = append(paths, paths4...)
 
-	fs = worktree.Filesystem
+	fs := worktree.Filesystem
 
 	var stackIgnorePatterns []gitignore.Pattern
 	const stackIgnoreFile = ".stackignore"

web/src/client/client.js

@@ -117,6 +117,8 @@ export default class GimletClient {
 
   restartDeploymentRequest = (namespace, name) =>this.post(`/api/restartDeployment?namespace=${namespace}&name=${name}`);
 
+  getPlainModules = () =>  this.get(`/api/plainModules`);
+
   get = async (path, signal) => {
     try {
       const { data } = await axios.get(path, {