diff --git a/cmd/dashboard/config/config.go b/cmd/dashboard/config/config.go index 5579df754..311cfe8c6 100644 --- a/cmd/dashboard/config/config.go +++ b/cmd/dashboard/config/config.go @@ -9,6 +9,7 @@ import ( ) const DEFAULT_CHARTS = "title=Web application template,description=Deploy any web application. Multiple container image build options available.,name=onechart,repo=https://chart.onechart.dev,version=0.70.0;title=static website template,description=If your build generates static files%2C we will host it in an Nginx container.,name=static-site,repo=https://chart.onechart.dev,version=0.70.0" +const DEFAULT_PLAIN_MODULES_URL = "https://github.com/gimlet-io/plain-modules.git" // LoadConfig returns the static config from the environment. func LoadConfig() (*Config, error) { @@ -35,6 +36,9 @@ func defaults(c *Config) { if c.DefaultCharts == nil { c.DefaultCharts.Decode(DEFAULT_CHARTS) } + if c.PlainModulesURL == "" { + c.PlainModulesURL = DEFAULT_PLAIN_MODULES_URL + } if c.GitSSHAddressFormat == "" { c.GitSSHAddressFormat = "git@github.com:%s.git" } @@ -118,6 +122,8 @@ type Config struct { Instance string `envconfig:"INSTANCE"` License string `envconfig:"LICENSE"` + + PlainModulesURL string `envconfig:"PLAIN_MODULES_URL"` } // Logging provides the logging configuration. diff --git a/pkg/dashboard/server/api.go b/pkg/dashboard/server/api.go index 16471f429..aaf5baaad 100644 --- a/pkg/dashboard/server/api.go +++ b/pkg/dashboard/server/api.go @@ -316,6 +316,74 @@ func stackConfig(w http.ResponseWriter, r *http.Request) { w.Write(gitopsEnvString) } +func plainModules(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + config := ctx.Value("config").(*config.Config) + + repo, err := nativeGit.FlexibleURLCloneToMemory(config.PlainModulesURL) + if err != nil { + logrus.Errorf("cannot get plain modules: %s", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + headBranch, err := helper.HeadBranch(repo) + if err != nil { + logrus.Errorf("cannot get plain modules: %s", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + folders, err := helper.RemoteFoldersOnBranchWithoutCheckout(repo, headBranch, "") + if err != nil { + logrus.Errorf("cannot get plain modules: %s", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + plainModules := []*dx.PlainModule{} + for _, folder := range folders { + fmt.Println(folder) + files, err := helper.RemoteFolderOnBranchWithoutCheckout(repo, headBranch, folder) + if err != nil { + logrus.Errorf("cannot get plain modules: %s", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + pm := &dx.PlainModule{URL: config.PlainModulesURL + "?path=" + folder} + for path, content := range files { + if path == "schema.json" { + var schema map[string]interface{} + err := json.Unmarshal([]byte(content), &schema) + if err != nil { + logrus.Warn(err) + } + pm.Schema = schema + } else if path == "uiSchema.json" { + var schema []map[string]interface{} + err := json.Unmarshal([]byte(content), &schema) + if err != nil { + logrus.Warn(err) + } + pm.UISchema = schema + } else if path == "template.yaml" { + pm.Template = content + } + } + plainModules = append(plainModules, pm) + } + + modulesString, err := json.Marshal(plainModules) + if err != nil { + logrus.Errorf("cannot serialize plain modules: %s", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + w.WriteHeader(200) + w.Write([]byte(modulesString)) +} + func StackConfig(gitRepoCache *nativeGit.RepoCache, stackYamlPath, infraRepo string) (*dx.StackConfig, error) { var stackConfig *dx.StackConfig err := gitRepoCache.PerformAction(infraRepo, func(repo *git.Repository) error { diff --git a/pkg/dashboard/server/router.go b/pkg/dashboard/server/router.go index 27d63f495..cb23589de 100644 --- a/pkg/dashboard/server/router.go +++ b/pkg/dashboard/server/router.go @@ -188,6 +188,7 @@ func userRoutes(r *chi.Mux, clientHub *streaming.ClientHub) { r.Get(("/api/env/{env}/stackConfig"), stackConfig) r.Post("/api/silenceAlert", silenceAlert) r.Post("/api/restartDeployment", restartDeployment) + r.Get(("/api/plainModules"), plainModules) r.Get("/ws/", func(w http.ResponseWriter, r *http.Request) { streaming.ServeWs(clientHub, w, r) diff --git a/pkg/dx/stack.go b/pkg/dx/stack.go index 8b385de4b..049949a28 100644 --- a/pkg/dx/stack.go +++ b/pkg/dx/stack.go @@ -9,6 +9,13 @@ type StackConfig struct { Config map[string]interface{} `yaml:"config" json:"config"` } +type PlainModule struct { + URL string `json:"url"` + Schema map[string]interface{} `json:"schema"` + UISchema []map[string]interface{} `json:"uiSchema"` + Template string `json:"-"` +} + type Component struct { Name string `json:"name,omitempty" yaml:"name"` Description string `json:"description,omitempty" yaml:"description"` diff --git a/pkg/git/nativeGit/helper.go b/pkg/git/nativeGit/helper.go index f2c0cf350..a8cad14dd 100644 --- a/pkg/git/nativeGit/helper.go +++ b/pkg/git/nativeGit/helper.go @@ -4,19 +4,23 @@ import ( "context" "fmt" "io/ioutil" + "net/url" "os" "os/exec" "path/filepath" "strings" "time" + "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/go-git/go-git/v5/storage/memory" "github.com/pkg/errors" "github.com/sirupsen/logrus" + giturl "github.com/whilp/git-urls" ) const File_RW_RW_R = 0664 @@ -572,3 +576,53 @@ func StageFile(worktree *git.Worktree, content string, path string) error { _, err = worktree.Add(path) return err } + +func FlexibleURLCloneToMemory(repoURL string) (*git.Repository, error) { + gitAddress, err := giturl.ParseScp(repoURL) + if err != nil { + return nil, fmt.Errorf("cannot parse git address: %s", err) + } + gitUrl := strings.ReplaceAll(repoURL, gitAddress.RawQuery, "") + gitUrl = strings.ReplaceAll(gitUrl, "?", "") + + fs := memfs.New() + opts := &git.CloneOptions{ + URL: gitUrl, + } + repo, err := git.Clone(memory.NewStorage(), fs, opts) + if err != nil { + return nil, fmt.Errorf("cannot clone: %s", err) + } + worktree, err := repo.Worktree() + if err != nil { + return nil, fmt.Errorf("cannot get worktree: %s", err) + } + + params, _ := url.ParseQuery(gitAddress.RawQuery) + if v, found := params["sha"]; found { + err = worktree.Checkout(&git.CheckoutOptions{ + Hash: plumbing.NewHash(v[0]), + }) + if err != nil { + return nil, fmt.Errorf("cannot checkout sha: %s", err) + } + } + if v, found := params["tag"]; found { + err = worktree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewTagReferenceName(v[0]), + }) + if err != nil { + return nil, fmt.Errorf("cannot checkout tag: %s", err) + } + } + if v, found := params["branch"]; found { + err = worktree.Checkout(&git.CheckoutOptions{ + Branch: plumbing.NewRemoteReferenceName("origin", v[0]), + }) + if err != nil { + return nil, fmt.Errorf("cannot checkout branch: %s", err) + } + } + + return repo, nil +} diff --git a/pkg/stack/template.go b/pkg/stack/template.go index d57cf385a..eb82d52d7 100644 --- a/pkg/stack/template.go +++ b/pkg/stack/template.go @@ -15,6 +15,7 @@ import ( "github.com/blang/semver/v4" "github.com/fluxcd/pkg/sourceignore" "github.com/gimlet-io/gimlet/pkg/dx" + "github.com/gimlet-io/gimlet/pkg/git/nativeGit" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-billy/v5/util" "github.com/go-git/go-git/v5" @@ -75,7 +76,7 @@ func StackDefinitionFromRepo(repoUrl string) (string, error) { // cloneStackFromRepo takes a git repo url, and returns the files of the git reference // if the repoUrl is a local filesystem location, it loads the files from there func cloneStackFromRepo(repoURL string) (map[string]string, error) { - gitAddress, err := giturl.ParseScp(repoURL) + repo, err := nativeGit.FlexibleURLCloneToMemory(repoURL) if err != nil { _, err2 := os.Stat(repoURL) if err2 != nil { @@ -84,48 +85,12 @@ func cloneStackFromRepo(repoURL string) (map[string]string, error) { return loadStackFromFS(repoURL) } } - gitUrl := strings.ReplaceAll(repoURL, gitAddress.RawQuery, "") - gitUrl = strings.ReplaceAll(gitUrl, "?", "") - fs := memfs.New() - opts := &git.CloneOptions{ - URL: gitUrl, - } - repo, err := git.Clone(memory.NewStorage(), fs, opts) - if err != nil { - return nil, fmt.Errorf("cannot clone: %s", err) - } worktree, err := repo.Worktree() if err != nil { return nil, fmt.Errorf("cannot get worktree: %s", err) } - params, _ := url.ParseQuery(gitAddress.RawQuery) - if v, found := params["sha"]; found { - err = worktree.Checkout(&git.CheckoutOptions{ - Hash: plumbing.NewHash(v[0]), - }) - if err != nil { - return nil, fmt.Errorf("cannot checkout sha: %s", err) - } - } - if v, found := params["tag"]; found { - err = worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewTagReferenceName(v[0]), - }) - if err != nil { - return nil, fmt.Errorf("cannot checkout tag: %s", err) - } - } - if v, found := params["branch"]; found { - err = worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewRemoteReferenceName("origin", v[0]), - }) - if err != nil { - return nil, fmt.Errorf("cannot checkout branch: %s", err) - } - } - paths, err := util.Glob(worktree.Filesystem, "*/*") if err != nil { return nil, fmt.Errorf("cannot list files: %s", err) @@ -146,7 +111,7 @@ func cloneStackFromRepo(repoURL string) (map[string]string, error) { } paths = append(paths, paths4...) - fs = worktree.Filesystem + fs := worktree.Filesystem var stackIgnorePatterns []gitignore.Pattern const stackIgnoreFile = ".stackignore" diff --git a/web/src/client/client.js b/web/src/client/client.js index 9712df590..a58d0b2cf 100644 --- a/web/src/client/client.js +++ b/web/src/client/client.js @@ -117,6 +117,8 @@ export default class GimletClient { restartDeploymentRequest = (namespace, name) =>this.post(`/api/restartDeployment?namespace=${namespace}&name=${name}`); + getPlainModules = () => this.get(`/api/plainModules`); + get = async (path, signal) => { try { const { data } = await axios.get(path, { diff --git a/web/src/views/envConfig/databasesTab.jsx b/web/src/views/envConfig/databasesTab.jsx index a9ff1133e..26e888479 100644 --- a/web/src/views/envConfig/databasesTab.jsx +++ b/web/src/views/envConfig/databasesTab.jsx @@ -1,11 +1,14 @@ import {InfraComponent} from '../environment/category'; import { useState, useEffect } from 'react'; import {produce} from 'immer'; +import { Combobox, ComboboxButton, ComboboxInput, ComboboxOption, ComboboxOptions, Label } from '@headlessui/react' +import { CheckIcon, ChevronUpDownIcon } from '@heroicons/react/20/solid' export function DatabasesTab(props) { const { gimletClient, store } = props; const { environment } = props; const { databaseConfig, setDatabaseValues } = props + const { plainModules } = props; const validationCallback = (variable, validationErrors) => { if(validationErrors) { @@ -13,11 +16,12 @@ export function DatabasesTab(props) { } } - console.log(environment) + console.log(databaseConfig) return (
- {databases.map(component => + + {/* {databases.map(component => - )} + )} */}
) } + +const people = [ + { id: 1, name: 'Leslie Alexander' }, +] + + +export default function Example() { + const [query, setQuery] = useState('') + const [selectedPerson, setSelectedPerson] = useState(null) + + const filteredPeople = + query === '' + ? people + : people.filter((person) => { + return person.name.toLowerCase().includes(query.toLowerCase()) + }) + + return ( + { + setQuery('') + setSelectedPerson(person) + }} + > + +
+ setQuery(event.target.value)} + onBlur={() => setQuery('')} + displayValue={(person) => person?.name} + /> + + + + {filteredPeople.length > 0 && ( + + {filteredPeople.map((person) => ( + + {person.name} + + + + + ))} + + )} +
+ + ) +} + const databases = [ { variable: "redis", @@ -74,7 +138,7 @@ const databases = [ } } } - ], + ] }, { variable: "postgresql", @@ -82,7 +146,7 @@ const databases = [ "$schema": "http://json-schema.org/draft-07/schema", "$id": "#postgresql", "type": "object", - "title": "PostgreSQL database", + "title": "PostgreSQL", "description": "Provision a logical database with a user and password in the centralized PostgreSQL instance", "properties": { "enabled": { @@ -126,6 +190,6 @@ const databases = [ } } } - ], + ] } ] diff --git a/web/src/views/envConfig/envConfig.jsx b/web/src/views/envConfig/envConfig.jsx index 048708b2a..a42a58845 100644 --- a/web/src/views/envConfig/envConfig.jsx +++ b/web/src/views/envConfig/envConfig.jsx @@ -31,6 +31,7 @@ export function EnvConfig(props) { const [popupWindow, setPopupWindow] = useState(reduxState.popupWindow) const [scmUrl, setScmUrl] = useState(reduxState.settings.scmUrl) const [fileInfos, setFileInfos] = useState(reduxState.fileInfos) + const [plainModules, setPlainModules] = useState(reduxState.fileInfos) const [configFile, setConfigFile] = useState() const [databaseConfig, setDatabaseConfig] = useState() const [savedConfigFile, setSavedConfigFile] = useState() @@ -100,6 +101,11 @@ export function EnvConfig(props) { }, () => {/* Generic error handler deals with it */ }); } + gimletClient.getPlainModules() + .then(data => { + setPlainModules(data) + }, () => {/* Generic error handler deals with it */ }); + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); @@ -553,7 +559,17 @@ export function EnvConfig(props) { } } - { selectedNavigation?.name === "Databases" && + { selectedNavigation?.name === "Containerized Dependencies" && + + } + { selectedNavigation?.name === "Cloud Dependencies" && { function translateToNavigation(template) { const navigation = template.uiSchema.map((elem, idx) => ({name: elem.metaData.name, href: ref(elem.metaData.name), uiSchemaOrder: idx})) navigation.unshift({name: "General", href: "/general"}) - navigation.push({name: "Databases", href: "/databases"}) + navigation.push({name: "Containerized Dependencies", href: "/containerized-databases"}) + // navigation.push({name: "Cloud Dependencies", href: "/cloud-databases"}) return navigation }