From 2b58a8bdd5a71e857953d9b3c7b555c31db99ab7 Mon Sep 17 00:00:00 2001 From: MeurillonGuillaume Date: Wed, 6 Dec 2023 11:45:24 +0100 Subject: [PATCH] cfg: add configuration for json index --- cmd/chartmuseum/main.go | 1 + pkg/chartmuseum/server.go | 2 ++ pkg/chartmuseum/server/multitenant/cache.go | 35 ++++++++++++-------- pkg/chartmuseum/server/multitenant/server.go | 3 ++ pkg/config/vars.go | 9 +++++ pkg/repo/index.go | 4 +-- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go index 86089fa6..77f7ad49 100644 --- a/cmd/chartmuseum/main.go +++ b/cmd/chartmuseum/main.go @@ -119,6 +119,7 @@ func cliHandler(c *cli.Context) { WebTemplatePath: conf.GetString("web-template-path"), ArtifactHubRepoID: conf.GetStringMapString("artifact-hub-repo-id"), AlwaysRegenerateIndex: conf.GetBool("always-regenerate-chart-index"), + JSONIndex: conf.GetBool("json-index"), } server, err := newServer(options) diff --git a/pkg/chartmuseum/server.go b/pkg/chartmuseum/server.go index 568dfd36..c9d88f03 100644 --- a/pkg/chartmuseum/server.go +++ b/pkg/chartmuseum/server.go @@ -84,6 +84,7 @@ type ( // AlwaysRegenerateIndex represents if the museum always return the up-to-date chart // which means that the GetChart will increase its latency , be careful to enable this . AlwaysRegenerateIndex bool + JSONIndex bool } // Server is a generic interface for web servers @@ -151,6 +152,7 @@ func NewServer(options ServerOptions) (Server, error) { // EnforceSemver2 - see https://github.com/helm/chartmuseum/issues/485 for more info EnforceSemver2: options.EnforceSemver2, AlwaysRegenerateIndex: options.AlwaysRegenerateIndex, + JSONIndex: options.JSONIndex, }) return server, err diff --git a/pkg/chartmuseum/server/multitenant/cache.go b/pkg/chartmuseum/server/multitenant/cache.go index e7cc69e4..60fdfb9b 100644 --- a/pkg/chartmuseum/server/multitenant/cache.go +++ b/pkg/chartmuseum/server/multitenant/cache.go @@ -152,11 +152,12 @@ func (server *MultiTenantServer) regenerateRepositoryIndexWorker(log cm_logger.L "repo", repo, ) index := &cm_repo.Index{ - IndexFile: entry.RepoIndex.IndexFile, - RepoName: repo, - Raw: entry.RepoIndex.Raw, - ChartURL: entry.RepoIndex.ChartURL, - IndexLock: sync.RWMutex{}, + IndexFile: entry.RepoIndex.IndexFile, + RepoName: repo, + Raw: entry.RepoIndex.Raw, + ChartURL: entry.RepoIndex.ChartURL, + IndexLock: sync.RWMutex{}, + OutputJSON: server.JSONIndex, } for _, object := range diff.Removed { @@ -442,23 +443,28 @@ func (server *MultiTenantServer) newRepositoryIndex(log cm_logger.LoggingFn, rep } if !server.UseStatefiles { - return cm_repo.NewIndex(chartURL, repo, serverInfo) + return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex) } objectPath := pathutil.Join(repo, cm_repo.StatefileFilename) object, err := server.StorageBackend.GetObject(objectPath) if err != nil { - return cm_repo.NewIndex(chartURL, repo, serverInfo) + return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex) } indexFile := &cm_repo.IndexFile{} - err = yaml.Unmarshal(object.Content, indexFile) + if json.Valid(object.Content) { + err = json.Unmarshal(object.Content, indexFile) + } else { + err = yaml.Unmarshal(object.Content, indexFile) + } + if err != nil { log(cm_logger.WarnLevel, "index-cache.yaml found but could not be parsed", "repo", repo, "error", err.Error(), ) - return cm_repo.NewIndex(chartURL, repo, serverInfo) + return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex) } log(cm_logger.DebugLevel, "index-cache.yaml loaded", @@ -466,11 +472,12 @@ func (server *MultiTenantServer) newRepositoryIndex(log cm_logger.LoggingFn, rep ) return &cm_repo.Index{ - IndexFile: indexFile, - RepoName: repo, - Raw: object.Content, - ChartURL: chartURL, - IndexLock: sync.RWMutex{}, + IndexFile: indexFile, + RepoName: repo, + Raw: object.Content, + ChartURL: chartURL, + IndexLock: sync.RWMutex{}, + OutputJSON: server.JSONIndex, } } diff --git a/pkg/chartmuseum/server/multitenant/server.go b/pkg/chartmuseum/server/multitenant/server.go index d6221f33..bb5c7437 100644 --- a/pkg/chartmuseum/server/multitenant/server.go +++ b/pkg/chartmuseum/server/multitenant/server.go @@ -73,6 +73,7 @@ type ( EnforceSemver2 bool WebTemplatePath string AlwaysRegenerateIndex bool + JSONIndex bool } ObjectsPerChartLimit struct { @@ -106,6 +107,7 @@ type ( // Deprecated: see https://github.com/helm/chartmuseum/issues/485 for more info EnforceSemver2 bool AlwaysRegenerateIndex bool + JSONIndex bool } tenantInternals struct { @@ -166,6 +168,7 @@ func NewMultiTenantServer(options MultiTenantServerOptions) (*MultiTenantServer, WebTemplatePath: options.WebTemplatePath, ArtifactHubRepoID: options.ArtifactHubRepoID, AlwaysRegenerateIndex: options.AlwaysRegenerateIndex, + JSONIndex: options.JSONIndex, } if server.WebTemplatePath != "" { diff --git a/pkg/config/vars.go b/pkg/config/vars.go index e3cfd0c1..668b2637 100644 --- a/pkg/config/vars.go +++ b/pkg/config/vars.go @@ -138,6 +138,15 @@ var configVars = map[string]configVar{ EnvVar: "DISABLE_STATEFILES", }, }, + "json-index": { + Type: boolType, + Default: false, + CLIFlag: cli.BoolFlag{ + Name: "json-index", + Usage: "generates an index in JSON format, improves parsing performance for large index files", + EnvVar: "JSON_INDEX", + }, + }, "allowoverwrite": { Type: boolType, Default: false, diff --git a/pkg/repo/index.go b/pkg/repo/index.go index 8ac1e54f..1f68d5b7 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -52,7 +52,7 @@ type ( Raw []byte `json:"c"` ChartURL string `json:"d"` IndexLock sync.RWMutex - outputJSON bool + OutputJSON bool } ) @@ -75,7 +75,7 @@ func (index *Index) Regenerate() (err error) { index.Generated = time.Now().Round(time.Second) var raw []byte - if index.outputJSON { + if index.OutputJSON { raw, err = json.Marshal(index.IndexFile) } else { raw, err = yaml.Marshal(index.IndexFile)