implement caching layer for digger config

backend/bootstrap/main.go

@@ -221,6 +221,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 	runsApiGroup.GET("/:run_id", controllers.RunDetails)
 	runsApiGroup.POST("/:run_id/approve", controllers.ApproveRun)
 
+	// internal endpoints not meant to be exposed to public and protected behing webhook secret
+	r.POST("_internal/update_repo_cache", middleware.WebhookAuth(), diggerController.UpdateRepoCache)
+
 	fronteggWebhookProcessor.POST("/create-org-from-frontegg", controllers.CreateFronteggOrgFromWebhook)
 
 	return r

backend/controllers/cache.go

@@ -0,0 +1,90 @@
+package controllers
+
+import (
+	"fmt"
+	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
+	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
+	"github.com/dominikbraun/graph"
+	"github.com/gin-gonic/gin"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+)
+
+func (d DiggerController) UpdateRepoCache(c *gin.Context) {
+	type UpdateCacheRequest struct {
+		RepoFullName string `json:"repo_full_name"`
+		Branch       string `json:"branch"`
+		orgId        uint   `json:"org_id"`
+	}
+
+	var request UpdateCacheRequest
+	err := c.BindJSON(&request)
+	if err != nil {
+		log.Printf("Error binding JSON: %v", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error binding JSON"})
+		return
+	}
+
+	orgId := request.orgId
+	repoFullName := request.RepoFullName
+
+	repoOwner, repoName, _ := strings.Cut(repoFullName, "/")
+
+	repo, err := models.DB.GetRepo(orgId, repoName)
+	if err != nil {
+		log.Printf("could not get repo: %v", err)
+		c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
+		return
+	}
+
+	cloneUrl := repo.RepoUrl
+	branch := request.Branch
+
+	ghInstallation, err := models.DB.GetInstallationForRepo(repoFullName)
+	if err != nil {
+		log.Printf("could not get repo: %v", err)
+		c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
+		return
+	}
+
+	_, token, err := utils.GetGithubService(d.GithubClientProvider, ghInstallation.GithubInstallationId, repoFullName, repoOwner, repoName)
+	if err != nil {
+		log.Printf("could not get github service :%v", err)
+		c.String(500, fmt.Sprintf("could not get github service %v %v", repoFullName, orgId))
+		return
+	}
+
+	var diggerYmlStr string
+	var config *dg_configuration.DiggerConfig
+	var dependencyGraph graph.Graph[string, dg_configuration.Project]
+	err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
+		diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
+		diggerYmlStr = string(diggerYmlBytes)
+		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
+		if err != nil {
+			log.Printf("Error loading digger config: %v", err)
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("could not load digger config :%v", err)
+		c.String(500, fmt.Sprintf("could load digger config %v %v", repoFullName, orgId))
+		return
+	}
+	// update the cache here
+	_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config, dependencyGraph)
+	if err != nil {
+		log.Printf("could upadate repo cache :%v", err)
+		c.String(500, fmt.Sprintf("could not udpate repo cache %v %v", repoFullName, orgId))
+		return
+
+	}
+
+	c.String(200, "ok")
+}

backend/controllers/github.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -316,6 +317,10 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	commitSha := payload.PullRequest.Head.GetSHA()
 	branch := payload.PullRequest.Head.GetRef()
 	action := *payload.Action
+	labels := payload.PullRequest.Labels
+	prLabelsStr := lo.Map(labels, func(label *github.Label, i int) string {
+		return *label.Name
+	})
 
 	link, err := models.DB.GetGithubAppInstallationLink(installationId)
 	if err != nil {
@@ -361,7 +366,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		}
 	}
 
-	diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
+	diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, organisationId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
 	if err != nil {
 		log.Printf("getDiggerConfigForPR error: %v", err)
 		return fmt.Errorf("error getting digger config")
@@ -563,7 +568,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 }
 
 // TODO: Refactor this func to receive ghService as input
-func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
+func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []string, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
 	ghService, _, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
 	if err != nil {
 		log.Printf("Error getting github service: %v", err)
@@ -583,6 +588,17 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
 		return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting changed files")
 	}
 
+	// check if items should be loaded from cache
+	if val, _ := os.LookupEnv("DIGGER_CONFIG_REPO_CACHE"); val == "1" && !slices.Contains(prLabels, "digger:no-cache") {
+		diggerYmlStr, config, dependencyGraph, err := retrieveConfigFromCache(orgId, repoFullName)
+		if err != nil {
+			log.Printf("could not load from cache")
+		} else {
+			log.Printf("successfully loaded from cache")
+			return diggerYmlStr, ghService, config, *dependencyGraph, &prBranch, &prCommitSha, nil
+		}
+	}
+
 	diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles)
 	if err != nil {
 		log.Printf("Error loading digger.yml: %v", err)
@@ -593,6 +609,28 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
 	return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, nil
 }
 
+func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], error) {
+	repoCache, err := models.DB.GetRepoCache(orgId, repoFullName)
+	if err != nil {
+		log.Printf("Error: failed to load repoCache, going to try live load %v", err)
+		return "", nil, nil, fmt.Errorf("")
+	}
+	var config dg_configuration.DiggerConfig
+	err = json.Unmarshal(repoCache.DiggerConfig, &config)
+	if err != nil {
+		log.Printf("Error: failed to load repoCache unmarshall config %v", err)
+		return "", nil, nil, fmt.Errorf("failed to load repoCache unmarshall config %v", err)
+	}
+
+	var ProjectGraph graph.Graph[string, dg_configuration.Project]
+	err = json.Unmarshal(repoCache.ProjectsGraph, &ProjectGraph)
+	if err != nil {
+		log.Printf("Error: failed to load repoCache unmarshall ProjectGraph %v", err)
+		return "", nil, nil, fmt.Errorf("failed to load repoCache unmarshall ProjectGraph %v", err)
+	}
+	return repoCache.DiggerYmlStr, &config, &ProjectGraph, nil
+}
+
 func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {
 	link, err := models.DB.GetGithubAppInstallationLink(installationId)
 	if err != nil {
@@ -634,6 +672,10 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 	commentBody := *payload.Comment.Body
 	defaultBranch := *payload.Repo.DefaultBranch
 	isPullRequest := payload.Issue.IsPullRequest()
+	labels := payload.Issue.Labels
+	prLabelsStr := lo.Map(labels, func(label *github.Label, i int) string {
+		return *label.Name
+	})
 
 	if !isPullRequest {
 		log.Printf("comment not on pullrequest, ignroning")
@@ -671,7 +713,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		}
 	}
 
-	diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
+	diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
 	if err != nil {
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not load digger config, error: %v", err))
 		log.Printf("getDiggerConfigForPR error: %v", err)