From d27b91cfd0e3ff6f092c7034e81bfefc946aaf04 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 27 May 2024 17:51:50 +0100 Subject: [PATCH 1/2] add updating comments --- backend/controllers/github.go | 2 +- backend/controllers/projects.go | 51 ++++++++++- backend/models/scheduler.go | 10 +-- cli/pkg/core/execution/execution.go | 2 +- cli/pkg/digger/digger.go | 2 +- cli/pkg/locking/locking.go | 2 +- libs/comment_utils/reporting/reporting.go | 2 +- .../reporting/source_grouping.go | 86 ++++++++++--------- .../{summary => reporting}/utils.go | 7 +- .../comment_utils}/utils/comments.go | 10 ++- 10 files changed, 117 insertions(+), 57 deletions(-) rename libs/comment_utils/{summary => reporting}/utils.go (84%) rename {cli/pkg/core => libs/comment_utils}/utils/comments.go (80%) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 3a9a751c8..ca3672df0 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/diggerhq/digger/backend/segment" "github.com/diggerhq/digger/backend/services" - comment_updater "github.com/diggerhq/digger/libs/comment_utils/summary" + comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting" orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler" "github.com/google/uuid" "log" diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index 6218d919c..ad22bca9f 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -8,6 +8,7 @@ import ( "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/services" "github.com/diggerhq/digger/backend/utils" + "github.com/diggerhq/digger/libs/comment_utils/reporting" "github.com/diggerhq/digger/libs/digger_config" "github.com/diggerhq/digger/libs/orchestrator" orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler" @@ -490,7 +491,8 @@ func SetJobStatusForProject(c *gin.Context) { } - log.Printf("!!!Batch to json struct: %v", res) + UpdateCommentsForBatchGroup(&utils.DiggerGithubRealClientProvider{}, batch, res.Jobs) + c.JSON(http.StatusOK, res) } @@ -570,6 +572,53 @@ func CreateRunForProject(c *gin.Context) { c.JSON(http.StatusOK, run.MapToJsonStruct()) } +func UpdateCommentsForBatchGroup(gh utils.GithubClientProvider, batch *models.DiggerBatch, serializedJobs []orchestrator_scheduler.SerializedJob) error { + diggerYmlString := batch.DiggerConfig + diggerConfigYml, err := digger_config.LoadDiggerConfigYamlFromString(diggerYmlString) + if err != nil { + log.Printf("Error loading digger config from batch: %v", err) + return fmt.Errorf("error loading digger config from batch: %v", err) + } + + if diggerConfigYml.CommentRenderMode != nil && + *diggerConfigYml.CommentRenderMode != digger_config.CommentRenderModeGroupByModule { + log.Printf("render mode is not group_by_module, skipping") + return nil + } + + ghService, _, err := utils.GetGithubService( + gh, + batch.GithubInstallationId, + batch.RepoFullName, + batch.RepoOwner, + batch.RepoName, + ) + + var sourceDetails []reporting.SourceDetails + err = json.Unmarshal(batch.SourceDetails, &sourceDetails) + if err != nil { + log.Printf("failed to unmarshall sourceDetails: %v", err) + return fmt.Errorf("failed to unmarshall sourceDetails: %v", err) + } + + // project_name => terraform output + projectToTerraformOutput := make(map[string]string) + // TODO: add projectName as a field of Job + for _, serialJob := range serializedJobs { + job, err := models.DB.GetDiggerJob(serialJob.DiggerJobId) + if err != nil { + return fmt.Errorf("Could not get digger job: %v", err) + } + projectToTerraformOutput[serialJob.ProjectName] = job.TerraformOutput + } + + for _, detail := range sourceDetails { + reporter := reporting.SourceGroupingReporter{serializedJobs, batch.PrNumber, ghService} + reporter.UpdateComment(sourceDetails, detail.SourceLocation, projectToTerraformOutput) + } + return nil +} + func AutomergePRforBatchIfEnabled(gh utils.GithubClientProvider, batch *models.DiggerBatch) error { diggerYmlString := batch.DiggerConfig diggerConfigYml, err := digger_config.LoadDiggerConfigYamlFromString(diggerYmlString) diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 0635cb426..29734a674 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -85,7 +85,7 @@ type GithubDiggerJobLink struct { Status DiggerJobLinkStatus } -func (j *DiggerJob) MapToJsonStruct() (interface{}, error) { +func (j *DiggerJob) MapToJsonStruct() (orchestrator_scheduler.SerializedJob, error) { var job orchestrator.JobJson err := json.Unmarshal(j.SerializedJobSpec, &job) if err != nil { @@ -104,7 +104,7 @@ func (j *DiggerJob) MapToJsonStruct() (interface{}, error) { ResourcesDeleted: j.DiggerJobSummary.ResourcesDeleted, }, nil } -func (b *DiggerBatch) MapToJsonStruct() (interface{}, error) { +func (b *DiggerBatch) MapToJsonStruct() (orchestrator_scheduler.SerializedBatch, error) { res := orchestrator_scheduler.SerializedBatch{ ID: b.ID.String(), PrNumber: b.PrNumber, @@ -119,14 +119,14 @@ func (b *DiggerBatch) MapToJsonStruct() (interface{}, error) { serializedJobs := make([]orchestrator_scheduler.SerializedJob, 0) jobs, err := DB.GetDiggerJobsForBatch(b.ID) if err != nil { - return nil, fmt.Errorf("could not unmarshall digger batch: %v", err) + return res, fmt.Errorf("could not unmarshall digger batch: %v", err) } for _, job := range jobs { jobJson, err := job.MapToJsonStruct() if err != nil { - return nil, fmt.Errorf("error mapping job to struct (ID: %v); %v", job.ID, err) + return res, fmt.Errorf("error mapping job to struct (ID: %v); %v", job.ID, err) } - serializedJobs = append(serializedJobs, jobJson.(orchestrator_scheduler.SerializedJob)) + serializedJobs = append(serializedJobs, jobJson) } res.Jobs = serializedJobs return res, nil diff --git a/cli/pkg/core/execution/execution.go b/cli/pkg/core/execution/execution.go index ba9760fa2..781119135 100644 --- a/cli/pkg/core/execution/execution.go +++ b/cli/pkg/core/execution/execution.go @@ -2,6 +2,7 @@ package execution import ( "fmt" + "github.com/diggerhq/digger/libs/comment_utils/utils" "github.com/diggerhq/digger/libs/terraform_utils" "github.com/samber/lo" "log" @@ -15,7 +16,6 @@ import ( "github.com/diggerhq/digger/cli/pkg/core/runners" "github.com/diggerhq/digger/cli/pkg/core/storage" "github.com/diggerhq/digger/cli/pkg/core/terraform" - "github.com/diggerhq/digger/cli/pkg/core/utils" "github.com/diggerhq/digger/libs/comment_utils/reporting" configuration "github.com/diggerhq/digger/libs/digger_config" "github.com/diggerhq/digger/libs/orchestrator" diff --git a/cli/pkg/digger/digger.go b/cli/pkg/digger/digger.go index f8bdde0de..fa3614b53 100644 --- a/cli/pkg/digger/digger.go +++ b/cli/pkg/digger/digger.go @@ -3,6 +3,7 @@ package digger import ( "errors" "fmt" + coreutils "github.com/diggerhq/digger/libs/comment_utils/utils" "log" "os" "path" @@ -19,7 +20,6 @@ import ( "github.com/diggerhq/digger/cli/pkg/core/runners" "github.com/diggerhq/digger/cli/pkg/core/storage" "github.com/diggerhq/digger/cli/pkg/core/terraform" - coreutils "github.com/diggerhq/digger/cli/pkg/core/utils" "github.com/diggerhq/digger/cli/pkg/locking" "github.com/diggerhq/digger/cli/pkg/usage" utils "github.com/diggerhq/digger/cli/pkg/utils" diff --git a/cli/pkg/locking/locking.go b/cli/pkg/locking/locking.go index 28912f304..733e28627 100644 --- a/cli/pkg/locking/locking.go +++ b/cli/pkg/locking/locking.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/diggerhq/digger/libs/comment_utils/utils" "log" "os" "strconv" @@ -13,7 +14,6 @@ import ( "github.com/diggerhq/digger/cli/pkg/aws/envprovider" "github.com/diggerhq/digger/cli/pkg/azure" "github.com/diggerhq/digger/cli/pkg/core/locking" - "github.com/diggerhq/digger/cli/pkg/core/utils" "github.com/diggerhq/digger/cli/pkg/gcp" "github.com/diggerhq/digger/libs/comment_utils/reporting" "github.com/diggerhq/digger/libs/orchestrator" diff --git a/libs/comment_utils/reporting/reporting.go b/libs/comment_utils/reporting/reporting.go index be26e0d59..5263893e4 100644 --- a/libs/comment_utils/reporting/reporting.go +++ b/libs/comment_utils/reporting/reporting.go @@ -2,7 +2,7 @@ package reporting import ( "fmt" - "github.com/diggerhq/digger/cli/pkg/core/utils" + "github.com/diggerhq/digger/libs/comment_utils/utils" "github.com/diggerhq/digger/libs/orchestrator" "log" "strings" diff --git a/libs/comment_utils/reporting/source_grouping.go b/libs/comment_utils/reporting/source_grouping.go index 1a8c87c7e..fbf25ae01 100644 --- a/libs/comment_utils/reporting/source_grouping.go +++ b/libs/comment_utils/reporting/source_grouping.go @@ -3,12 +3,14 @@ package reporting import ( "encoding/json" "fmt" + "github.com/diggerhq/digger/libs/comment_utils/utils" "github.com/diggerhq/digger/libs/digger_config" "github.com/diggerhq/digger/libs/orchestrator" "github.com/diggerhq/digger/libs/orchestrator/scheduler" "github.com/diggerhq/digger/libs/terraform_utils" "github.com/samber/lo" "log" + "strconv" ) type ProjectNameSourceDetail struct { @@ -25,24 +27,29 @@ type SourceGroupingReporter struct { PrService orchestrator.PullRequestService } -func (r SourceGroupingReporter) Report(report string, reportFormatting func(report string) string) (string, string, error) { +func (r SourceGroupingReporter) UpdateComment(sourceDetails []SourceDetails, location string, terraformOutputs map[string]string) error { jobSpecs, err := scheduler.GetJobSpecs(r.Jobs) if err != nil { - return "", "", fmt.Errorf("could not get job specs: %v", err) + return fmt.Errorf("could not get job specs: %v", err) } - //impactedSources := jobSpecs[0].ImpactedSources - // TODO: populate from batch field - var impactedSources map[string]digger_config.ProjectToSourceMapping + sourceDetaiItem, found := lo.Find(sourceDetails, func(item SourceDetails) bool { + return item.SourceLocation == location + }) + + if !found { + log.Printf("location not found in sourcedetails list") + return fmt.Errorf("location not found in sourcedetails list") + } projectNameToJobMap, err := scheduler.JobsToProjectMap(r.Jobs) if err != nil { - return "", "", fmt.Errorf("could not convert jobs to map: %v", err) + return fmt.Errorf("could not convert jobs to map: %v", err) } projectNameToJobSpecMap, err := orchestrator.JobsSpecsToProjectMap(jobSpecs) if err != nil { - return "", "", fmt.Errorf("could not convert jobs to map: %v", err) + return fmt.Errorf("could not convert jobs to map: %v", err) } projectNameToFootPrintMap := make(map[string]terraform_utils.TerraformPlanFootprint) @@ -52,7 +59,7 @@ func (r SourceGroupingReporter) Report(report string, reportFormatting func(repo err := json.Unmarshal(job.PlanFootprint, &footprint) if err != nil { log.Printf("could not unmarshal footprint: %v", err) - return "", "", fmt.Errorf("could not unmarshal footprint: %v", err) + return fmt.Errorf("could not unmarshal footprint: %v", err) } } else { footprint = terraform_utils.TerraformPlanFootprint{} @@ -60,43 +67,42 @@ func (r SourceGroupingReporter) Report(report string, reportFormatting func(repo projectNameToFootPrintMap[job.ProjectName] = footprint } - groupsToProjectMap := ImpactedSourcesMapToGroupMapping(impactedSources, projectNameToJobMap, projectNameToJobSpecMap, projectNameToFootPrintMap) - - message := ":construction_worker: Jobs status:\n\n" - for sourceLocation, projectSourceDetailList := range groupsToProjectMap { - footprints := lo.Map(projectSourceDetailList, func(detail ProjectNameSourceDetail, i int) terraform_utils.TerraformPlanFootprint { - return detail.PlanFootPrint - }) - allSimilarInGroup, err := terraform_utils.SimilarityCheck(footprints) - if err != nil { - return "", "", fmt.Errorf("error performing similar check: %v", err) + footprints := lo.FilterMap(sourceDetaiItem.Projects, func(project string, i int) (terraform_utils.TerraformPlanFootprint, bool) { + if projectNameToJobMap[project].Status == scheduler.DiggerJobSucceeded { + return projectNameToFootPrintMap[project], true } + return terraform_utils.TerraformPlanFootprint{}, false + }) + allSimilarInGroup, err := terraform_utils.SimilarityCheck(footprints) + if err != nil { + return fmt.Errorf("error performing similar check: %v", err) + } - message = message + fmt.Sprintf("# Group: %v (similar: %v)", sourceLocation, allSimilarInGroup) - for _, projectSourceDetail := range projectSourceDetailList { - job := projectSourceDetail.Job - jobSpec := projectSourceDetail.JobSpec - isPlan := jobSpec.IsPlan() - message = message + fmt.Sprintf("\n", job.ProjectName) - message = message + fmt.Sprintf("%v **%v** %v%v\n", job.Status.ToEmoji(), job.ProjectName, *job.WorkflowRunUrl, job.Status.ToString(), job.ResourcesSummaryString(isPlan)) - message = message + fmt.Sprintf("\n", job.ProjectName) + message := "" + message = message + fmt.Sprintf("# Group: %v (similar: %v)\n", location, allSimilarInGroup) + for i, project := range sourceDetaiItem.Projects { + job := projectNameToJobMap[project] + if job.Status != scheduler.DiggerJobSucceeded { + continue } - + jobSpec := projectNameToJobSpecMap[project] + isPlan := jobSpec.JobType == orchestrator.DiggerCommandPlan + expanded := i == 0 || !allSimilarInGroup + var commenter func(terraformOutput string) string + if isPlan { + commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Plan for %v", project), expanded) + } else { + commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Apply for %v", project), expanded) + } + message = message + commenter(terraformOutputs[project]) + "\n" } - r.PrService.PublishComment(r.PrNumber, message) - return "", "", nil -} - -func (reporter SourceGroupingReporter) Flush() (string, string, error) { - return "", "", nil -} - -func (reporter SourceGroupingReporter) SupportsMarkdown() bool { - return false -} - -func (reporter SourceGroupingReporter) Suppress() error { + CommentId, err := strconv.ParseInt(sourceDetaiItem.CommentId, 10, 64) + if err != nil { + log.Printf("Could not convert commentId to int64: %v", err) + return fmt.Errorf("could not convert commentId to int64: %v", err) + } + r.PrService.EditComment(r.PrNumber, CommentId, message) return nil } diff --git a/libs/comment_utils/summary/utils.go b/libs/comment_utils/reporting/utils.go similarity index 84% rename from libs/comment_utils/summary/utils.go rename to libs/comment_utils/reporting/utils.go index 4ac174cfe..277fe163e 100644 --- a/libs/comment_utils/summary/utils.go +++ b/libs/comment_utils/reporting/utils.go @@ -1,8 +1,7 @@ -package comment_updater +package reporting import ( "fmt" - "github.com/diggerhq/digger/libs/comment_utils/reporting" dg_configuration "github.com/diggerhq/digger/libs/digger_config" dg_github "github.com/diggerhq/digger/libs/orchestrator/github" "log" @@ -26,10 +25,10 @@ func PostInitialSourceComments(ghService *dg_github.GithubService, prNumber int, } } for location, projects := range locations { - reporter := reporting.CiReporter{ + reporter := CiReporter{ PrNumber: prNumber, CiService: ghService, - ReportStrategy: reporting.CommentPerRunStrategy{fmt.Sprintf("Report for location: %v", location), time.Now()}, + ReportStrategy: CommentPerRunStrategy{fmt.Sprintf("Report for location: %v", location), time.Now()}, } commentId, _, err := reporter.Report("Comment Reporter", func(report string) string { return "" }) if err != nil { diff --git a/cli/pkg/core/utils/comments.go b/libs/comment_utils/utils/comments.go similarity index 80% rename from cli/pkg/core/utils/comments.go rename to libs/comment_utils/utils/comments.go index 497461fff..1f1cf21ae 100644 --- a/cli/pkg/core/utils/comments.go +++ b/libs/comment_utils/utils/comments.go @@ -3,14 +3,20 @@ package utils import "fmt" func GetTerraformOutputAsCollapsibleComment(summary string, open bool) func(string) string { + var openTag string + if open { + openTag = "open=\"true\"" + } else { + openTag = "" + } return func(comment string) string { - return fmt.Sprintf(`
`+summary+` + return fmt.Sprintf(`
`+summary+` `+"```terraform"+` `+comment+` `+"```"+` -
`, open) +
`, openTag) } } From df7538422fa15c28b6b11b5f45905e9be5b71975 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 27 May 2024 18:00:48 +0100 Subject: [PATCH 2/2] fix tests --- cli/pkg/digger/digger_test.go | 2 +- libs/comment_utils/reporting/source_grouping.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/pkg/digger/digger_test.go b/cli/pkg/digger/digger_test.go index 53709552c..4e652cd5c 100644 --- a/cli/pkg/digger/digger_test.go +++ b/cli/pkg/digger/digger_test.go @@ -281,7 +281,7 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) { commandStrings := allCommandsInOrderWithParams(terraformExecutor, commandRunner, prManager, lock, planStorage, planPathProvider) - assert.Equal(t, []string{"RetrievePlan plan", "Init ", "Apply -lock-timeout=3m", "PublishComment 1
Apply output\n\n```terraform\n\n```\n
", "Run echo"}, commandStrings) + assert.Equal(t, []string{"RetrievePlan plan", "Init ", "Apply -lock-timeout=3m", "PublishComment 1
Apply output\n\n```terraform\n\n```\n
", "Run echo"}, commandStrings) } func TestCorrectCommandExecutionWhenDestroying(t *testing.T) { diff --git a/libs/comment_utils/reporting/source_grouping.go b/libs/comment_utils/reporting/source_grouping.go index fbf25ae01..7009e34b6 100644 --- a/libs/comment_utils/reporting/source_grouping.go +++ b/libs/comment_utils/reporting/source_grouping.go @@ -92,7 +92,7 @@ func (r SourceGroupingReporter) UpdateComment(sourceDetails []SourceDetails, loc if isPlan { commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Plan for %v", project), expanded) } else { - commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Apply for %v", project), expanded) + commenter = utils.GetTerraformOutputAsCollapsibleComment(fmt.Sprintf("Apply for %v", project), false) } message = message + commenter(terraformOutputs[project]) + "\n" }