Skip to content

Commit

Permalink
Merge pull request #1705 from diggerhq/feat/ee-gh-drift-handle-reconc…
Browse files Browse the repository at this point in the history
…iliation

Feat/ee gh drift handle reconciliation
  • Loading branch information
ZIJ authored Sep 12, 2024
2 parents 8c13c7c + 2027648 commit e213714
Show file tree
Hide file tree
Showing 6 changed files with 294 additions and 15 deletions.
32 changes: 23 additions & 9 deletions backend/controllers/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,12 @@ import (
"golang.org/x/oauth2"
)

type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error

type DiggerController struct {
CiBackendProvider ci_backends.CiBackendProvider
GithubClientProvider utils.GithubClientProvider
CiBackendProvider ci_backends.CiBackendProvider
GithubClientProvider utils.GithubClientProvider
GithubWebhookPostIssueCommentHooks []IssueCommentHook
}

func (d DiggerController) GithubAppWebHook(c *gin.Context) {
Expand Down Expand Up @@ -89,6 +92,16 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
c.String(http.StatusInternalServerError, err.Error())
return
}

log.Printf("executing issue comment event post hooks:")
for _, hook := range d.GithubWebhookPostIssueCommentHooks {
err := hook(gh, event, d.CiBackendProvider)
if err != nil {
log.Printf("handleIssueCommentEvent post hook error: %v", err)
c.String(http.StatusInternalServerError, err.Error())
return
}
}
case *github.PullRequestEvent:
log.Printf("Got pull request event for %d", *event.PullRequest.ID)
err := handlePullRequestEvent(gh, event, d.CiBackendProvider)
Expand Down Expand Up @@ -599,7 +612,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
return nil
}

func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
ghService, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
if err != nil {
log.Printf("Error getting github service: %v", err)
Expand All @@ -610,11 +623,6 @@ func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
var diggerYmlStr string
var dependencyGraph graph.Graph[string, dg_configuration.Project]

changedFiles, err := ghService.GetChangedFiles(prNumber)
if err != nil {
log.Printf("Error getting changed files: %v", err)
return "", nil, nil, nil, fmt.Errorf("error getting changed files")
}
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
diggerYmlStr = string(diggerYmlBytes)
Expand Down Expand Up @@ -649,7 +657,13 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting branch name")
}

diggerYmlStr, ghService, config, dependencyGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, prNumber)
changedFiles, err := ghService.GetChangedFiles(prNumber)
if err != nil {
log.Printf("Error getting changed files: %v", err)
return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting changed files")
}

diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles)
if err != nil {
log.Printf("Error loading digger.yml: %v", err)
return "", nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml")
Expand Down
6 changes: 6 additions & 0 deletions backend/controllers/projects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) {
defer teardownSuite(t)
isMergeCalled := false
mockedHTTPClient := mock.NewMockedHTTPClient(
mock.WithRequestMatch(
mock.GetReposIssuesByOwnerByRepoByIssueNumber,
github.Issue{
Number: github.Int(2),
PullRequestLinks: &github.PullRequestLinks{},
}),
mock.WithRequestMatch(
mock.GetReposPullsByOwnerByRepoByPullNumber,
github.PullRequest{
Expand Down
5 changes: 3 additions & 2 deletions backend/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ var templates embed.FS

func main() {
ghController := controllers.DiggerController{
CiBackendProvider: ci_backends.DefaultBackendProvider{},
GithubClientProvider: utils.DiggerGithubRealClientProvider{},
CiBackendProvider: ci_backends.DefaultBackendProvider{},
GithubClientProvider: utils.DiggerGithubRealClientProvider{},
GithubWebhookPostIssueCommentHooks: make([]controllers.IssueCommentHook, 0),
}
r := bootstrap.Bootstrap(templates, ghController)
r.GET("/", controllers.Home)
Expand Down
191 changes: 191 additions & 0 deletions ee/backend/hooks/github.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package hooks

import (
"fmt"
"github.com/diggerhq/digger/backend/ci_backends"
ce_controllers "github.com/diggerhq/digger/backend/controllers"
"github.com/diggerhq/digger/backend/locking"
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/backend/utils"
"github.com/diggerhq/digger/libs/ci/generic"
dg_github "github.com/diggerhq/digger/libs/ci/github"
comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting"
"github.com/diggerhq/digger/libs/digger_config"
dg_locking "github.com/diggerhq/digger/libs/locking"
"github.com/diggerhq/digger/libs/scheduler"
"github.com/google/go-github/v61/github"
"github.com/samber/lo"
"log"
"regexp"
"strconv"
"strings"
)

var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
log.Printf("handling the drift reconcilliation hook")
installationId := *payload.Installation.ID
repoName := *payload.Repo.Name
repoOwner := *payload.Repo.Owner.Login
repoFullName := *payload.Repo.FullName
cloneURL := *payload.Repo.CloneURL
issueTitle := *payload.Issue.Title
issueNumber := *payload.Issue.Number
userCommentId := *payload.GetComment().ID
actor := *payload.Sender.Login
commentBody := *payload.Comment.Body
defaultBranch := *payload.Repo.DefaultBranch
isPullRequest := payload.Issue.IsPullRequest()

if isPullRequest {
log.Printf("Comment is not an issue, ignoring")
return nil
}

// checking that the title of the issue matches regex
var projectName string
re := regexp.MustCompile(`^Drift detected in project:\s*(\S+)`)
matches := re.FindStringSubmatch(issueTitle)
if len(matches) > 1 {
projectName = matches[1]
} else {
log.Printf("does not look like a drift issue, ignoring")
}

link, err := models.DB.GetGithubAppInstallationLink(installationId)
if err != nil {
log.Printf("Error getting GetGithubAppInstallationLink: %v", err)
return fmt.Errorf("error getting github app link")
}
orgId := link.OrganisationId

if *payload.Action != "created" {
log.Printf("comment is not of type 'created', ignoring")
return nil
}

allowedCommands := []string{"digger apply", "digger unlock"}
if !lo.Contains(allowedCommands, strings.TrimSpace(*payload.Comment.Body)) {
log.Printf("comment is not in allowed commands, ignoring")
log.Printf("allowed commands: %v", allowedCommands)
return nil
}

diggerYmlStr, ghService, config, projectsGraph, err := ce_controllers.GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, defaultBranch, nil)
if err != nil {
log.Printf("Error loading digger.yml: %v", err)
return fmt.Errorf("error loading digger.yml")
}

commentIdStr := strconv.FormatInt(userCommentId, 10)
err = ghService.CreateCommentReaction(commentIdStr, string(dg_github.GithubCommentEyesReaction))
if err != nil {
log.Printf("CreateCommentReaction error: %v", err)
}

diggerCommand, err := scheduler.GetCommandFromComment(*payload.Comment.Body)
if err != nil {
log.Printf("unkown digger command in comment: %v", *payload.Comment.Body)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Could not recognise comment, error: %v", err))
return fmt.Errorf("unkown digger command in comment %v", err)
}

// attempting to lock for performing drift apply command
prLock := dg_locking.PullRequestLock{
InternalLock: locking.BackendDBLock{
OrgId: orgId,
},
CIService: ghService,
Reporter: comment_updater.NoopReporter{},
ProjectName: projectName,
ProjectNamespace: repoFullName,
PrNumber: issueNumber,
}
err = dg_locking.PerformLockingActionFromCommand(prLock, *diggerCommand)
if err != nil {
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Failed perform lock action on project: %v %v", projectName, err))
return fmt.Errorf("failed perform lock action on project: %v %v", projectName, err)
}

if *diggerCommand == scheduler.DiggerCommandUnlock {
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":white_check_mark: Command %v completed successfully", *diggerCommand))
return nil
}

// === if we get here its a "digger apply command and we are already locked for this project ====
// perform apply here then unlock the project
commentReporter, err := utils.InitCommentReporter(ghService, issueNumber, ":construction_worker: Digger starting....")
if err != nil {
log.Printf("Error initializing comment reporter: %v", err)
return fmt.Errorf("error initializing comment reporter")
}

impactedProjects := config.GetProjects(projectName)
jobs, _, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, nil, config.Workflows, defaultBranch, defaultBranch)
if err != nil {
log.Printf("Error converting event to jobs: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Error converting event to jobs: %v", err))
return fmt.Errorf("error converting event to jobs")
}
log.Printf("GitHub IssueComment event converted to Jobs successfully\n")

err = utils.ReportInitialJobsStatus(commentReporter, jobs)
if err != nil {
log.Printf("Failed to comment initial status for jobs: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Failed to comment initial status for jobs: %v", err))
return fmt.Errorf("failed to comment initial status for jobs")
}

impactedProjectsMap := make(map[string]digger_config.Project)
for _, p := range impactedProjects {
impactedProjectsMap[p.Name] = p
}

impactedProjectsJobMap := make(map[string]scheduler.Job)
for _, j := range jobs {
impactedProjectsJobMap[j.ProjectName] = j
}

reporterCommentId, err := strconv.ParseInt(commentReporter.CommentId, 10, 64)
if err != nil {
log.Printf("strconv.ParseInt error: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err))
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0)
if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
return fmt.Errorf("error convertingjobs")
}

ciBackend, err := ciBackendProvider.GetCiBackend(
ci_backends.CiBackendOptions{
GithubClientProvider: gh,
GithubInstallationId: installationId,
RepoName: repoName,
RepoOwner: repoOwner,
RepoFullName: repoFullName,
},
)
if err != nil {
log.Printf("GetCiBackend error: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: GetCiBackend error: %v", err))
return fmt.Errorf("error fetching ci backed %v", err)
}

err = ce_controllers.TriggerDiggerJobs(ciBackend, repoFullName, repoOwner, repoName, batchId, issueNumber, ghService, gh)
if err != nil {
log.Printf("TriggerDiggerJobs error: %v", err)
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err))
return fmt.Errorf("error triggerring Digger Jobs")
}

// === now unlocking the project ===
err = dg_locking.PerformLockingActionFromCommand(prLock, scheduler.DiggerCommandUnlock)
if err != nil {
utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Failed perform lock action on project: %v %v", projectName, err))
return fmt.Errorf("failed perform lock action on project: %v %v", projectName, err)
}

return nil
}
8 changes: 5 additions & 3 deletions ee/backend/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/diggerhq/digger/backend/utils"
ci_backends2 "github.com/diggerhq/digger/ee/backend/ci_backends"
"github.com/diggerhq/digger/ee/backend/controllers"
"github.com/diggerhq/digger/ee/backend/hooks"
"github.com/diggerhq/digger/ee/backend/providers/github"
"github.com/diggerhq/digger/libs/license"
"github.com/gin-gonic/gin"
Expand All @@ -31,8 +32,9 @@ func main() {
os.Exit(1)
}
diggerController := ce_controllers.DiggerController{
CiBackendProvider: ci_backends2.EEBackendProvider{},
GithubClientProvider: github.DiggerGithubEEClientProvider{},
CiBackendProvider: ci_backends2.EEBackendProvider{},
GithubClientProvider: github.DiggerGithubEEClientProvider{},
GithubWebhookPostIssueCommentHooks: []ce_controllers.IssueCommentHook{hooks.DriftReconcilliationHook},
}

r := bootstrap.Bootstrap(templates, diggerController)
Expand Down Expand Up @@ -92,7 +94,7 @@ func main() {
jobArtefactsGroup.Use(middleware.GetApiMiddleware())
jobArtefactsGroup.PUT("/", controllers.SetJobArtefact)
jobArtefactsGroup.GET("/", controllers.DownloadJobArtefact)

port := config.GetPort()
r.Run(fmt.Sprintf(":%d", port))
}
Expand Down
Loading

0 comments on commit e213714

Please sign in to comment.