From 6649ea3195483538fa5bdc33b228745d1ae2cd37 Mon Sep 17 00:00:00 2001 From: Harold Wanyama Date: Thu, 28 Mar 2024 17:31:17 +0300 Subject: [PATCH] [#3846] Feature/Approval List Date Added - Added script to migrate approval list details to the cla--approvals table - Updated the signatures table with the right date_added column Signed-off-by: Harold Wanyama --- cla-backend-go/approval_list/models.go | 12 + .../cmd/dynamo_events_lambda/main.go | 6 +- .../cmd/migrate_approval_list/main.go | 250 ++++++++++++++++++ cla-backend-go/cmd/server.go | 7 +- cla-backend-go/events/mockrepo.go | 5 + cla-backend-go/events/repository.go | 74 ++++++ cla-backend-go/signatures/repository.go | 128 ++++++++- cla-backend-go/signatures/service.go | 6 + cla-backend-go/utils/constants.go | 12 + cla-backend-go/v2/approvals/models.go | 19 ++ cla-backend-go/v2/approvals/repository.go | 243 +++++++++++++++++ cla-backend-go/v2/signatures/converters.go | 66 ++--- cla-backend-go/v2/signatures/service.go | 7 +- 13 files changed, 783 insertions(+), 52 deletions(-) create mode 100644 cla-backend-go/cmd/migrate_approval_list/main.go create mode 100644 cla-backend-go/v2/approvals/models.go create mode 100644 cla-backend-go/v2/approvals/repository.go diff --git a/cla-backend-go/approval_list/models.go b/cla-backend-go/approval_list/models.go index 08f312b79..d1ba671ce 100644 --- a/cla-backend-go/approval_list/models.go +++ b/cla-backend-go/approval_list/models.go @@ -40,3 +40,15 @@ type CclaWhitelistRequest struct { DateModified string `dynamodbav:"date_modified"` Version string `dynamodbav:"version"` } + +// ApprovalItem data model + +type ApprovalItem struct { + ApprovalID string `dynamodbav:"approval_id"` + SignatureID string `dynamodbav:"signature_id"` + DateAdded string `dynamodbav:"date_added"` + DateCreated string `dynamodbav:"date_created"` + DateModified string `dynamodbav:"date_modified"` + ApprovalName string `dynamodbav:"approval_name"` + ApprovalCriteria string `dynamodbav:"approval_criteria"` +} diff --git a/cla-backend-go/cmd/dynamo_events_lambda/main.go b/cla-backend-go/cmd/dynamo_events_lambda/main.go index 066f0a4e1..ee910d335 100644 --- a/cla-backend-go/cmd/dynamo_events_lambda/main.go +++ b/cla-backend-go/cmd/dynamo_events_lambda/main.go @@ -6,6 +6,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "github.com/communitybridge/easycla/cla-backend-go/project/repository" @@ -35,6 +36,7 @@ import ( "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" + "github.com/communitybridge/easycla/cla-backend-go/v2/approvals" "github.com/communitybridge/easycla/cla-backend-go/v2/dynamo_events" "github.com/communitybridge/easycla/cla-backend-go/token" @@ -100,6 +102,8 @@ func init() { githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage) gitlabOrganizationRepo := gitlab_organizations.NewRepository(awsSession, stage) storeRepo := store.NewRepository(awsSession, stage) + approvalsTableName := fmt.Sprintf("cla-%s-approvals", stage) + approvalRepo := approvals.NewRepository(stage, awsSession, approvalsTableName) token.Init(configFile.Auth0Platform.ClientID, configFile.Auth0Platform.ClientSecret, configFile.Auth0Platform.URL, configFile.Auth0Platform.Audience) github.Init(configFile.GitHub.AppID, configFile.GitHub.AppPrivateKey, configFile.GitHub.AccessToken) @@ -135,7 +139,7 @@ func init() { }) usersService := users.NewService(usersRepo, eventsService) - signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo, gerritService) + signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo, gerritService, approvalRepo) v2RepositoryService := v2Repositories.NewService(repositoriesRepo, v2Repository, projectClaGroupRepo, githubOrganizationsRepo, gitlabOrganizationRepo, eventsService) gitlabOrgService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoryService, projectClaGroupRepo, storeRepo, usersService, signaturesRepo, companyRepo) diff --git a/cla-backend-go/cmd/migrate_approval_list/main.go b/cla-backend-go/cmd/migrate_approval_list/main.go new file mode 100644 index 000000000..cbfaf6089 --- /dev/null +++ b/cla-backend-go/cmd/migrate_approval_list/main.go @@ -0,0 +1,250 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/gofrs/uuid" + "github.com/sirupsen/logrus" + + "github.com/communitybridge/easycla/cla-backend-go/company" + "github.com/communitybridge/easycla/cla-backend-go/events" + v1Models "github.com/communitybridge/easycla/cla-backend-go/gen/v1/models" + "github.com/communitybridge/easycla/cla-backend-go/gerrits" + "github.com/communitybridge/easycla/cla-backend-go/github_organizations" + log "github.com/communitybridge/easycla/cla-backend-go/logging" + "github.com/communitybridge/easycla/cla-backend-go/project/repository" + "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" + "github.com/communitybridge/easycla/cla-backend-go/repositories" + "github.com/communitybridge/easycla/cla-backend-go/signatures" + "github.com/communitybridge/easycla/cla-backend-go/users" + "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/communitybridge/easycla/cla-backend-go/v2/approvals" +) + +var stage string +var approvalRepo approvals.IRepository +var signatureRepo signatures.SignatureRepository +var eventsRepo events.Repository +var usersRepo users.UserRepository +var eventsService events.Service +var awsSession = session.Must(session.NewSession(&aws.Config{})) +var approvalsTableName string +var companyRepo company.IRepository +var v1ProjectClaGroupRepo projects_cla_groups.Repository +var ghRepo repositories.Repository +var gerritsRepo gerrits.Repository +var ghOrgRepo github_organizations.Repository +var gerritService gerrits.Service +var eventFound int +var eventNotFound int +var recordExists int + +type combinedRepo struct { + users.UserRepository + company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository +} + +func init() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + log.Infof("STAGE set to %s\n", stage) + approvalsTableName = fmt.Sprintf("cla-%s-approvals", stage) + approvalRepo = approvals.NewRepository(stage, awsSession, approvalsTableName) + eventsRepo = events.NewRepository(awsSession, stage) + usersRepo = users.NewRepository(awsSession, stage) + companyRepo = company.NewRepository(awsSession, stage) + ghRepo = *repositories.NewRepository(awsSession, stage) + gerritsRepo = gerrits.NewRepository(awsSession, stage) + v1CLAGroupRepo := repository.NewRepository(awsSession, stage, &ghRepo, gerritsRepo, v1ProjectClaGroupRepo) + v1ProjectClaGroupRepo = projects_cla_groups.NewRepository(awsSession, stage) + eventsService = events.NewService(eventsRepo, combinedRepo{ + usersRepo, + companyRepo, + v1CLAGroupRepo, + v1ProjectClaGroupRepo, + }) + ghOrgRepo = github_organizations.NewRepository(awsSession, stage) + gerritService = gerrits.NewService(gerritsRepo, nil) + signatureRepo = signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, &ghRepo, ghOrgRepo, gerritService, approvalRepo) + + log.Info("initialized repositories\n") +} + +func main() { + f := logrus.Fields{ + "functionName": "main", + } + log.WithFields(f).Info("Starting migration") + log.Info("Fetching ccla signatures") + signed := true + approved := true + cclaSignatures, err := signatureRepo.GetCCLASignatures(context.Background(), &signed, &approved) + if err != nil { + log.Fatalf("Error fetching ccla signatures : %v", err) + } + log.Info("Fetched ccla signatures") + eventFound = 0 + eventNotFound = 0 + recordExists = 0 + + var wg sync.WaitGroup + + for _, cclaSignature := range cclaSignatures { + wg.Add(1) + go func(signature *signatures.ItemSignature) { + defer wg.Done() + err := updateApprovalsTable(signature) + if err != nil { + log.WithFields(f).Warnf("Error updating approvals table for signature : %s, error: %v", signature.SignatureID, err) + } + }(cclaSignature) + } + wg.Wait() + log.WithFields(f).Debugf("Events found : %d, Events not found : %d , existing Record count: %d", eventFound, eventNotFound, recordExists) +} + +func updateApprovalsTable(signature *signatures.ItemSignature) error { + f := logrus.Fields{ + "functionName": "updateApprovalsTable", + "signatureID": signature.SignatureID, + } + log.WithFields(f).Debugf("updating approvals table for signature : %s", signature.SignatureID) + var wg sync.WaitGroup + var errMutex sync.Mutex + var err error + + update := func(approvalList []string, listType string) { + defer wg.Done() + for _, item := range approvalList { + searchTerm := fmt.Sprintf("%s was added to the approval list", item) + pageSize := int64(1000) + eventType := events.ClaApprovalListUpdated + + // check if approval item already exists + approvalItems, searchErr := approvalRepo.SearchApprovalList(listType, item, signature.SignatureProjectID, "", signature.SignatureID) + if err != nil { + errMutex.Lock() + err = searchErr + errMutex.Unlock() + log.WithFields(f).Warnf("Error searching approval list for item : %s, error: %v", item, err) + return + } + + if len(approvalItems) > 0 { + log.WithFields(f).Debugf("Approval item already exists for : %s, %s", listType, item) + recordExists++ + return + } + + log.WithFields(f).Debugf("searching for events with search term : %s, projectID: %s, eventType: %s ", searchTerm, signature.SignatureProjectID, eventType) + + events, eventErr := eventsRepo.GetCCLAEvents(signature.SignatureProjectID, signature.SignatureReferenceID, searchTerm, eventType, pageSize) + + if eventErr != nil { + errMutex.Lock() + err = eventErr + errMutex.Unlock() + return + } + + approvalID, approvalErr := uuid.NewV4() + if err != nil { + errMutex.Lock() + err = approvalErr + log.WithFields(f).Warnf("Error creating new UUIDv4, error: %v", err) + errMutex.Unlock() + return + } + currentTime := time.Now().UTC().String() + approvalItem := approvals.ApprovalItem{ + ApprovalID: approvalID.String(), + SignatureID: signature.SignatureID, + DateCreated: currentTime, + DateModified: currentTime, + ApprovalName: item, + ApprovalCriteria: listType, + CompanyID: signature.SignatureReferenceID, + ProjectID: signature.SignatureProjectID, + ApprovalCompanyName: signature.SignatureReferenceName, + } + + log.WithFields(f).Debugf("Adding approval item : %+v", approvalItem) + + if len(events) > 0 { + event := getLatestEvent(events) + approvalItem.DateAdded = event.EventTime + log.WithFields(f).Debugf("found event with id: %s , approval: %+v ", event.EventID, approvalItem) + eventFound++ + } else { + log.WithFields(f).Debugf("no events found for %s: %s", listType, item) + approvalItem.DateAdded = signature.DateModified + eventNotFound++ + } + + log.WithFields(f).Debugf("adding approval item : %+v", approvalItem) + approvalErr = approvalRepo.AddApprovalList(approvalItem) + if err != nil { + errMutex.Lock() + err = approvalErr + errMutex.Unlock() + log.WithFields(f).Warnf("Error adding approval item : %v", err) + return + } + } + } + + wg.Add(1) + go update(signature.EmailDomainApprovalList, utils.DomainApprovalCriteria) + + wg.Add(1) + go update(signature.EmailApprovalList, utils.EmailApprovalCriteria) + + wg.Add(1) + go update(signature.GitHubOrgApprovalList, utils.GithubOrgApprovalCriteria) + + wg.Add(1) + go update(signature.GitHubUsernameApprovalList, utils.GithubUsernameApprovalCriteria) + + wg.Add(1) + go update(signature.GitlabOrgApprovalList, utils.GitlabOrgApprovalCriteria) + + wg.Add(1) + go update(signature.GitlabUsernameApprovalList, utils.GitlabUsernameApprovalCriteria) + + wg.Wait() + + return err +} + +func getLatestEvent(events []*v1Models.Event) *v1Models.Event { + var latest *v1Models.Event + var latestTime time.Time + + for _, item := range events { + t, err := utils.ParseDateTime(item.EventTime) + if err != nil { + log.Debugf("Error parsing time: %+v ", err) + continue + } + + if latest == nil || t.After(latestTime) { + latest = item + latestTime = t + } + } + + return latest +} diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index eda3f46d0..1e67f9680 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -6,6 +6,7 @@ package cmd import ( "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -71,6 +72,7 @@ import ( "github.com/communitybridge/easycla/cla-backend-go/events" "github.com/communitybridge/easycla/cla-backend-go/project" + "github.com/communitybridge/easycla/cla-backend-go/v2/approvals" v2Project "github.com/communitybridge/easycla/cla-backend-go/v2/project" "github.com/communitybridge/easycla/cla-backend-go/users" @@ -261,6 +263,7 @@ func server(localMode bool) http.Handler { gitlabOrganizationRepo := gitlab_organizations.NewRepository(awsSession, stage) claManagerReqRepo := cla_manager.NewRepository(awsSession, stage) storeRepository := store.NewRepository(awsSession, stage) + approvalsRepo := approvals.NewRepository(stage, awsSession, fmt.Sprintf("cla-%s-approvals", stage)) // Our service layer handlers eventsService := events.NewService(eventsRepo, combinedRepo{ @@ -279,7 +282,7 @@ func server(localMode bool) http.Handler { }) // Signature repository handler - signaturesRepo := signatures.NewRepository(awsSession, stage, v1CompanyRepo, usersRepo, eventsService, gitV1Repository, githubOrganizationsRepo, gerritService) + signaturesRepo := signatures.NewRepository(awsSession, stage, v1CompanyRepo, usersRepo, eventsService, gitV1Repository, githubOrganizationsRepo, gerritService, approvalsRepo) // Initialize the external platform services - these are external APIs that // we download the swagger specification, generate the models, and have @@ -305,7 +308,7 @@ func server(localMode bool) http.Handler { githubOrganizationsService := github_organizations.NewService(githubOrganizationsRepo, gitV1Repository, v1ProjectClaGroupRepo) gitlabOrganizationsService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoriesService, v1ProjectClaGroupRepo, storeRepository, usersService, signaturesRepo, v1CompanyRepo) v1SignaturesService := signatures.NewService(signaturesRepo, v1CompanyService, usersService, eventsService, githubOrgValidation, v1RepositoriesService, githubOrganizationsService, v1ProjectService, gitlabApp, configFile.ClaV1ApiURL, configFile.CLALandingPage, configFile.CLALogoURL) - v2SignatureService := v2Signatures.NewService(awsSession, configFile.SignatureFilesBucket, v1ProjectService, v1CompanyService, v1SignaturesService, v1ProjectClaGroupRepo, signaturesRepo, usersService, eventsService) + v2SignatureService := v2Signatures.NewService(awsSession, configFile.SignatureFilesBucket, v1ProjectService, v1CompanyService, v1SignaturesService, v1ProjectClaGroupRepo, signaturesRepo, usersService, approvalsRepo) v1ClaManagerService := cla_manager.NewService(claManagerReqRepo, v1ProjectClaGroupRepo, v1CompanyService, v1ProjectService, usersService, v1SignaturesService, eventsService, emailTemplateService, configFile.CorporateConsoleV1URL) v2ClaManagerService := v2ClaManager.NewService(emailTemplateService, v1CompanyService, v1ProjectService, v1ClaManagerService, usersService, v1RepositoriesService, v2CompanyService, eventsService, v1ProjectClaGroupRepo) v1ApprovalListService := approval_list.NewService(approvalListRepo, v1ProjectClaGroupRepo, v1ProjectService, usersRepo, v1CompanyRepo, v1CLAGroupRepo, signaturesRepo, emailTemplateService, configFile.CorporateConsoleV2URL, http.DefaultClient) diff --git a/cla-backend-go/events/mockrepo.go b/cla-backend-go/events/mockrepo.go index c51dab806..3a96a29b8 100644 --- a/cla-backend-go/events/mockrepo.go +++ b/cla-backend-go/events/mockrepo.go @@ -20,6 +20,11 @@ import ( // mockRepository data model type mockRepository struct{} +// GetCCLAEvents implements Repository. +func (repo *mockRepository) GetCCLAEvents(claGroupId string, companyID string, searchTerm string, eventType string, pageSize int64) ([]*models.Event, error) { + panic("unimplemented") +} + func (repo *mockRepository) AddDataToEvent(eventID, foundationSFID, projectSFID, projectSFName, companySFID, projectID, claGroupID string) error { panic("implement me") } diff --git a/cla-backend-go/events/repository.go b/cla-backend-go/events/repository.go index 71a34ce55..187f18b8b 100644 --- a/cla-backend-go/events/repository.go +++ b/cla-backend-go/events/repository.go @@ -62,6 +62,7 @@ type Repository interface { CreateEvent(event *models.Event) error AddDataToEvent(eventID, parentProjectSFID, projectSFID, projectSFName, companySFID, projectID, claGroupID string) error SearchEvents(params *eventOps.SearchEventsParams, pageSize int64) (*models.EventList, error) + GetCCLAEvents(claGroupId, companyID, searchTerm, eventType string, pageSize int64) ([]*models.Event, error) GetRecentEvents(pageSize int64) (*models.EventList, error) GetCompanyFoundationEvents(companySFID, companyID, foundationSFID string, nextKey *string, paramPageSize *int64, searchTerm *string, all bool) (*models.EventList, error) @@ -248,6 +249,79 @@ func addTimeExpression(keyCond expression.KeyConditionBuilder, params *eventOps. return keyCond } +// GetEvents +func (repo *repository) GetCCLAEvents(claGroupId, companyID, searchTerm, eventType string, pageSize int64) ([]*models.Event, error) { + f := logrus.Fields{ + "functionName": "v1.events.repository.GetCCLAEvents", + "claGroupId": claGroupId, + "companyID": companyID, + "eventType": eventType, + "pageSize": pageSize, + } + + log.WithFields(f).Debug("querying events table...") + condition := expression.Key("event_cla_group_id").Equal(expression.Value(claGroupId)) + builder := expression.NewBuilder().WithKeyCondition(condition) + + filter := expression.Name("event_company_id").Equal(expression.Value(companyID)). + And(expression.Name("event_type").Equal(expression.Value(eventType))).And(expression.Name("event_data_lower").Contains(strings.ToLower(searchTerm))) + + builder = builder.WithFilter(filter) + + // Use the nice builder to create the expression + expr, err := builder.Build() + if err != nil { + return nil, err + } + + // Assemble the query input parameters + queryInput := &dynamodb.QueryInput{ + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + KeyConditionExpression: expr.KeyCondition(), + ProjectionExpression: expr.Projection(), + FilterExpression: expr.Filter(), + TableName: aws.String(repo.eventsTable), + IndexName: aws.String(EventCLAGroupIDEpochIndex), + Limit: aws.Int64(pageSize), // The maximum number of items to evaluate (not necessarily the number of matching items) + } + + events := make([]*models.Event, 0) + + var results *dynamodb.QueryOutput + + for { + // Perform the query... + var errQuery error + results, errQuery = repo.dynamoDBClient.Query(queryInput) + if errQuery != nil { + log.WithFields(f).WithError(errQuery).Warn("error retrieving events") + return nil, errQuery + } + + // Build the result models + eventsList, modelErr := buildEventListModels(results) + if modelErr != nil { + log.WithFields(f).WithError(modelErr).Warn("error convert event list models") + return nil, modelErr + } + + events = append(events, eventsList...) + log.WithFields(f).Debugf("loaded %d events", len(events)) + + // We have more records if last evaluated key has a value + log.WithFields(f).Debugf("last evaluated key %+v", results.LastEvaluatedKey) + if len(results.LastEvaluatedKey) > 0 { + queryInput.ExclusiveStartKey = results.LastEvaluatedKey + } else { + break + } + } + + return events, nil + +} + // SearchEvents returns list of events matching with filter criteria. func (repo *repository) SearchEvents(params *eventOps.SearchEventsParams, pageSize int64) (*models.EventList, error) { f := logrus.Fields{ diff --git a/cla-backend-go/signatures/repository.go b/cla-backend-go/signatures/repository.go index 91d5b245a..5723432b9 100644 --- a/cla-backend-go/signatures/repository.go +++ b/cla-backend-go/signatures/repository.go @@ -34,6 +34,7 @@ import ( "github.com/communitybridge/easycla/cla-backend-go/github" "github.com/communitybridge/easycla/cla-backend-go/github_organizations" "github.com/communitybridge/easycla/cla-backend-go/repositories" + "github.com/communitybridge/easycla/cla-backend-go/v2/approvals" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" @@ -80,7 +81,8 @@ type SignatureRepository interface { GetIndividualSignature(ctx context.Context, claGroupID, userID string, approved, signed *bool) (*models.Signature, error) GetIndividualSignatures(ctx context.Context, claGroupID, userID string, approved, signed *bool) ([]*models.Signature, error) GetCorporateSignature(ctx context.Context, claGroupID, companyID string, approved, signed *bool) (*models.Signature, error) - GetCorporateSignatures(ctx context.Context, claGroupID, userID string, approved, signed *bool) ([]*models.Signature, error) + GetCorporateSignatures(ctx context.Context, claGroupID, companyID string, approved, signed *bool) ([]*models.Signature, error) + GetCCLASignatures(ctx context.Context, signed, approved *bool) ([]*ItemSignature, error) GetSignatureACL(ctx context.Context, signatureID string) ([]string, error) GetProjectSignatures(ctx context.Context, params signatures.GetProjectSignaturesParams) (*models.Signatures, error) CreateProjectSummaryReport(ctx context.Context, params signatures.CreateProjectSummaryReportParams) (*models.SignatureReport, error) @@ -121,10 +123,11 @@ type repository struct { ghOrgRepo github_organizations.RepositoryInterface gerritService gerrits.Service signatureTableName string + approvalRepo approvals.IRepository } // NewRepository creates a new instance of the signature repository service -func NewRepository(awsSession *session.Session, stage string, companyRepo company.IRepository, usersRepo users.UserRepository, eventsService events.Service, repositoriesRepo repositories.RepositoryInterface, ghOrgRepo github_organizations.RepositoryInterface, gerritService gerrits.Service) SignatureRepository { +func NewRepository(awsSession *session.Session, stage string, companyRepo company.IRepository, usersRepo users.UserRepository, eventsService events.Service, repositoriesRepo repositories.RepositoryInterface, ghOrgRepo github_organizations.RepositoryInterface, gerritService gerrits.Service, approvalRepo approvals.IRepository) SignatureRepository { return repository{ stage: stage, dynamoDBClient: dynamodb.New(awsSession), @@ -135,6 +138,7 @@ func NewRepository(awsSession *session.Session, stage string, companyRepo compan ghOrgRepo: ghOrgRepo, gerritService: gerritService, signatureTableName: fmt.Sprintf("cla-%s-signatures", stage), + approvalRepo: approvalRepo, } } @@ -198,6 +202,60 @@ func (repo repository) SaveOrUpdateSignature(ctx context.Context, signature *Ite return nil } +// GetCCCLASignatures returns a list of CCLA signatures +func (repo repository) GetCCLASignatures(ctx context.Context, signed, approved *bool) ([]*ItemSignature, error) { + f := logrus.Fields{ + "functionName": "v1.signatures.repository.GetCCLASignatures", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "signed": signed, + "approved": approved, + } + + var filter expression.ConditionBuilder + + filter = expression.Name("signature_type").Equal(expression.Value("ccla")) + if signed != nil { + filter = filter.And(expression.Name("signature_signed").Equal(expression.Value(signed))) + } + if approved != nil { + filter = filter.And(expression.Name("signature_approved").Equal(expression.Value(approved))) + } + + // Use the expression builder to build the expression + expr, err := expression.NewBuilder().WithFilter(filter).Build() + + if err != nil { + log.WithFields(f).Warnf("error building expression for CCLA signatures query, error: %v", err) + return nil, err + } + + // Make the DynamoDB Query API call + input := &dynamodb.ScanInput{ + TableName: aws.String(repo.signatureTableName), + FilterExpression: expr.Filter(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + } + + results, err := repo.dynamoDBClient.Scan(input) + if err != nil { + log.WithFields(f).Warnf("error retrieving CCLA signatures, error: %v", err) + return nil, err + } + + // The scan returns a list of matching records - we need to convert these to a list of models + log.WithFields(f).Debugf("retrieved %d CCLA signatures", len(results.Items)) + + var signatures []*ItemSignature + err = dynamodbattribute.UnmarshalListOfMaps(results.Items, &signatures) + if err != nil { + log.WithFields(f).Warnf("error unmarshalling CCLA signatures from database, error: %v", err) + return nil, err + } + + return signatures, nil +} + // UpdateSignature updates an existing signature func (repo repository) UpdateSignature(ctx context.Context, signatureID string, updates map[string]interface{}) error { f := logrus.Fields{ @@ -3037,6 +3095,8 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model return nil, errors.New(msg) } + signatureID := cclaSignature.SignatureID + // Get CLA Manager var cclaManagers []ClaManagerInfoParams for i := range cclaSignature.SignatureACL { @@ -3127,6 +3187,12 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model updateExpression = updateExpression + " #E = :e, " } + log.WithFields(f).Debugf("updating approval list table") + + if params.AddEmailApprovalList != nil { + repo.updateApprovalTable(ctx, params.AddEmailApprovalList, utils.EmailApprovalCriteria, signatureID, projectID, companyID, cclaSignature.SignatureReferenceName) + } + // if email removal update signature approvals if params.RemoveEmailApprovalList != nil { log.WithFields(f).Debugf("removing email: %+v the approval list", params.RemoveDomainApprovalList) @@ -3245,6 +3311,12 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model expressionAttributeValues[":d"] = attrList updateExpression = updateExpression + " #D = :d, " } + + log.WithFields(f).Debugf("updating approval list table") + if params.AddDomainApprovalList != nil { + repo.updateApprovalTable(ctx, params.AddDomainApprovalList, utils.EmailApprovalCriteria, signatureID, projectID, companyID, cclaSignature.SignatureReferenceName) + } + if params.RemoveDomainApprovalList != nil { // Get ICLAs log.WithFields(f).Debug("getting icla records... ") @@ -3304,6 +3376,10 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model expressionAttributeValues[":ghu"] = attrList updateExpression = updateExpression + " #GHU = :ghu, " } + + if params.AddGithubUsernameApprovalList != nil { + repo.updateApprovalTable(ctx, params.AddGithubUsernameApprovalList, utils.GithubUsernameApprovalCriteria, signatureID, projectID, companyID, cclaSignature.SignatureReferenceName) + } if params.RemoveGithubUsernameApprovalList != nil { // if email removal update signature approvals if params.RemoveGithubUsernameApprovalList != nil { @@ -3388,6 +3464,10 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model updateExpression = updateExpression + " #GHO = :gho, " } + if params.AddGithubOrgApprovalList != nil { + repo.updateApprovalTable(ctx, params.AddGithubOrgApprovalList, utils.GithubOrgApprovalCriteria, signatureID, projectID, companyID, cclaSignature.SignatureReferenceName) + } + if params.RemoveGithubOrgApprovalList != nil { approvalList.Criteria = utils.GitHubOrgCriteria approvalList.ApprovalList = params.RemoveGithubOrgApprovalList @@ -3454,6 +3534,9 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model expressionAttributeValues[":glu"] = attrList updateExpression = updateExpression + " #GLU = :glu, " } + if params.AddGitlabUsernameApprovalList != nil { + repo.updateApprovalTable(ctx, params.AddGitlabUsernameApprovalList, utils.GitlabUsernameApprovalCriteria, signatureID, projectID, companyID, cclaSignature.SignatureReferenceName) + } if params.RemoveGitlabUsernameApprovalList != nil { // if email removal update signature approvals if params.RemoveGitlabUsernameApprovalList != nil { @@ -3540,6 +3623,10 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model updateExpression = updateExpression + " #GLO = :glo, " } + if params.AddGitlabOrgApprovalList != nil { + repo.updateApprovalTable(ctx, params.AddGitlabOrgApprovalList, utils.GitlabOrgApprovalCriteria, signatureID, projectID, companyID, cclaSignature.SignatureReferenceName) + } + if params.RemoveGitlabOrgApprovalList != nil { approvalList.Criteria = utils.GitlabOrgCriteria approvalList.ApprovalList = params.RemoveGitlabOrgApprovalList @@ -3628,6 +3715,43 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model return updatedSig, nil } +func (repo *repository) updateApprovalTable(ctx context.Context, approvalList []string, criteria, signatureID, projectID, companyID, companyName string) { + f := logrus.Fields{ + "functionName": "v1.signatures.repository.addApprovalList", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + } + + for _, item := range approvalList { + log.WithFields(f).Debugf("adding approval request for item: %s with criteria: %s", item, criteria) + approvalID, err := uuid.NewV4() + if err != nil { + log.WithFields(f).WithError(err).Warnf("unable to generate UUID for email: %s", item) + continue + } + _, currentTime := utils.CurrentTime() + approvalItem := approvals.ApprovalItem{ + ApprovalID: approvalID.String(), + SignatureID: signatureID, + ApprovalName: item, + ProjectID: projectID, + CompanyID: companyID, + ApprovalCriteria: criteria, + DateCreated: currentTime, + DateModified: currentTime, + ApprovalCompanyName: companyName, + DateAdded: currentTime, + Note: "Auto-Added", + } + + err = repo.approvalRepo.AddApprovalList(approvalItem) + if err != nil { + log.WithFields(f).WithError(err).Warnf("unable to add approval request for item: %s", item) + continue + } + log.WithFields(f).Debugf("added approval request for item: %s with criteria: %s", item, criteria) + } +} + // sendEmail is a helper function used to render email for (CCLA, ICLA, ECLA cases) func (repo repository) sendEmail(ctx context.Context, email string, approvalList *ApprovalList, iclas []*models.IclaSignature, eclas []*models.Signature) { f := logrus.Fields{ diff --git a/cla-backend-go/signatures/service.go b/cla-backend-go/signatures/service.go index e7b2c922f..6602285dc 100644 --- a/cla-backend-go/signatures/service.go +++ b/cla-backend-go/signatures/service.go @@ -46,6 +46,7 @@ type SignatureService interface { GetCorporateSignature(ctx context.Context, claGroupID, companyID string, approved, signed *bool) (*models.Signature, error) GetCorporateSignatures(ctx context.Context, claGroupID, companyID string, approved, signed *bool) ([]*models.Signature, error) GetProjectSignatures(ctx context.Context, params signatures.GetProjectSignaturesParams) (*models.Signatures, error) + GetCCLASignatures(ctx context.Context, signed, approved *bool) ([]*ItemSignature, error) CreateProjectSummaryReport(ctx context.Context, params signatures.CreateProjectSummaryReportParams) (*models.SignatureReport, error) GetProjectCompanySignature(ctx context.Context, companyID, projectID string, approved, signed *bool, nextKey *string, pageSize *int64) (*models.Signature, error) GetProjectCompanySignatures(ctx context.Context, params signatures.GetProjectCompanySignaturesParams) (*models.Signatures, error) @@ -240,6 +241,11 @@ func (s service) GetCompanyIDsWithSignedCorporateSignatures(ctx context.Context, return s.repo.GetCompanyIDsWithSignedCorporateSignatures(ctx, claGroupID) } +// GetCCLASignatures returns the list of CCLA signatures +func (s service) GetCCLASignatures(ctx context.Context, signed, approved *bool) ([]*ItemSignature, error) { + return s.repo.GetCCLASignatures(ctx, signed, approved) +} + // GetUserSignatures returns the list of user signatures associated with the specified user func (s service) GetUserSignatures(ctx context.Context, params signatures.GetUserSignaturesParams, projectID *string) (*models.Signatures, error) { diff --git a/cla-backend-go/utils/constants.go b/cla-backend-go/utils/constants.go index b2ce50800..8c5034ccf 100644 --- a/cla-backend-go/utils/constants.go +++ b/cla-backend-go/utils/constants.go @@ -224,3 +224,15 @@ const GitHubRepositoryType = "GitHub" type contextKey string const XREQUESTIDKey contextKey = "x-request-id" + +const GithubUsernameApprovalCriteria = "githubUsername" + +const GithubOrgApprovalCriteria = "githubOrg" + +const GitlabUsernameApprovalCriteria = "gitlabUsername" + +const GitlabOrgApprovalCriteria = "gitlabOrg" + +const EmailApprovalCriteria = "email" + +const DomainApprovalCriteria = "domain" diff --git a/cla-backend-go/v2/approvals/models.go b/cla-backend-go/v2/approvals/models.go new file mode 100644 index 000000000..03efe2d32 --- /dev/null +++ b/cla-backend-go/v2/approvals/models.go @@ -0,0 +1,19 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package approvals + +type ApprovalItem struct { + ApprovalID string `dynamodbav:"approval_id"` + SignatureID string `dynamodbav:"signature_id"` + DateAdded string `dynamodbav:"date_added"` + DateRemoved string `dynamodbav:"date_removed"` + DateCreated string `dynamodbav:"date_created"` + DateModified string `dynamodbav:"date_modified"` + ApprovalName string `dynamodbav:"approval_name"` + ApprovalCriteria string `dynamodbav:"approval_criteria"` + CompanyID string `dynamodbav:"company_id"` + ProjectID string `dynamodbav:"project_id"` + ApprovalCompanyName string `dynamodbav:"approval_company_name"` + Note string `dynamodbav:"note"` +} diff --git a/cla-backend-go/v2/approvals/repository.go b/cla-backend-go/v2/approvals/repository.go new file mode 100644 index 000000000..4681cc0af --- /dev/null +++ b/cla-backend-go/v2/approvals/repository.go @@ -0,0 +1,243 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package approvals + +import ( + "errors" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/expression" + log "github.com/communitybridge/easycla/cla-backend-go/logging" + "github.com/sirupsen/logrus" +) + +type IRepository interface { + GetApprovalList(approvalID string) (*ApprovalItem, error) + GetApprovalListBySignature(signatureID string) ([]ApprovalItem, error) + AddApprovalList(approvalItem ApprovalItem) error + DeleteApprovalList(approvalID string) error + SearchApprovalList(criteria, approvalListName, claGroupID, companyID, signatureID string) ([]ApprovalItem, error) +} + +type repository struct { + stage string + dynamoDBClient *dynamodb.DynamoDB + tableName string +} + +func NewRepository(stage string, awsSession *session.Session, tableName string) IRepository { + return &repository{ + stage: stage, + dynamoDBClient: dynamodb.New(awsSession), + tableName: tableName, + } +} + +func (repo *repository) GetApprovalList(approvalID string) (*ApprovalItem, error) { + f := logrus.Fields{ + "functionName": "GetApprovalList", + "approvalID": approvalID, + } + + log.WithFields(f).Debugf("repository.GetApprovalList - fetching approval list by approvalID: %s", approvalID) + + result, err := repo.dynamoDBClient.GetItem(&dynamodb.GetItemInput{ + TableName: aws.String(repo.tableName), + Key: map[string]*dynamodb.AttributeValue{ + "approval_id": { + S: aws.String(approvalID), + }, + }, + }) + if err != nil { + log.WithFields(f).Warnf("repository.GetApprovalList - unable to read data from table, error: %+v", err) + return nil, err + } + + if len(result.Item) == 0 { + log.WithFields(f).Warnf("repository.GetApprovalList - no approval list found for approvalID: %s", approvalID) + return nil, errors.New("approval list not found") + } + + approvalItem := ApprovalItem{} + err = dynamodbattribute.UnmarshalMap(result.Item, &approvalItem) + if err != nil { + log.WithFields(f).Warnf("repository.GetApprovalList - unable to unmarshal data from table, error: %+v", err) + return nil, err + } + + return &approvalItem, nil +} + +func (repo *repository) GetApprovalListBySignature(signatureID string) ([]ApprovalItem, error) { + f := logrus.Fields{ + "functionName": "GetApprovalListBySignature", + "signatureID": signatureID, + } + + log.WithFields(f).Debugf("repository.GetApprovalListBySignature - fetching approval list by signatureID: %s", signatureID) + + result, err := repo.dynamoDBClient.Scan(&dynamodb.ScanInput{ + TableName: aws.String(repo.tableName), + FilterExpression: aws.String("signature_id = :signature_id"), + ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ + ":signature_id": { + S: aws.String(signatureID), + }, + }, + }) + if err != nil { + log.WithFields(f).Warnf("repository.GetApprovalListBySignature - unable to read data from table, error: %+v", err) + return nil, err + } + + approvalItems := make([]ApprovalItem, 0) + err = dynamodbattribute.UnmarshalListOfMaps(result.Items, &approvalItems) + if err != nil { + log.WithFields(f).Warnf("repository.GetApprovalListBySignature - unable to unmarshal data from table, error: %+v", err) + return nil, err + } + + return approvalItems, nil +} + +func (repo *repository) AddApprovalList(approvalItem ApprovalItem) error { + f := logrus.Fields{ + "functionName": "v2.approvals.repository.AddApprovalList", + "approvalID": approvalItem.ApprovalID, + "approvalName": approvalItem.ApprovalName, + "tableName": repo.tableName, + } + + log.WithFields(f).Debugf("repository.AddApprovalList - adding approval list: %+v", approvalItem) + + av, err := dynamodbattribute.MarshalMap(approvalItem) + if err != nil { + log.WithFields(f).Warnf("repository.AddApprovalList - unable to marshal data, error: %+v", err) + return err + } + + _, err = repo.dynamoDBClient.PutItem(&dynamodb.PutItemInput{ + TableName: aws.String(repo.tableName), + Item: av, + }) + if err != nil { + log.WithFields(f).Warnf("repository.AddApprovalList - unable to add data to table, error: %+v", err) + return err + } + + return nil +} + +func (repo *repository) DeleteApprovalList(approvalID string) error { + f := logrus.Fields{ + "functionName": "DeleteApprovalList", + "approvalID": approvalID, + } + + log.WithFields(f).Debugf("repository.DeleteApprovalList - deleting approval list by approvalID: %s", approvalID) + + _, err := repo.dynamoDBClient.DeleteItem(&dynamodb.DeleteItemInput{ + TableName: aws.String(repo.tableName), + Key: map[string]*dynamodb.AttributeValue{ + "approval_id": { + S: aws.String(approvalID), + }, + }, + }) + if err != nil { + log.WithFields(f).Warnf("repository.DeleteApprovalList - unable to delete data from table, error: %+v", err) + return err + } + + return nil +} + +func (repo *repository) SearchApprovalList(criteria, approvalListName, claGroupID, companyID, signatureID string) ([]ApprovalItem, error) { + f := logrus.Fields{ + "functionName": "approvals.repository.SearchApprovalList", + "criteria": criteria, + "approvalName": approvalListName, + "claGroupID": claGroupID, + "companyID": companyID, + "signatureID": signatureID, + } + + pageSize := int64(100) + + if signatureID == "" { + return nil, errors.New("signatureID is required") + } + if approvalListName == "" { + return nil, errors.New("approvalListName is required") + } + + condition := expression.Key("signature_id").Equal(expression.Value(signatureID)) + + log.WithFields(f).Debugf("searching for approval list by approvalName: %s", approvalListName) + filter := expression.Name("approval_name").Contains(approvalListName) + + if criteria != "" { + log.WithFields(f).Debugf("searching for criteria: %s", criteria) + filter = filter.And(expression.Name("approval_criteria").Contains(criteria)) + } + + if claGroupID != "" { + log.WithFields(f).Debugf("searching for claGroupID: %s", claGroupID) + filter = filter.And(expression.Name("project_id").Equal(expression.Value(claGroupID))) + } + + if companyID != "" { + log.WithFields(f).Debugf("searching for companyID: %s", companyID) + filter = filter.And(expression.Name("company_id").Equal(expression.Value(companyID))) + } + + expr, err := expression.NewBuilder().WithFilter(filter).WithKeyCondition(condition).Build() + + if err != nil { + log.WithFields(f).Warnf("error building expression, error: %+v", err) + return nil, err + } + + input := &dynamodb.QueryInput{ + TableName: aws.String(repo.tableName), + IndexName: aws.String("signature-id-index"), + KeyConditionExpression: expr.KeyCondition(), + FilterExpression: expr.Filter(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + Limit: aws.Int64(pageSize), + } + + var results []ApprovalItem + + for { + output, err := repo.dynamoDBClient.Query(input) + if err != nil { + log.WithFields(f).Warnf("error retrieving approval list, error: %+v", err) + return nil, err + } + + var items []ApprovalItem + err = dynamodbattribute.UnmarshalListOfMaps(output.Items, &items) + if err != nil { + log.WithFields(f).Warnf("error unmarshalling data, error: %+v", err) + return nil, err + } + + results = append(results, items...) + + if output.LastEvaluatedKey == nil { + break + } + + input.ExclusiveStartKey = output.LastEvaluatedKey + } + + return results, nil + +} diff --git a/cla-backend-go/v2/signatures/converters.go b/cla-backend-go/v2/signatures/converters.go index ab96e2cc6..93dd5c49f 100644 --- a/cla-backend-go/v2/signatures/converters.go +++ b/cla-backend-go/v2/signatures/converters.go @@ -79,38 +79,36 @@ func (s *Service) TransformSignatureToCorporateSignature(signature *models.Signa } var wg sync.WaitGroup - // var errMutex sync.Mutex + var errMutex sync.Mutex var err error transformApprovalList := func(approvalList []string, listType string, destinationList *[]*models.ApprovalItem) { defer wg.Done() for _, item := range approvalList { - // searchTerm := fmt.Sprintf("%s was added to the approval list", item) - - // pageSize := int64(10000) - // eventType := v1Events.ClaApprovalListUpdated - // result, eventErr := s.eventService.SearchEvents(&events.SearchEventsParams{ - // SearchTerm: &searchTerm, - // ProjectSFID: &projectSFID, - // EventType: &eventType, - // PageSize: &pageSize, - // }) - // if eventErr != nil { - // errMutex.Lock() - // err = eventErr - // errMutex.Unlock() - // return - // } + approvals, approvalErr := s.approvalsRepos.SearchApprovalList(listType, item, signature.ProjectID, "", signature.SignatureID) + if approvalErr != nil { + errMutex.Lock() + err = approvalErr + errMutex.Unlock() + return + } + + // Handle scenarios of records with no attached event logs + dateAdded := signature.SignatureModified + + if len(approvals) > 0 { + log.WithFields(f).Debugf("approval found: for %s: %s", listType, item) + // ideally this should be one record + dateAdded = approvals[0].DateAdded + } else { + log.WithFields(f).Debugf("no approval found for %s: %s", listType, item) + } + approvalItem := &models.ApprovalItem{ ApprovalItem: item, - DateAdded: signature.SignatureModified, + DateAdded: dateAdded, } - // if len(result.Events) > 0 { - // event := getLatestEvent(result.Events) - // approvalItem.DateAdded = event.EventTime - // } else { - // log.WithFields(f).Debugf("no events found for %s: %s", listType, item) - // } + log.WithFields(f).Debugf("approvalItem: %+v and list type: %s", approvalItem, listType) *destinationList = append(*destinationList, approvalItem) } @@ -145,26 +143,6 @@ func (s *Service) TransformSignatureToCorporateSignature(signature *models.Signa return err } -// func getLatestEvent(events []*v1Models.Event) *v1Models.Event { -// var latest *v1Models.Event -// var latestTime time.Time - -// for _, item := range events { -// t, err := utils.ParseDateTime(item.EventTime) -// if err != nil { -// log.Debugf("Error parsing time: %+v ", err) -// continue -// } - -// if latest == nil || t.After(latestTime) { -// latest = item -// latestTime = t -// } -// } - -// return latest -// } - func iclaSigCsvHeader() string { return `Name,GitHub Username,GitLab Username,LF_ID,Email,Signed Date,Approved,Signed` } diff --git a/cla-backend-go/v2/signatures/service.go b/cla-backend-go/v2/signatures/service.go index b83da4a33..99d548c19 100644 --- a/cla-backend-go/v2/signatures/service.go +++ b/cla-backend-go/v2/signatures/service.go @@ -31,6 +31,7 @@ import ( "github.com/communitybridge/easycla/cla-backend-go/signatures" "github.com/communitybridge/easycla/cla-backend-go/users" "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/communitybridge/easycla/cla-backend-go/v2/approvals" "github.com/sirupsen/logrus" ) @@ -70,14 +71,14 @@ type Service struct { projectsClaGroupsRepo projects_cla_groups.Repository s3 *s3.S3 signaturesBucket string - eventService events.Service + approvalsRepos approvals.IRepository } // NewService creates instance of v2 signature service func NewService(awsSession *session.Session, signaturesBucketName string, v1ProjectService service.Service, v1CompanyService company.IService, v1SignatureService signatures.SignatureService, - pcgRepo projects_cla_groups.Repository, v1SignatureRepo signatures.SignatureRepository, usersService users.Service, eventService events.Service) *Service { + pcgRepo projects_cla_groups.Repository, v1SignatureRepo signatures.SignatureRepository, usersService users.Service, approvalsRepo approvals.IRepository) *Service { return &Service{ v1ProjectService: v1ProjectService, v1CompanyService: v1CompanyService, @@ -87,7 +88,7 @@ func NewService(awsSession *session.Session, signaturesBucketName string, v1Proj projectsClaGroupsRepo: pcgRepo, s3: s3.New(awsSession), signaturesBucket: signaturesBucketName, - eventService: eventService, + approvalsRepos: approvalsRepo, } }