From 9c4ed92603b31cda2ab1c9f20d09c54ac09aede5 Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Fri, 27 Sep 2024 16:04:28 +0200 Subject: [PATCH 1/3] feat(services): add aggregations --- .../api/graphql/graph/baseResolver/service.go | 2 +- internal/api/graphql/graph/generated.go | 231 ++++++++++++++++++ internal/api/graphql/graph/model/models.go | 17 ++ .../api/graphql/graph/model/models_gen.go | 6 + .../service/withMetadata.graphql | 33 +++ .../api/graphql/graph/schema/service.graphqls | 6 + internal/app/service/service_handler.go | 50 +++- internal/app/service/service_handler_test.go | 92 +++++++ internal/database/interface.go | 2 + internal/database/mariadb/entity.go | 38 +++ internal/database/mariadb/service.go | 115 +++++++-- internal/database/mariadb/service_test.go | 55 +++++ internal/e2e/service_query_test.go | 39 +++ internal/entity/common.go | 1 + internal/entity/service.go | 7 + internal/entity/test/service.go | 18 ++ internal/mocks/mock_Database.go | 116 +++++++++ 17 files changed, 807 insertions(+), 21 deletions(-) create mode 100644 internal/api/graphql/graph/queryCollection/service/withMetadata.graphql diff --git a/internal/api/graphql/graph/baseResolver/service.go b/internal/api/graphql/graph/baseResolver/service.go index 5542352f..8c9ea076 100644 --- a/internal/api/graphql/graph/baseResolver/service.go +++ b/internal/api/graphql/graph/baseResolver/service.go @@ -116,7 +116,7 @@ func ServiceBaseResolver(app app.Heureka, ctx context.Context, filter *model.Ser edges := []*model.ServiceEdge{} for _, result := range services.Elements { - s := model.NewService(result.Service) + s := model.NewServiceWithAggregations(&result) edge := model.ServiceEdge{ Node: &s, Cursor: result.Cursor(), diff --git a/internal/api/graphql/graph/generated.go b/internal/api/graphql/graph/generated.go index 249d6e4f..c79b7e1a 100644 --- a/internal/api/graphql/graph/generated.go +++ b/internal/api/graphql/graph/generated.go @@ -466,6 +466,7 @@ type ComplexityRoot struct { ComponentInstances func(childComplexity int, filter *model.ComponentInstanceFilter, first *int, after *string) int ID func(childComplexity int) int IssueRepositories func(childComplexity int, filter *model.IssueRepositoryFilter, first *int, after *string) int + Metadata func(childComplexity int) int Name func(childComplexity int) int Owners func(childComplexity int, filter *model.UserFilter, first *int, after *string) int SupportGroups func(childComplexity int, filter *model.SupportGroupFilter, first *int, after *string) int @@ -490,6 +491,11 @@ type ComplexityRoot struct { UserName func(childComplexity int, filter *model.UserFilter) int } + ServiceMetadata struct { + ComponentInstanceCount func(childComplexity int) int + IssueMatchCount func(childComplexity int) int + } + Severity struct { Cvss func(childComplexity int) int Score func(childComplexity int) int @@ -3095,6 +3101,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Service.IssueRepositories(childComplexity, args["filter"].(*model.IssueRepositoryFilter), args["first"].(*int), args["after"].(*string)), true + case "Service.metadata": + if e.complexity.Service.Metadata == nil { + break + } + + return e.complexity.Service.Metadata(childComplexity), true + case "Service.name": if e.complexity.Service.Name == nil { break @@ -3216,6 +3229,20 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.ServiceFilterValue.UserName(childComplexity, args["filter"].(*model.UserFilter)), true + case "ServiceMetadata.componentInstanceCount": + if e.complexity.ServiceMetadata.ComponentInstanceCount == nil { + break + } + + return e.complexity.ServiceMetadata.ComponentInstanceCount(childComplexity), true + + case "ServiceMetadata.issueMatchCount": + if e.complexity.ServiceMetadata.IssueMatchCount == nil { + break + } + + return e.complexity.ServiceMetadata.IssueMatchCount(childComplexity), true + case "Severity.cvss": if e.complexity.Severity.Cvss == nil { break @@ -12585,6 +12612,8 @@ func (ec *executionContext) fieldContext_ComponentInstance_service(_ context.Con return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -20241,6 +20270,8 @@ func (ec *executionContext) fieldContext_Mutation_createService(ctx context.Cont return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -20312,6 +20343,8 @@ func (ec *executionContext) fieldContext_Mutation_updateService(ctx context.Cont return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -20438,6 +20471,8 @@ func (ec *executionContext) fieldContext_Mutation_addOwnerToService(ctx context. return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -20509,6 +20544,8 @@ func (ec *executionContext) fieldContext_Mutation_removeOwnerFromService(ctx con return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -20580,6 +20617,8 @@ func (ec *executionContext) fieldContext_Mutation_addIssueRepositoryToService(ct return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -20651,6 +20690,8 @@ func (ec *executionContext) fieldContext_Mutation_removeIssueRepositoryFromServi return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -24540,6 +24581,53 @@ func (ec *executionContext) fieldContext_Service_componentInstances(ctx context. return fc, nil } +func (ec *executionContext) _Service_metadata(ctx context.Context, field graphql.CollectedField, obj *model.Service) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Service_metadata(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Metadata, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*model.ServiceMetadata) + fc.Result = res + return ec.marshalOServiceMetadata2ᚖgithubᚗcomᚋcloudoperatorsᚋheurekaᚋinternalᚋapiᚋgraphqlᚋgraphᚋmodelᚐServiceMetadata(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Service_metadata(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Service", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "issueMatchCount": + return ec.fieldContext_ServiceMetadata_issueMatchCount(ctx, field) + case "componentInstanceCount": + return ec.fieldContext_ServiceMetadata_componentInstanceCount(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ServiceMetadata", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _ServiceConnection_totalCount(ctx context.Context, field graphql.CollectedField, obj *model.ServiceConnection) (ret graphql.Marshaler) { fc, err := ec.fieldContext_ServiceConnection_totalCount(ctx, field) if err != nil { @@ -24741,6 +24829,8 @@ func (ec *executionContext) fieldContext_ServiceEdge_node(_ context.Context, fie return ec.fieldContext_Service_issueRepositories(ctx, field) case "componentInstances": return ec.fieldContext_Service_componentInstances(ctx, field) + case "metadata": + return ec.fieldContext_Service_metadata(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Service", field.Name) }, @@ -25070,6 +25160,94 @@ func (ec *executionContext) fieldContext_ServiceFilterValue_supportGroupName(ctx return fc, nil } +func (ec *executionContext) _ServiceMetadata_issueMatchCount(ctx context.Context, field graphql.CollectedField, obj *model.ServiceMetadata) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ServiceMetadata_issueMatchCount(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.IssueMatchCount, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ServiceMetadata_issueMatchCount(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ServiceMetadata", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _ServiceMetadata_componentInstanceCount(ctx context.Context, field graphql.CollectedField, obj *model.ServiceMetadata) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ServiceMetadata_componentInstanceCount(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.ComponentInstanceCount, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ServiceMetadata_componentInstanceCount(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ServiceMetadata", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _Severity_value(ctx context.Context, field graphql.CollectedField, obj *model.Severity) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Severity_value(ctx, field) if err != nil { @@ -33306,6 +33484,8 @@ func (ec *executionContext) _Service(ctx context.Context, sel ast.SelectionSet, } out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) + case "metadata": + out.Values[i] = ec._Service_metadata(ctx, field, obj) default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -33581,6 +33761,50 @@ func (ec *executionContext) _ServiceFilterValue(ctx context.Context, sel ast.Sel return out } +var serviceMetadataImplementors = []string{"ServiceMetadata"} + +func (ec *executionContext) _ServiceMetadata(ctx context.Context, sel ast.SelectionSet, obj *model.ServiceMetadata) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, serviceMetadataImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("ServiceMetadata") + case "issueMatchCount": + out.Values[i] = ec._ServiceMetadata_issueMatchCount(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "componentInstanceCount": + out.Values[i] = ec._ServiceMetadata_componentInstanceCount(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var severityImplementors = []string{"Severity"} func (ec *executionContext) _Severity(ctx context.Context, sel ast.SelectionSet, obj *model.Severity) graphql.Marshaler { @@ -36210,6 +36434,13 @@ func (ec *executionContext) marshalOServiceFilterValue2ᚖgithubᚗcomᚋcloudop return ec._ServiceFilterValue(ctx, sel, v) } +func (ec *executionContext) marshalOServiceMetadata2ᚖgithubᚗcomᚋcloudoperatorsᚋheurekaᚋinternalᚋapiᚋgraphqlᚋgraphᚋmodelᚐServiceMetadata(ctx context.Context, sel ast.SelectionSet, v *model.ServiceMetadata) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return ec._ServiceMetadata(ctx, sel, v) +} + func (ec *executionContext) marshalOSeverity2ᚖgithubᚗcomᚋcloudoperatorsᚋheurekaᚋinternalᚋapiᚋgraphqlᚋgraphᚋmodelᚐSeverity(ctx context.Context, sel ast.SelectionSet, v *model.Severity) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/internal/api/graphql/graph/model/models.go b/internal/api/graphql/graph/model/models.go index 0952c290..beea612a 100644 --- a/internal/api/graphql/graph/model/models.go +++ b/internal/api/graphql/graph/model/models.go @@ -338,6 +338,23 @@ func NewService(s *entity.Service) Service { } } +func NewServiceWithAggregations(service *entity.ServiceResult) Service { + var metadata ServiceMetadata + + if service.ServiceAggregations != nil { + metadata = ServiceMetadata{ + IssueMatchCount: int(service.ServiceAggregations.IssueMatches), + ComponentInstanceCount: int(service.ServiceAggregations.ComponentInstances), + } + } + + return Service{ + ID: fmt.Sprintf("%d", service.Id), + Name: &service.Name, + Metadata: &metadata, + } +} + func NewServiceEntity(service *ServiceInput) entity.Service { return entity.Service{ BaseService: entity.BaseService{ diff --git a/internal/api/graphql/graph/model/models_gen.go b/internal/api/graphql/graph/model/models_gen.go index 379d808f..9abb392a 100644 --- a/internal/api/graphql/graph/model/models_gen.go +++ b/internal/api/graphql/graph/model/models_gen.go @@ -601,6 +601,7 @@ type Service struct { Activities *ActivityConnection `json:"activities,omitempty"` IssueRepositories *IssueRepositoryConnection `json:"issueRepositories,omitempty"` ComponentInstances *ComponentInstanceConnection `json:"componentInstances,omitempty"` + Metadata *ServiceMetadata `json:"metadata,omitempty"` } func (Service) IsNode() {} @@ -646,6 +647,11 @@ type ServiceInput struct { Name *string `json:"name,omitempty"` } +type ServiceMetadata struct { + IssueMatchCount int `json:"issueMatchCount"` + ComponentInstanceCount int `json:"componentInstanceCount"` +} + type Severity struct { Value *SeverityValues `json:"value,omitempty"` Score *float64 `json:"score,omitempty"` diff --git a/internal/api/graphql/graph/queryCollection/service/withMetadata.graphql b/internal/api/graphql/graph/queryCollection/service/withMetadata.graphql new file mode 100644 index 00000000..8b0c471a --- /dev/null +++ b/internal/api/graphql/graph/queryCollection/service/withMetadata.graphql @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + +query ($filter: ServiceFilter, $first: Int, $after: String) { + Services ( + filter: $filter, + first: $first, + after: $after + ) { + totalCount + edges { + node { + id + name + componentInstances { + edges { + node { + count + issueMatches { + totalCount + } + } + } + } + metadata { + issueMatchCount + componentInstanceCount + } + } + cursor + } + } +} \ No newline at end of file diff --git a/internal/api/graphql/graph/schema/service.graphqls b/internal/api/graphql/graph/schema/service.graphqls index dc202967..7cf7d38f 100644 --- a/internal/api/graphql/graph/schema/service.graphqls +++ b/internal/api/graphql/graph/schema/service.graphqls @@ -9,6 +9,12 @@ type Service implements Node { activities(filter: ActivityFilter, first: Int, after: String): ActivityConnection issueRepositories(filter: IssueRepositoryFilter, first: Int, after: String): IssueRepositoryConnection componentInstances(filter: ComponentInstanceFilter, first: Int, after: String): ComponentInstanceConnection + metadata: ServiceMetadata +} + +type ServiceMetadata { + issueMatchCount: Int! + componentInstanceCount: Int! } input ServiceInput { diff --git a/internal/app/service/service_handler.go b/internal/app/service/service_handler.go index 415b883a..b7c7728c 100644 --- a/internal/app/service/service_handler.go +++ b/internal/app/service/service_handler.go @@ -9,6 +9,7 @@ import ( "github.com/cloudoperators/heureka/internal/app/common" "github.com/cloudoperators/heureka/internal/app/event" "github.com/cloudoperators/heureka/internal/database" + "github.com/cloudoperators/heureka/pkg/util" "github.com/cloudoperators/heureka/internal/entity" "github.com/sirupsen/logrus" @@ -38,6 +39,36 @@ func NewServiceHandlerError(msg string) *ServiceHandlerError { return &ServiceHandlerError{msg: msg} } +func (s *serviceHandler) getServiceResultsWithAggregations(filter *entity.ServiceFilter) ([]entity.ServiceResult, error) { + var serviceResults []entity.ServiceResult + servicesCiCount, err := s.database.GetServicesWithComponentInstanceCount(filter) + if err != nil { + return nil, err + } + + servicesImCount, err := s.database.GetServicesWithIssueMatchCount(filter) + if err != nil { + return nil, err + } + + for i := 0; i < len(servicesCiCount); i++ { + serviceCi := servicesCiCount[i] + serviceIm := servicesImCount[i] + cursor := fmt.Sprintf("%d", serviceCi.Id) + serviceResults = append(serviceResults, entity.ServiceResult{ + WithCursor: entity.WithCursor{Value: cursor}, + ServiceAggregations: &entity.ServiceAggregations{ + IssueMatches: serviceIm.IssueMatches, + ComponentInstances: serviceCi.ComponentInstances, + }, + Service: util.Ptr(serviceCi.Service), + }) + + } + + return serviceResults, nil +} + func (s *serviceHandler) getServiceResults(filter *entity.ServiceFilter) ([]entity.ServiceResult, error) { var serviceResults []entity.ServiceResult services, err := s.database.GetServices(filter) @@ -81,6 +112,8 @@ func (s *serviceHandler) GetService(serviceId int64) (*entity.Service, error) { func (s *serviceHandler) ListServices(filter *entity.ServiceFilter, options *entity.ListOptions) (*entity.List[entity.ServiceResult], error) { var count int64 var pageInfo *entity.PageInfo + var res []entity.ServiceResult + var err error common.EnsurePaginated(&filter.Paginated) @@ -89,11 +122,18 @@ func (s *serviceHandler) ListServices(filter *entity.ServiceFilter, options *ent "filter": filter, }) - res, err := s.getServiceResults(filter) - - if err != nil { - l.Error(err) - return nil, NewServiceHandlerError("Error while filtering for Services") + if options.IncludeAggregations { + res, err = s.getServiceResultsWithAggregations(filter) + if err != nil { + l.Error(err) + return nil, NewServiceHandlerError("Internal error while retrieving list results with aggregations") + } + } else { + res, err = s.getServiceResults(filter) + if err != nil { + l.Error(err) + return nil, NewServiceHandlerError("Internal error while retrieving list results.") + } } if options.ShowPageInfo { diff --git a/internal/app/service/service_handler_test.go b/internal/app/service/service_handler_test.go index d4491a86..0a7a84ab 100644 --- a/internal/app/service/service_handler_test.go +++ b/internal/app/service/service_handler_test.go @@ -4,10 +4,12 @@ package service_test import ( + "errors" "math" "testing" "github.com/cloudoperators/heureka/internal/app/event" + "github.com/cloudoperators/heureka/internal/app/service" s "github.com/cloudoperators/heureka/internal/app/service" "github.com/cloudoperators/heureka/internal/entity" @@ -109,6 +111,96 @@ var _ = Describe("When listing Services", Label("app", "ListServices"), func() { Entry("When pageSize is 10 and the database was returning 11 elements", 10, 11, 10, true), ) }) + When("the list options does include aggregations", func() { + BeforeEach(func() { + options.IncludeAggregations = true + }) + Context("and the given filter does not have any matches in the database", func() { + + BeforeEach(func() { + db.On("GetServicesWithComponentInstanceCount", filter).Return([]entity.ServiceWithAggregations{}, nil) + db.On("GetServicesWithIssueMatchCount", filter).Return([]entity.ServiceWithAggregations{}, nil) + }) + + It("should return an empty result", func() { + serviceHandler = service.NewServiceHandler(db, er) + res, err := serviceHandler.ListServices(filter, options) + Expect(err).To(BeNil(), "no error should be thrown") + Expect(len(res.Elements)).Should(BeEquivalentTo(0), "return no results") + + }) + }) + Context("and the filter does have results in the database", func() { + BeforeEach(func() { + services := test.NNewFakeServiceEntitiesWithAggregations(10) + db.On("GetServicesWithComponentInstanceCount", filter).Return(services, nil) + db.On("GetServicesWithIssueMatchCount", filter).Return(services, nil) + }) + It("should return the expected services in the result", func() { + serviceHandler = service.NewServiceHandler(db, er) + res, err := serviceHandler.ListServices(filter, options) + Expect(err).To(BeNil(), "no error should be thrown") + Expect(len(res.Elements)).Should(BeEquivalentTo(10), "return 10 results") + }) + }) + Context("and the database operations throw an error", func() { + BeforeEach(func() { + db.On("GetServicesWithComponentInstanceCount", filter).Return([]entity.ServiceWithAggregations{}, errors.New("some error")) + }) + + It("should return the expected services in the result", func() { + serviceHandler = service.NewServiceHandler(db, er) + _, err := serviceHandler.ListServices(filter, options) + Expect(err).Error() + Expect(err.Error()).ToNot(BeEquivalentTo("some error"), "error gets not passed through") + }) + }) + }) + When("the list options does NOT include aggregations", func() { + + BeforeEach(func() { + options.IncludeAggregations = false + }) + + Context("and the given filter does not have any matches in the database", func() { + + BeforeEach(func() { + db.On("GetServices", filter).Return([]entity.Service{}, nil) + }) + It("should return an empty result", func() { + + serviceHandler = service.NewServiceHandler(db, er) + res, err := serviceHandler.ListServices(filter, options) + Expect(err).To(BeNil(), "no error should be thrown") + Expect(len(res.Elements)).Should(BeEquivalentTo(0), "return no results") + + }) + }) + Context("and the filter does have results in the database", func() { + BeforeEach(func() { + db.On("GetServices", filter).Return(test.NNewFakeServiceEntities(15), nil) + }) + It("should return the expected services in the result", func() { + serviceHandler = service.NewServiceHandler(db, er) + res, err := serviceHandler.ListServices(filter, options) + Expect(err).To(BeNil(), "no error should be thrown") + Expect(len(res.Elements)).Should(BeEquivalentTo(15), "return 15 results") + }) + }) + + Context("and the database operations throw an error", func() { + BeforeEach(func() { + db.On("GetServices", filter).Return([]entity.Service{}, errors.New("some error")) + }) + + It("should return the expected services in the result", func() { + serviceHandler = service.NewServiceHandler(db, er) + _, err := serviceHandler.ListServices(filter, options) + Expect(err).Error() + Expect(err.Error()).ToNot(BeEquivalentTo("some error"), "error gets not passed through") + }) + }) + }) }) var _ = Describe("When creating Service", Label("app", "CreateService"), func() { diff --git a/internal/database/interface.go b/internal/database/interface.go index 34c2f297..a0243dd7 100644 --- a/internal/database/interface.go +++ b/internal/database/interface.go @@ -51,6 +51,8 @@ type Database interface { RemoveEvidenceFromIssueMatch(int64, int64) error GetServices(*entity.ServiceFilter) ([]entity.Service, error) + GetServicesWithComponentInstanceCount(*entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) + GetServicesWithIssueMatchCount(*entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) GetAllServiceIds(*entity.ServiceFilter) ([]int64, error) CountServices(*entity.ServiceFilter) (int64, error) CreateService(*entity.Service) (*entity.Service, error) diff --git a/internal/database/mariadb/entity.go b/internal/database/mariadb/entity.go index 5a584e6b..d4481395 100644 --- a/internal/database/mariadb/entity.go +++ b/internal/database/mariadb/entity.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cloudoperators/heureka/internal/entity" + "github.com/samber/lo" ) func GetInt64Value(v sql.NullInt64) int64 { @@ -65,6 +66,8 @@ type DatabaseRow interface { ComponentVersionRow | BaseServiceRow | ServiceRow | + GetServicesByRow | + ServiceAggregationsRow | ActivityRow | UserRow | EvidenceRow | @@ -505,6 +508,41 @@ func (sr *ServiceRow) FromService(s *entity.Service) { sr.BaseServiceRow.UpdatedAt = sql.NullTime{Time: s.BaseService.UpdatedAt, Valid: true} } +type GetServicesByRow struct { + ServiceAggregationsRow + ServiceRow +} + +type ServiceAggregationsRow struct { + ComponentInstances sql.NullInt64 `db:"agg_component_instances"` + IssueMatches sql.NullInt64 `db:"agg_issue_matches"` +} + +func (sbr *GetServicesByRow) AsServiceWithAggregations() entity.ServiceWithAggregations { + return entity.ServiceWithAggregations{ + ServiceAggregations: entity.ServiceAggregations{ + ComponentInstances: lo.Max([]int64{0, GetInt64Value(sbr.ServiceAggregationsRow.ComponentInstances)}), + IssueMatches: lo.Max([]int64{0, GetInt64Value(sbr.ServiceAggregationsRow.IssueMatches)}), + }, + Service: entity.Service{ + BaseService: entity.BaseService{ + Id: GetInt64Value(sbr.BaseServiceRow.Id), + Name: GetStringValue(sbr.BaseServiceRow.Name), + Owners: []entity.User{}, + Activities: []entity.Activity{}, + CreatedAt: GetTimeValue(sbr.BaseServiceRow.CreatedAt), + DeletedAt: GetTimeValue(sbr.BaseServiceRow.DeletedAt), + UpdatedAt: GetTimeValue(sbr.BaseServiceRow.UpdatedAt), + }, + IssueRepositoryService: entity.IssueRepositoryService{ + ServiceId: GetInt64Value(sbr.IssueRepositoryServiceRow.ServiceId), + IssueRepositoryId: GetInt64Value(sbr.IssueRepositoryServiceRow.IssueRepositoryId), + Priority: GetInt64Value(sbr.IssueRepositoryServiceRow.Priority), + }, + }, + } +} + type ActivityRow struct { Id sql.NullInt64 `db:"activity_id" json:"id"` Status sql.NullString `db:"activity_status" json:"status"` diff --git a/internal/database/mariadb/service.go b/internal/database/mariadb/service.go index 65e15cb4..045892eb 100644 --- a/internal/database/mariadb/service.go +++ b/internal/database/mariadb/service.go @@ -16,6 +16,31 @@ const ( serviceWildCardFilterQuery = "S.service_name LIKE Concat('%',?,'%')" ) +type ServiceAggregationOptions struct { + IssueMatchCount bool + ComponentInstanceCount bool +} + +func (sa *ServiceAggregationOptions) GetIssueMatchCountAggregation() string { + return ", COUNT(distinct IM.issuematch_id) as agg_issue_matches" +} + +func (sa *ServiceAggregationOptions) GetComponentInstanceCountAggregation() string { + return ", SUM(CI.componentinstance_count) as agg_component_instances" +} + +func (sa *ServiceAggregationOptions) GetAggregationQuery() string { + if sa.ComponentInstanceCount { + return sa.GetComponentInstanceCountAggregation() + } + + if sa.IssueMatchCount { + return sa.GetIssueMatchCountAggregation() + } + + return "" +} + func (s *SqlDatabase) getServiceFilterString(filter *entity.ServiceFilter) string { var fl []string fl = append(fl, buildFilterQuery(filter.Name, "S.service_name = ?", OP_OR)) @@ -33,7 +58,7 @@ func (s *SqlDatabase) getServiceFilterString(filter *entity.ServiceFilter) strin return combineFilterQueries(fl, OP_AND) } -func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter) string { +func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter, aggOps ServiceAggregationOptions) string { joins := "" if len(filter.OwnerName) > 0 || len(filter.OwnerId) > 0 { joins = fmt.Sprintf("%s\n%s", joins, ` @@ -61,7 +86,7 @@ func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter) string { LEFT JOIN Activity A on AHS.activityhasservice_activity_id = A.activity_id `) } - if len(filter.ComponentInstanceId) > 0 { + if len(filter.ComponentInstanceId) > 0 || aggOps.ComponentInstanceCount || aggOps.IssueMatchCount { joins = fmt.Sprintf("%s\n%s", joins, ` LEFT JOIN ComponentInstance CI on S.service_id = CI.componentinstance_service_id `) @@ -71,6 +96,11 @@ func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter) string { LEFT JOIN IssueRepositoryService IRS on IRS.issuerepositoryservice_service_id = S.service_id `) } + if aggOps.IssueMatchCount { + joins = fmt.Sprintf("%s\n%s", joins, ` + LEFT JOIN IssueMatch IM on IM.issuematch_component_instance_id = CI.componentinstance_id + `) + } return joins } @@ -118,13 +148,13 @@ func (s *SqlDatabase) getServiceUpdateFields(service *entity.Service) string { return strings.Join(fl, ", ") } -func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.ServiceFilter, withCursor bool, l *logrus.Entry) (*sqlx.Stmt, []interface{}, error) { +func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.ServiceFilter, aggOpts ServiceAggregationOptions, withCursor bool, l *logrus.Entry) (*sqlx.Stmt, []interface{}, error) { var query string filter = s.ensureServiceFilter(filter) l.WithFields(logrus.Fields{"filter": filter}) filterStr := s.getServiceFilterString(filter) - joins := s.getServiceJoins(filter) + joins := s.getServiceJoins(filter, aggOpts) cursor := getCursor(filter.Paginated, filterStr, "S.service_id > ?") whereClause := "" @@ -132,11 +162,13 @@ func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.Ser whereClause = fmt.Sprintf("WHERE %s", filterStr) } + ags := aggOpts.GetAggregationQuery() + // construct final query if withCursor { - query = fmt.Sprintf(baseQuery, joins, whereClause, cursor.Statement) + query = fmt.Sprintf(baseQuery, ags, joins, whereClause, cursor.Statement) } else { - query = fmt.Sprintf(baseQuery, joins, whereClause) + query = fmt.Sprintf(baseQuery, ags, joins, whereClause) } //construct prepared statement and if where clause does exist add parameters @@ -181,11 +213,11 @@ func (s *SqlDatabase) CountServices(filter *entity.ServiceFilter) (int64, error) }) baseQuery := ` - SELECT count(distinct S.service_id) FROM Service S + SELECT count(distinct S.service_id) %s FROM Service S %s %s ` - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, false, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, false, l) if err != nil { return -1, err @@ -202,12 +234,12 @@ func (s *SqlDatabase) GetAllServiceIds(filter *entity.ServiceFilter) ([]int64, e }) baseQuery := ` - SELECT S.service_id FROM Service S + SELECT S.service_id %s FROM Service S %s %s GROUP BY S.service_id ORDER BY S.service_id ` - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, false, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, false, l) if err != nil { return nil, err @@ -224,7 +256,7 @@ func (s *SqlDatabase) GetServices(filter *entity.ServiceFilter) ([]entity.Servic }) baseQuery := ` - SELECT %s FROM Service S + SELECT %s %s FROM Service S %s %s %s GROUP BY S.service_id ORDER BY S.service_id LIMIT ? @@ -232,9 +264,9 @@ func (s *SqlDatabase) GetServices(filter *entity.ServiceFilter) ([]entity.Servic filter = s.ensureServiceFilter(filter) columns := s.getServiceColumns(filter) - baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s") + baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s", "%s") - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, true, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, true, l) if err != nil { return nil, err @@ -252,6 +284,59 @@ func (s *SqlDatabase) GetServices(filter *entity.ServiceFilter) ([]entity.Servic ) } +func (s *SqlDatabase) GetServicesWithComponentInstanceCount(filter *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { + aggOpts := ServiceAggregationOptions{ + ComponentInstanceCount: true, + } + return s.getServicesWithAggregation(filter, aggOpts) +} + +func (s *SqlDatabase) GetServicesWithIssueMatchCount(filter *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { + aggOpts := ServiceAggregationOptions{ + IssueMatchCount: true, + } + return s.getServicesWithAggregation(filter, aggOpts) +} + +func (s *SqlDatabase) getServicesWithAggregation(filter *entity.ServiceFilter, aggOpts ServiceAggregationOptions) ([]entity.ServiceWithAggregations, error) { + filter = s.ensureServiceFilter(filter) + l := logrus.WithFields(logrus.Fields{ + "filter": filter, + "event": "database.GetServicesWithAggregations", + }) + + baseQuery := ` + SELECT %s %s FROM Service S + %s + %s + %s GROUP BY S.service_id ORDER BY S.service_id LIMIT ? + ` + columns := s.getServiceColumns(filter) + baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s", "%s") + + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, aggOpts, true, l) + + if err != nil { + msg := ERROR_MSG_PREPARED_STMT + l.WithFields( + logrus.Fields{ + "error": err, + "aggregationOptionss": aggOpts, + }).Error(msg) + return nil, fmt.Errorf("%s", msg) + } + defer stmt.Close() + + return performListScan( + stmt, + filterParameters, + l, + func(l []entity.ServiceWithAggregations, e GetServicesByRow) []entity.ServiceWithAggregations { + return append(l, e.AsServiceWithAggregations()) + }, + ) +} + func (s *SqlDatabase) CreateService(service *entity.Service) (*entity.Service, error) { l := logrus.WithFields(logrus.Fields{ "service": service, @@ -435,7 +520,7 @@ func (s *SqlDatabase) GetServiceNames(filter *entity.ServiceFilter) ([]string, e }) baseQuery := ` - SELECT service_name FROM Service S + SELECT service_name %s FROM Service S %s %s ` @@ -444,7 +529,7 @@ func (s *SqlDatabase) GetServiceNames(filter *entity.ServiceFilter) ([]string, e filter = s.ensureServiceFilter(filter) // Builds full statement with possible joins and filters - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, false, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, false, l) if err != nil { l.Error("Error preparing statement: ", err) return nil, err diff --git a/internal/database/mariadb/service_test.go b/internal/database/mariadb/service_test.go index 3dee040b..587a7e66 100644 --- a/internal/database/mariadb/service_test.go +++ b/internal/database/mariadb/service_test.go @@ -436,6 +436,61 @@ var _ = Describe("Service", Label("database", "Service"), func() { }) }) }) + When("Getting Services with Aggregations", Label("GetServicesWithAggregations"), func() { + BeforeEach(func() { + _ = seeder.SeedDbWithNFakeData(10) + }) + Context("and and we have 10 elements in the database", func() { + It("returns the services with componentInstance count", func() { + entriesWithAggregations, err := db.GetServicesWithComponentInstanceCount(nil) + + By("throwing no error", func() { + Expect(err).To(BeNil()) + }) + + By("returning some aggregations", func() { + for _, entryWithAggregations := range entriesWithAggregations { + Expect(entryWithAggregations).NotTo( + BeEquivalentTo(entity.ServiceAggregations{})) + } + }) + }) + It("returns correct aggregation values", func() { + //Should be filled with a check for each aggregation value, + // this is currently skipped due to the complexity of the test implementation + // as we would need to implement for each of the aggregations a manual aggregation + // based on the seederCollection. + // + // This tests should therefore only get implemented in case we encourage errors in this area to test against + // possible regressions + }) + }) + Context("and and we have 10 elements in the database", func() { + It("returns the services with issueMatch count", func() { + entriesWithAggregations, err := db.GetServicesWithIssueMatchCount(nil) + + By("throwing no error", func() { + Expect(err).To(BeNil()) + }) + + By("returning some aggregations", func() { + for _, entryWithAggregations := range entriesWithAggregations { + Expect(entryWithAggregations).NotTo( + BeEquivalentTo(entity.ServiceAggregations{})) + } + }) + }) + It("returns correct aggregation values", func() { + //Should be filled with a check for each aggregation value, + // this is currently skipped due to the complexity of the test implementation + // as we would need to implement for each of the aggregations a manual aggregation + // based on the seederCollection. + // + // This tests should therefore only get implemented in case we encourage errors in this area to test against + // possible regressions + }) + }) + }) When("Counting Services", Label("CountServices"), func() { Context("and the database is empty", func() { It("can count correctly", func() { diff --git a/internal/e2e/service_query_test.go b/internal/e2e/service_query_test.go index 4e83159c..29d3bcaa 100644 --- a/internal/e2e/service_query_test.go +++ b/internal/e2e/service_query_test.go @@ -113,6 +113,45 @@ var _ = Describe("Getting Services via API", Label("e2e", "Services"), func() { Expect(len(respData.Services.Edges)).To(Equal(5)) }) + }) + Context("and we request metadata", func() { + It("returns correct metadata counts", func() { + // create a queryCollection (safe to share across requests) + client := graphql.NewClient(fmt.Sprintf("http://localhost:%s/query", cfg.Port)) + + //@todo may need to make this more fault proof?! What if the test is executed from the root dir? does it still work? + b, err := os.ReadFile("../api/graphql/graph/queryCollection/service/withMetadata.graphql") + + Expect(err).To(BeNil()) + str := string(b) + req := graphql.NewRequest(str) + + req.Var("filter", map[string]string{}) + req.Var("first", 5) + req.Var("after", "0") + + req.Header.Set("Cache-Control", "no-cache") + ctx := context.Background() + + var respData struct { + Services model.ServiceConnection `json:"Services"` + } + if err := util2.RequestWithBackoff(func() error { return client.Run(ctx, req, &respData) }); err != nil { + logrus.WithError(err).WithField("request", req).Fatalln("Error while unmarshaling") + } + + for _, serviceEdge := range respData.Services.Edges { + imCount := 0 + ciCount := 0 + for _, ciEdge := range serviceEdge.Node.ComponentInstances.Edges { + imCount += ciEdge.Node.IssueMatches.TotalCount + ciCount += *ciEdge.Node.Count + } + Expect(serviceEdge.Node.Metadata.IssueMatchCount).To(Equal(imCount)) + Expect(serviceEdge.Node.Metadata.ComponentInstanceCount).To(Equal(ciCount)) + } + }) + }) Context("and we query to resolve levels of relations", Label("directRelations.graphql"), func() { diff --git a/internal/entity/common.go b/internal/entity/common.go index 92f9aa27..d2a6cb1a 100644 --- a/internal/entity/common.go +++ b/internal/entity/common.go @@ -35,6 +35,7 @@ type HeurekaEntity interface { BaseService | Service | ServiceAggregations | + ServiceWithAggregations | SupportGroup | SupportGroupService | SupportGroupUser | diff --git a/internal/entity/service.go b/internal/entity/service.go index cb7c7e67..80785cdc 100644 --- a/internal/entity/service.go +++ b/internal/entity/service.go @@ -19,6 +19,13 @@ type BaseService struct { } type ServiceAggregations struct { + ComponentInstances int64 + IssueMatches int64 +} + +type ServiceWithAggregations struct { + Service + ServiceAggregations } type ServiceFilter struct { diff --git a/internal/entity/test/service.go b/internal/entity/test/service.go index fb695bcc..9381a189 100644 --- a/internal/entity/test/service.go +++ b/internal/entity/test/service.go @@ -23,6 +23,24 @@ func NewFakeServiceEntity() entity.Service { } } +func NewFakeServiceWithAggregationsEntity() entity.ServiceWithAggregations { + return entity.ServiceWithAggregations{ + ServiceAggregations: entity.ServiceAggregations{ + IssueMatches: int64(gofakeit.Number(1, 10000000)), + ComponentInstances: int64(gofakeit.Number(1, 10000000)), + }, + Service: NewFakeServiceEntity(), + } +} + +func NNewFakeServiceEntitiesWithAggregations(n int) []entity.ServiceWithAggregations { + r := make([]entity.ServiceWithAggregations, n) + for i := 0; i < n; i++ { + r[i] = NewFakeServiceWithAggregationsEntity() + } + return r +} + func NNewFakeServiceEntities(n int) []entity.Service { r := make([]entity.Service, n) for i := 0; i < n; i++ { diff --git a/internal/mocks/mock_Database.go b/internal/mocks/mock_Database.go index b3549657..7609db5f 100644 --- a/internal/mocks/mock_Database.go +++ b/internal/mocks/mock_Database.go @@ -4297,6 +4297,122 @@ func (_c *MockDatabase_GetServices_Call) RunAndReturn(run func(*entity.ServiceFi return _c } +// GetServicesWithComponentInstanceCount provides a mock function with given fields: _a0 +func (_m *MockDatabase) GetServicesWithComponentInstanceCount(_a0 *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for GetServicesWithComponentInstanceCount") + } + + var r0 []entity.ServiceWithAggregations + var r1 error + if rf, ok := ret.Get(0).(func(*entity.ServiceFilter) ([]entity.ServiceWithAggregations, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*entity.ServiceFilter) []entity.ServiceWithAggregations); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]entity.ServiceWithAggregations) + } + } + + if rf, ok := ret.Get(1).(func(*entity.ServiceFilter) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDatabase_GetServicesWithComponentInstanceCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetServicesWithComponentInstanceCount' +type MockDatabase_GetServicesWithComponentInstanceCount_Call struct { + *mock.Call +} + +// GetServicesWithComponentInstanceCount is a helper method to define mock.On call +// - _a0 *entity.ServiceFilter +func (_e *MockDatabase_Expecter) GetServicesWithComponentInstanceCount(_a0 interface{}) *MockDatabase_GetServicesWithComponentInstanceCount_Call { + return &MockDatabase_GetServicesWithComponentInstanceCount_Call{Call: _e.mock.On("GetServicesWithComponentInstanceCount", _a0)} +} + +func (_c *MockDatabase_GetServicesWithComponentInstanceCount_Call) Run(run func(_a0 *entity.ServiceFilter)) *MockDatabase_GetServicesWithComponentInstanceCount_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*entity.ServiceFilter)) + }) + return _c +} + +func (_c *MockDatabase_GetServicesWithComponentInstanceCount_Call) Return(_a0 []entity.ServiceWithAggregations, _a1 error) *MockDatabase_GetServicesWithComponentInstanceCount_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDatabase_GetServicesWithComponentInstanceCount_Call) RunAndReturn(run func(*entity.ServiceFilter) ([]entity.ServiceWithAggregations, error)) *MockDatabase_GetServicesWithComponentInstanceCount_Call { + _c.Call.Return(run) + return _c +} + +// GetServicesWithIssueMatchCount provides a mock function with given fields: _a0 +func (_m *MockDatabase) GetServicesWithIssueMatchCount(_a0 *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for GetServicesWithIssueMatchCount") + } + + var r0 []entity.ServiceWithAggregations + var r1 error + if rf, ok := ret.Get(0).(func(*entity.ServiceFilter) ([]entity.ServiceWithAggregations, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*entity.ServiceFilter) []entity.ServiceWithAggregations); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]entity.ServiceWithAggregations) + } + } + + if rf, ok := ret.Get(1).(func(*entity.ServiceFilter) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDatabase_GetServicesWithIssueMatchCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetServicesWithIssueMatchCount' +type MockDatabase_GetServicesWithIssueMatchCount_Call struct { + *mock.Call +} + +// GetServicesWithIssueMatchCount is a helper method to define mock.On call +// - _a0 *entity.ServiceFilter +func (_e *MockDatabase_Expecter) GetServicesWithIssueMatchCount(_a0 interface{}) *MockDatabase_GetServicesWithIssueMatchCount_Call { + return &MockDatabase_GetServicesWithIssueMatchCount_Call{Call: _e.mock.On("GetServicesWithIssueMatchCount", _a0)} +} + +func (_c *MockDatabase_GetServicesWithIssueMatchCount_Call) Run(run func(_a0 *entity.ServiceFilter)) *MockDatabase_GetServicesWithIssueMatchCount_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*entity.ServiceFilter)) + }) + return _c +} + +func (_c *MockDatabase_GetServicesWithIssueMatchCount_Call) Return(_a0 []entity.ServiceWithAggregations, _a1 error) *MockDatabase_GetServicesWithIssueMatchCount_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDatabase_GetServicesWithIssueMatchCount_Call) RunAndReturn(run func(*entity.ServiceFilter) ([]entity.ServiceWithAggregations, error)) *MockDatabase_GetServicesWithIssueMatchCount_Call { + _c.Call.Return(run) + return _c +} + // GetSupportGroupNames provides a mock function with given fields: _a0 func (_m *MockDatabase) GetSupportGroupNames(_a0 *entity.SupportGroupFilter) ([]string, error) { ret := _m.Called(_a0) From 9efb6e056a1470331bd811efa3598490d6d52a33 Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Wed, 2 Oct 2024 15:35:48 +0200 Subject: [PATCH 2/3] refactor --- internal/app/service/service_handler.go | 45 ++++++-- internal/database/mariadb/service.go | 147 +++++++++++------------- 2 files changed, 102 insertions(+), 90 deletions(-) diff --git a/internal/app/service/service_handler.go b/internal/app/service/service_handler.go index b7c7728c..c990e1e1 100644 --- a/internal/app/service/service_handler.go +++ b/internal/app/service/service_handler.go @@ -5,11 +5,14 @@ package service import ( "fmt" + "slices" "github.com/cloudoperators/heureka/internal/app/common" "github.com/cloudoperators/heureka/internal/app/event" "github.com/cloudoperators/heureka/internal/database" "github.com/cloudoperators/heureka/pkg/util" + "github.com/samber/lo" + "golang.org/x/exp/maps" "github.com/cloudoperators/heureka/internal/entity" "github.com/sirupsen/logrus" @@ -51,19 +54,45 @@ func (s *serviceHandler) getServiceResultsWithAggregations(filter *entity.Servic return nil, err } - for i := 0; i < len(servicesCiCount); i++ { - serviceCi := servicesCiCount[i] - serviceIm := servicesImCount[i] - cursor := fmt.Sprintf("%d", serviceCi.Id) + if len(servicesImCount) != len(servicesCiCount) { + return nil, fmt.Errorf("Error") + } + + // don't assume that results have some order + // create map with id -> service + ciCounts := map[int64]entity.ServiceWithAggregations{} + imCounts := map[int64]entity.ServiceWithAggregations{} + + lo.ForEach(servicesCiCount, func(s entity.ServiceWithAggregations, _ int) { + ciCounts[s.Id] = s + }) + + lo.ForEach(servicesImCount, func(s entity.ServiceWithAggregations, _ int) { + imCounts[s.Id] = s + }) + + ciIds := maps.Keys(ciCounts) + imIds := maps.Keys(imCounts) + + slices.Sort(ciIds) + slices.Sort(imIds) + + // check if same services were returned by the aggregation queries + if !slices.Equal(ciIds, imIds) { + return nil, fmt.Errorf("aggregation queries returned different services") + } + + for id := range ciIds { + cursor := fmt.Sprintf("%d", id) + service := ciCounts[int64(id)].Service serviceResults = append(serviceResults, entity.ServiceResult{ WithCursor: entity.WithCursor{Value: cursor}, ServiceAggregations: &entity.ServiceAggregations{ - IssueMatches: serviceIm.IssueMatches, - ComponentInstances: serviceCi.ComponentInstances, + IssueMatches: imCounts[int64(id)].IssueMatches, + ComponentInstances: ciCounts[int64(id)].ComponentInstances, }, - Service: util.Ptr(serviceCi.Service), + Service: util.Ptr(service), }) - } return serviceResults, nil diff --git a/internal/database/mariadb/service.go b/internal/database/mariadb/service.go index 045892eb..2d7debc1 100644 --- a/internal/database/mariadb/service.go +++ b/internal/database/mariadb/service.go @@ -16,31 +16,6 @@ const ( serviceWildCardFilterQuery = "S.service_name LIKE Concat('%',?,'%')" ) -type ServiceAggregationOptions struct { - IssueMatchCount bool - ComponentInstanceCount bool -} - -func (sa *ServiceAggregationOptions) GetIssueMatchCountAggregation() string { - return ", COUNT(distinct IM.issuematch_id) as agg_issue_matches" -} - -func (sa *ServiceAggregationOptions) GetComponentInstanceCountAggregation() string { - return ", SUM(CI.componentinstance_count) as agg_component_instances" -} - -func (sa *ServiceAggregationOptions) GetAggregationQuery() string { - if sa.ComponentInstanceCount { - return sa.GetComponentInstanceCountAggregation() - } - - if sa.IssueMatchCount { - return sa.GetIssueMatchCountAggregation() - } - - return "" -} - func (s *SqlDatabase) getServiceFilterString(filter *entity.ServiceFilter) string { var fl []string fl = append(fl, buildFilterQuery(filter.Name, "S.service_name = ?", OP_OR)) @@ -58,7 +33,7 @@ func (s *SqlDatabase) getServiceFilterString(filter *entity.ServiceFilter) strin return combineFilterQueries(fl, OP_AND) } -func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter, aggOps ServiceAggregationOptions) string { +func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter) string { joins := "" if len(filter.OwnerName) > 0 || len(filter.OwnerId) > 0 { joins = fmt.Sprintf("%s\n%s", joins, ` @@ -86,7 +61,7 @@ func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter, aggOps Servi LEFT JOIN Activity A on AHS.activityhasservice_activity_id = A.activity_id `) } - if len(filter.ComponentInstanceId) > 0 || aggOps.ComponentInstanceCount || aggOps.IssueMatchCount { + if len(filter.ComponentInstanceId) > 0 { joins = fmt.Sprintf("%s\n%s", joins, ` LEFT JOIN ComponentInstance CI on S.service_id = CI.componentinstance_service_id `) @@ -96,11 +71,6 @@ func (s *SqlDatabase) getServiceJoins(filter *entity.ServiceFilter, aggOps Servi LEFT JOIN IssueRepositoryService IRS on IRS.issuerepositoryservice_service_id = S.service_id `) } - if aggOps.IssueMatchCount { - joins = fmt.Sprintf("%s\n%s", joins, ` - LEFT JOIN IssueMatch IM on IM.issuematch_component_instance_id = CI.componentinstance_id - `) - } return joins } @@ -148,13 +118,13 @@ func (s *SqlDatabase) getServiceUpdateFields(service *entity.Service) string { return strings.Join(fl, ", ") } -func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.ServiceFilter, aggOpts ServiceAggregationOptions, withCursor bool, l *logrus.Entry) (*sqlx.Stmt, []interface{}, error) { +func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.ServiceFilter, withCursor bool, l *logrus.Entry) (*sqlx.Stmt, []interface{}, error) { var query string filter = s.ensureServiceFilter(filter) l.WithFields(logrus.Fields{"filter": filter}) filterStr := s.getServiceFilterString(filter) - joins := s.getServiceJoins(filter, aggOpts) + joins := s.getServiceJoins(filter) cursor := getCursor(filter.Paginated, filterStr, "S.service_id > ?") whereClause := "" @@ -162,13 +132,11 @@ func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.Ser whereClause = fmt.Sprintf("WHERE %s", filterStr) } - ags := aggOpts.GetAggregationQuery() - // construct final query if withCursor { - query = fmt.Sprintf(baseQuery, ags, joins, whereClause, cursor.Statement) + query = fmt.Sprintf(baseQuery, joins, whereClause, cursor.Statement) } else { - query = fmt.Sprintf(baseQuery, ags, joins, whereClause) + query = fmt.Sprintf(baseQuery, joins, whereClause) } //construct prepared statement and if where clause does exist add parameters @@ -207,17 +175,44 @@ func (s *SqlDatabase) buildServiceStatement(baseQuery string, filter *entity.Ser return stmt, filterParameters, nil } +func (s *SqlDatabase) getServicesWithAggregations(query string, filter *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { + l := logrus.WithFields(logrus.Fields{ + "filter": filter, + "event": "database.getServicesWithAggregation", + }) + stmt, filterParameters, err := s.buildServiceStatement(query, filter, true, l) + + if err != nil { + msg := ERROR_MSG_PREPARED_STMT + l.WithFields( + logrus.Fields{ + "error": err, + }).Error(msg) + return nil, fmt.Errorf("%s", msg) + } + defer stmt.Close() + + return performListScan( + stmt, + filterParameters, + l, + func(l []entity.ServiceWithAggregations, e GetServicesByRow) []entity.ServiceWithAggregations { + return append(l, e.AsServiceWithAggregations()) + }, + ) +} + func (s *SqlDatabase) CountServices(filter *entity.ServiceFilter) (int64, error) { l := logrus.WithFields(logrus.Fields{ "event": "database.CountServices", }) baseQuery := ` - SELECT count(distinct S.service_id) %s FROM Service S + SELECT count(distinct S.service_id) FROM Service S %s %s ` - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, false, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, false, l) if err != nil { return -1, err @@ -234,12 +229,12 @@ func (s *SqlDatabase) GetAllServiceIds(filter *entity.ServiceFilter) ([]int64, e }) baseQuery := ` - SELECT S.service_id %s FROM Service S + SELECT S.service_id FROM Service S %s %s GROUP BY S.service_id ORDER BY S.service_id ` - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, false, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, false, l) if err != nil { return nil, err @@ -256,7 +251,7 @@ func (s *SqlDatabase) GetServices(filter *entity.ServiceFilter) ([]entity.Servic }) baseQuery := ` - SELECT %s %s FROM Service S + SELECT %s FROM Service S %s %s %s GROUP BY S.service_id ORDER BY S.service_id LIMIT ? @@ -264,9 +259,9 @@ func (s *SqlDatabase) GetServices(filter *entity.ServiceFilter) ([]entity.Servic filter = s.ensureServiceFilter(filter) columns := s.getServiceColumns(filter) - baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s", "%s") + baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s") - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, true, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, true, l) if err != nil { return nil, err @@ -285,56 +280,44 @@ func (s *SqlDatabase) GetServices(filter *entity.ServiceFilter) ([]entity.Servic } func (s *SqlDatabase) GetServicesWithComponentInstanceCount(filter *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { - aggOpts := ServiceAggregationOptions{ - ComponentInstanceCount: true, - } - return s.getServicesWithAggregation(filter, aggOpts) -} + filter = s.ensureServiceFilter(filter) -func (s *SqlDatabase) GetServicesWithIssueMatchCount(filter *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { - aggOpts := ServiceAggregationOptions{ - IssueMatchCount: true, + baseQuery := ` + SELECT %s, SUM(CI.componentinstance_count) AS agg_component_instances FROM Service S + %s + %s + %s GROUP BY S.service_id ORDER BY S.service_id LIMIT ? + ` + + columns := s.getServiceColumns(filter) + baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s") + + if len(filter.ComponentInstanceId) == 0 { + baseQuery = fmt.Sprintf(baseQuery, "\nLEFT JOIN ComponentInstance CI on S.service_id = CI.componentinstance_service_id\n%s ", "%s", "%s") } - return s.getServicesWithAggregation(filter, aggOpts) + + return s.getServicesWithAggregations(baseQuery, filter) } -func (s *SqlDatabase) getServicesWithAggregation(filter *entity.ServiceFilter, aggOpts ServiceAggregationOptions) ([]entity.ServiceWithAggregations, error) { +func (s *SqlDatabase) GetServicesWithIssueMatchCount(filter *entity.ServiceFilter) ([]entity.ServiceWithAggregations, error) { filter = s.ensureServiceFilter(filter) - l := logrus.WithFields(logrus.Fields{ - "filter": filter, - "event": "database.GetServicesWithAggregations", - }) baseQuery := ` - SELECT %s %s FROM Service S + SELECT %s, COUNT(IM.issuematch_id) AS agg_issue_matches FROM Service S %s + LEFT JOIN IssueMatch IM on CI.componentinstance_id = IM.issuematch_component_instance_id %s %s GROUP BY S.service_id ORDER BY S.service_id LIMIT ? ` - columns := s.getServiceColumns(filter) - baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s", "%s") - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, aggOpts, true, l) + columns := s.getServiceColumns(filter) + baseQuery = fmt.Sprintf(baseQuery, columns, "%s", "%s", "%s") - if err != nil { - msg := ERROR_MSG_PREPARED_STMT - l.WithFields( - logrus.Fields{ - "error": err, - "aggregationOptionss": aggOpts, - }).Error(msg) - return nil, fmt.Errorf("%s", msg) + if len(filter.ComponentInstanceId) == 0 { + baseQuery = fmt.Sprintf(baseQuery, "\nLEFT JOIN ComponentInstance CI on S.service_id = CI.componentinstance_service_id\n%s ", "%s", "%s") } - defer stmt.Close() - return performListScan( - stmt, - filterParameters, - l, - func(l []entity.ServiceWithAggregations, e GetServicesByRow) []entity.ServiceWithAggregations { - return append(l, e.AsServiceWithAggregations()) - }, - ) + return s.getServicesWithAggregations(baseQuery, filter) } func (s *SqlDatabase) CreateService(service *entity.Service) (*entity.Service, error) { @@ -520,7 +503,7 @@ func (s *SqlDatabase) GetServiceNames(filter *entity.ServiceFilter) ([]string, e }) baseQuery := ` - SELECT service_name %s FROM Service S + SELECT service_name FROM Service S %s %s ` @@ -529,7 +512,7 @@ func (s *SqlDatabase) GetServiceNames(filter *entity.ServiceFilter) ([]string, e filter = s.ensureServiceFilter(filter) // Builds full statement with possible joins and filters - stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, ServiceAggregationOptions{}, false, l) + stmt, filterParameters, err := s.buildServiceStatement(baseQuery, filter, false, l) if err != nil { l.Error("Error preparing statement: ", err) return nil, err From 963961c5fefa7a3825a4802b56f43af62c09bf59 Mon Sep 17 00:00:00 2001 From: Michael Reimsbach Date: Mon, 7 Oct 2024 17:19:55 +0200 Subject: [PATCH 3/3] feat(service): add tests for service aggregations --- internal/database/mariadb/service_test.go | 41 +++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/internal/database/mariadb/service_test.go b/internal/database/mariadb/service_test.go index 587a7e66..905a4226 100644 --- a/internal/database/mariadb/service_test.go +++ b/internal/database/mariadb/service_test.go @@ -437,10 +437,36 @@ var _ = Describe("Service", Label("database", "Service"), func() { }) }) When("Getting Services with Aggregations", Label("GetServicesWithAggregations"), func() { - BeforeEach(func() { - _ = seeder.SeedDbWithNFakeData(10) + Context("and the database contains service without aggregations", func() { + BeforeEach(func() { + newServiceRow := test.NewFakeService() + newService := newServiceRow.AsService() + db.CreateService(&newService) + }) + It("returns the services with componentInstance count", func() { + entriesWithAggregations, err := db.GetServicesWithComponentInstanceCount(nil) + + By("throwing no error", func() { + Expect(err).To(BeNil()) + }) + + By("returning some aggregations", func() { + for _, entryWithAggregations := range entriesWithAggregations { + Expect(entryWithAggregations).NotTo( + BeEquivalentTo(entity.ServiceAggregations{})) + Expect(entryWithAggregations.ServiceAggregations.ComponentInstances).To(BeEquivalentTo(0)) + Expect(entryWithAggregations.ServiceAggregations.IssueMatches).To(BeEquivalentTo(0)) + } + }) + By("returning all services", func() { + Expect(len(entriesWithAggregations)).To(BeEquivalentTo(1)) + }) + }) }) - Context("and and we have 10 elements in the database", func() { + Context("and we have 10 services in the database", func() { + BeforeEach(func() { + _ = seeder.SeedDbWithNFakeData(10) + }) It("returns the services with componentInstance count", func() { entriesWithAggregations, err := db.GetServicesWithComponentInstanceCount(nil) @@ -454,6 +480,9 @@ var _ = Describe("Service", Label("database", "Service"), func() { BeEquivalentTo(entity.ServiceAggregations{})) } }) + By("returning all services", func() { + Expect(len(entriesWithAggregations)).To(BeEquivalentTo(10)) + }) }) It("returns correct aggregation values", func() { //Should be filled with a check for each aggregation value, @@ -466,6 +495,9 @@ var _ = Describe("Service", Label("database", "Service"), func() { }) }) Context("and and we have 10 elements in the database", func() { + BeforeEach(func() { + _ = seeder.SeedDbWithNFakeData(10) + }) It("returns the services with issueMatch count", func() { entriesWithAggregations, err := db.GetServicesWithIssueMatchCount(nil) @@ -479,6 +511,9 @@ var _ = Describe("Service", Label("database", "Service"), func() { BeEquivalentTo(entity.ServiceAggregations{})) } }) + By("returning all services", func() { + Expect(len(entriesWithAggregations)).To(BeEquivalentTo(10)) + }) }) It("returns correct aggregation values", func() { //Should be filled with a check for each aggregation value,