From 537840b10423e6b436d8f11b05b569ff81879177 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 26 Jul 2024 07:55:02 -0500 Subject: [PATCH 01/15] :bug: Fix task reaper using TTL.Failed. (#734) Signed-off-by: Jeff Ortel --- reaper/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reaper/task.go b/reaper/task.go index b78f06a17..3f73432f1 100644 --- a/reaper/task.go +++ b/reaper/task.go @@ -113,7 +113,7 @@ func (r *TaskReaper) Run() { if m.Terminated != nil { mark = *m.Terminated } - if m.TTL.Succeeded > 0 { + if m.TTL.Failed > 0 { d := time.Duration(m.TTL.Failed) * Unit if time.Since(mark) > d { r.delete(m) From 8f03da266f5fed4cde53128b9cd20e3a79a5c958 Mon Sep 17 00:00:00 2001 From: Samuel Lucidi Date: Fri, 26 Jul 2024 15:29:20 -0400 Subject: [PATCH 02/15] :seedling: Omit associations during Create operations (#733) When inserting a new record, GORM will also attempt to insert records into tables refered to by many-to-many relationships on the inserted record. This commit attempts to ensure that associations are omitted when inserting records, and then the associations are added to the join tables separately. Also fixes some spots where the Transaction handler was in the wrong spot in the chain or missing entirely. Fixes https://github.com/konveyor/tackle2-hub/issues/727 --------- Signed-off-by: Sam Lucidi --- api/application.go | 39 +++++++++++++++++++++++++++++++-------- api/archetype.go | 35 +++++++++++++++++++++++++++++++++-- api/group.go | 12 +++++++++++- api/identity.go | 2 +- api/migrationwave.go | 17 ++++++++++++++++- api/stakeholder.go | 22 +++++++++++++++++++++- 6 files changed, 113 insertions(+), 14 deletions(-) diff --git a/api/application.go b/api/application.go index 7169d1c2e..3d4940c89 100644 --- a/api/application.go +++ b/api/application.go @@ -58,22 +58,22 @@ func (h ApplicationHandler) AddRoutes(e *gin.Engine) { routeGroup.DELETE(ApplicationRoot, h.Delete) // Tags routeGroup = e.Group("/") - routeGroup.Use(Required("applications")) + routeGroup.Use(Required("applications"), Transaction) routeGroup.GET(ApplicationTagsRoot, h.TagList) routeGroup.GET(ApplicationTagsRoot+"/", h.TagList) routeGroup.POST(ApplicationTagsRoot, h.TagAdd) routeGroup.DELETE(ApplicationTagRoot, h.TagDelete) - routeGroup.PUT(ApplicationTagsRoot, h.TagReplace, Transaction) + routeGroup.PUT(ApplicationTagsRoot, h.TagReplace) // Facts routeGroup = e.Group("/") - routeGroup.Use(Required("applications.facts")) + routeGroup.Use(Required("applications.facts"), Transaction) routeGroup.GET(ApplicationFactsRoot, h.FactGet) routeGroup.GET(ApplicationFactsRoot+"/", h.FactGet) routeGroup.POST(ApplicationFactsRoot, h.FactCreate) routeGroup.GET(ApplicationFactRoot, h.FactGet) routeGroup.PUT(ApplicationFactRoot, h.FactPut) routeGroup.DELETE(ApplicationFactRoot, h.FactDelete) - routeGroup.PUT(ApplicationFactsRoot, h.FactPut, Transaction) + routeGroup.PUT(ApplicationFactsRoot, h.FactPut) // Bucket routeGroup = e.Group("/") routeGroup.Use(Required("applications.bucket")) @@ -84,11 +84,11 @@ func (h ApplicationHandler) AddRoutes(e *gin.Engine) { routeGroup.DELETE(AppBucketContentRoot, h.BucketDelete) // Stakeholders routeGroup = e.Group("/") - routeGroup.Use(Required("applications.stakeholders")) + routeGroup.Use(Required("applications.stakeholders"), Transaction) routeGroup.PUT(AppStakeholdersRoot, h.StakeholdersUpdate) // Assessments routeGroup = e.Group("/") - routeGroup.Use(Required("applications.assessments")) + routeGroup.Use(Required("applications.assessments"), Transaction) routeGroup.GET(AppAssessmentsRoot, h.AssessmentList) routeGroup.POST(AppAssessmentsRoot, h.AssessmentCreate) } @@ -210,11 +210,23 @@ func (h ApplicationHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Omit("Tags").Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + db := h.DB(ctx).Model(m) + err = db.Association("Identities").Replace(m.Identities) + if err != nil { + _ = ctx.Error(err) + return + } + db = h.DB(ctx).Model(m) + err = db.Association("Contributors").Replace(m.Contributors) + if err != nil { + _ = ctx.Error(err) + return + } tags := []model.ApplicationTag{} if len(r.Tags) > 0 { @@ -1078,11 +1090,21 @@ func (h ApplicationHandler) AssessmentCreate(ctx *gin.Context) { assessment.PrepareForApplication(resolver, application, m) newAssessment = true } - result = h.DB(ctx).Create(m) + result = h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } if newAssessment { metrics.AssessmentsInitiated.Inc() } @@ -1138,6 +1160,7 @@ func (r *Application) With(m *model.Application, tags []model.ApplicationTag) { r.Identities, ref) } + r.Tags = []TagRef{} for i := range tags { ref := TagRef{} ref.With(tags[i].TagID, tags[i].Tag.Name, tags[i].Source, false) diff --git a/api/archetype.go b/api/archetype.go index d62d88a13..bcadf05cd 100644 --- a/api/archetype.go +++ b/api/archetype.go @@ -136,12 +136,33 @@ func (h ArchetypeHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("CriteriaTags").Replace("CriteriaTags", m.CriteriaTags) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Tags").Replace("Tags", m.Tags) + if err != nil { + _ = ctx.Error(err) + return + } + archetypes := []model.Archetype{} db := h.preLoad(h.DB(ctx), "Tags", "CriteriaTags") result = db.Find(&archetypes) @@ -319,11 +340,21 @@ func (h ArchetypeHandler) AssessmentCreate(ctx *gin.Context) { assessment.PrepareForArchetype(resolver, archetype, m) newAssessment = true } - result = h.DB(ctx).Create(m) + result = h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } if newAssessment { metrics.AssessmentsInitiated.Inc() } diff --git a/api/group.go b/api/group.go index 4e92337b3..dee59ff97 100644 --- a/api/group.go +++ b/api/group.go @@ -97,11 +97,21 @@ func (h StakeholderGroupHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace(m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("MigrationWaves").Replace(m.MigrationWaves) + if err != nil { + _ = ctx.Error(err) + return + } r.With(m) h.Respond(ctx, http.StatusCreated, r) diff --git a/api/identity.go b/api/identity.go index dbaf606dd..8c631c6f7 100644 --- a/api/identity.go +++ b/api/identity.go @@ -34,7 +34,7 @@ func (h IdentityHandler) AddRoutes(e *gin.Engine) { routeGroup.GET(IdentitiesRoot+"/", h.setDecrypted, h.List) routeGroup.POST(IdentitiesRoot, h.Create) routeGroup.GET(IdentityRoot, h.setDecrypted, h.Get) - routeGroup.PUT(IdentityRoot, h.Update, Transaction) + routeGroup.PUT(IdentityRoot, Transaction, h.Update) routeGroup.DELETE(IdentityRoot, h.Delete) } diff --git a/api/migrationwave.go b/api/migrationwave.go index 8651a369e..128bb485f 100644 --- a/api/migrationwave.go +++ b/api/migrationwave.go @@ -98,11 +98,26 @@ func (h MigrationWaveHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Applications").Replace("Applications", m.Applications) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } r.With(m) h.Respond(ctx, http.StatusCreated, r) diff --git a/api/stakeholder.go b/api/stakeholder.go index 4cb659310..1a551cf4d 100644 --- a/api/stakeholder.go +++ b/api/stakeholder.go @@ -97,11 +97,31 @@ func (h StakeholderHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Groups").Replace(m.Groups) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Owns").Replace(m.Owns) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Contributes").Replace(m.Contributes) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("MigrationWaves").Replace(m.MigrationWaves) + if err != nil { + _ = ctx.Error(err) + return + } r.With(m) h.Respond(ctx, http.StatusCreated, r) From 4634415b4166c951aa7b6583840f911f5971bc96 Mon Sep 17 00:00:00 2001 From: David Zager Date: Mon, 5 Aug 2024 09:52:56 -0400 Subject: [PATCH 03/15] :seedling: oil the pr ci workflow (#709) Needs konveyor/ci#54 Signed-off-by: David Zager --- .github/workflows/main.yml | 67 ++++++++------------ .github/workflows/march-image-build-push.yml | 18 ++---- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f7f5b43c..9e99967c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,88 +12,75 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19' - run: make fmt + vet: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19' - run: make vet + build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19' - run: make cmd - build-image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - run: make podman-build test-unit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19' - run: make test test-api: + needs: + - fmt + - vet + - build + - test-unit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19' - run: | - make vet DISCONNECTED=1 make run & sleep 15 # probably a dirty solution DISCONNECTED=1 HUB_BASE_URL=http://localhost:8080 make test-api DISCONNECTED=1 HUB_BASE_URL=http://localhost:8080 make test-api # Intentionaly run 2x to catch data left in Hub DB. - test-e2e: + build-image: + needs: + - fmt + - vet + - build + - test-unit + - test-api runs-on: ubuntu-latest + env: + IMG: ttl.sh/konveyor-hub-${{ github.sha }}:2h steps: - - uses: actions/checkout@v3 - - name: start-minikube - uses: konveyor/tackle2-operator/.github/actions/start-minikube@main - - name: Build image in minikube - run: | - export SHELL=/bin/bash - eval $(minikube -p minikube docker-env) - make docker-build - - name: install-tackle - uses: konveyor/tackle2-operator/.github/actions/install-tackle@main - with: - tackle-hub-image: tackle2-hub:latest - tackle-image-pull-policy: IfNotPresent - - - name: save image - run: | - IMG=quay.io/konveyor/tackle2-hub:latest make docker-build - docker save -o /tmp/tackle2-hub.tar quay.io/konveyor/tackle2-hub:latest - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 - with: - name: tackle2-hub - path: /tmp/tackle2-hub.tar - retention-days: 1 + - uses: actions/checkout@v4 + - run: make docker-build + - run: docker push ${IMG} test-integration: - needs: test-e2e - uses: konveyor/ci/.github/workflows/global-ci.yml@main + needs: build-image + uses: konveyor/ci/.github/workflows/global-ci-bundle.yml@main with: - component_name: tackle2-hub + tackle_hub: ttl.sh/konveyor-hub-${{ github.sha }}:2h api_hub_tests_ref: ${{ github.ref }} diff --git a/.github/workflows/march-image-build-push.yml b/.github/workflows/march-image-build-push.yml index 8729d4cda..25bbcf065 100644 --- a/.github/workflows/march-image-build-push.yml +++ b/.github/workflows/march-image-build-push.yml @@ -19,15 +19,9 @@ jobs: runs-on: ubuntu-20.04 strategy: fail-fast: false - steps: - - name: Checkout Push to Registry action - uses: konveyor/release-tools/build-push-quay@main - with: - architectures: "amd64, arm64" - containerfile: "./Dockerfile" - image_name: "tackle2-hub" - image_namespace: "konveyor" - image_registry: "quay.io" - quay_publish_robot: ${{ secrets.QUAY_PUBLISH_ROBOT }} - quay_publish_token: ${{ secrets.QUAY_PUBLISH_TOKEN }} - ref: ${{ github.ref }} + uses: konveyor/release-tools/build-push-images.yaml@main + with: + registry: "quay.io/konveyor" + image_name: "tackle2-hub" + containerfile: "./Dockerfile" + architectures: '[ "amd64", "arm64" ]' From 7b875a72f74a529d44677bc7613b0d644da4d2b5 Mon Sep 17 00:00:00 2001 From: Samuel Lucidi Date: Tue, 13 Aug 2024 16:22:03 -0400 Subject: [PATCH 04/15] :ghost: Use JSON serializer everywhere (#680) Extends the use of the JSON serializer to the rest of the models. Signed-off-by: Sam Lucidi --- api/analysis.go | 81 +++++------- api/application.go | 28 ++-- api/archetype.go | 2 +- api/assessment.go | 44 ++++--- api/base.go | 6 - api/businessservice.go | 2 +- api/group.go | 2 +- api/identity.go | 2 +- api/jobfunction.go | 2 +- api/migrationwave.go | 2 +- api/proxy.go | 9 +- api/questionnaire.go | 44 +++---- api/review.go | 25 +--- api/ruleset.go | 18 +-- api/setting.go | 22 ++-- api/stakeholder.go | 2 +- api/tag.go | 2 +- api/tagcategory.go | 2 +- api/target.go | 33 ++--- api/ticket.go | 12 +- api/tracker.go | 2 +- assessment/assessment.go | 169 ++++++++++++------------ assessment/assessment_test.go | 198 +++++++++++++++-------------- assessment/membership.go | 4 +- assessment/pkg.go | 75 +---------- assessment/tag.go | 10 +- binding/application.go | 6 +- hack/cmd/addon/main.go | 20 +-- importer/manager.go | 6 +- migration/json/fields.go | 4 +- migration/migrate.go | 14 +- migration/migrate_test.go | 3 +- migration/v14/model/analysis.go | 77 ++++++----- migration/v14/model/application.go | 14 +- migration/v14/model/assessment.go | 68 +++++++++- migration/v14/model/core.go | 60 ++++----- model/pkg.go | 20 ++- seed/questionnaire.go | 11 +- seed/ruleset.go | 3 +- seed/seed.go | 24 ++-- seed/target.go | 8 +- test/api/application/facts_test.go | 2 +- test/api/assessment/samples.go | 18 +-- test/api/questionnaire/samples.go | 26 ++-- trigger/application.go | 2 +- 45 files changed, 576 insertions(+), 608 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index eb115113e..4080157cc 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -1060,11 +1060,9 @@ func (h AnalysisHandler) RuleReports(ctx *gin.Context) { Name: m.Name, } resources = append(resources, r) - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) - } - if m.Links != nil { - _ = json.Unmarshal(m.Links, &r.Links) + r.Labels = m.Labels + for _, l := range m.Links { + r.Links = append(r.Links, Link(l)) } r.Effort += m.Effort } @@ -1197,11 +1195,9 @@ func (h AnalysisHandler) AppIssueReports(ctx *gin.Context) { ID: m.ID, } resources = append(resources, r) - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) - } - if m.Links != nil { - _ = json.Unmarshal(m.Links, &r.Links) + r.Labels = m.Labels + for _, l := range m.Links { + r.Links = append(r.Links, Link(l)) } r.Effort += m.Effort } @@ -1722,13 +1718,9 @@ func (h AnalysisHandler) DepReports(ctx *gin.Context) { Name: m.Name, Applications: m.Applications, } - if m.Labels != nil { - var aggregated []string - _ = json.Unmarshal(m.Labels, &aggregated) - for _, s := range aggregated { - if s != "" { - r.Labels = append(r.Labels, s) - } + for _, s := range m.Labels { + if s != "" { + r.Labels = append(r.Labels, s) } } resources = append(resources, r) @@ -2082,7 +2074,7 @@ func (h *AnalysisHandler) archive(ctx *gin.Context, q *gorm.DB) (err error) { db = db.Where("n.IssueID = i.ID") db = db.Where("i.AnalysisID", m.ID) db = db.Group("i.ID") - summary := []ArchivedIssue{} + summary := []model.ArchivedIssue{} err = db.Scan(&summary).Error if err != nil { return @@ -2091,8 +2083,8 @@ func (h *AnalysisHandler) archive(ctx *gin.Context, q *gorm.DB) (err error) { db = db.Model(m) db = db.Omit(clause.Associations) m.Archived = true - m.Summary, _ = json.Marshal(summary) - err = db.Updates(h.fields(&m)).Error + m.Summary = summary + err = db.Save(&m).Error if err != nil { return } @@ -2155,7 +2147,7 @@ type Issue struct { Effort int `json:"effort,omitempty" yaml:",omitempty"` Incidents []Incident `json:"incidents,omitempty" yaml:",omitempty"` Links []Link `json:"links,omitempty" yaml:",omitempty"` - Facts FactMap `json:"facts,omitempty" yaml:",omitempty"` + Facts Map `json:"facts,omitempty" yaml:",omitempty"` Labels []string `json:"labels"` } @@ -2176,15 +2168,11 @@ func (r *Issue) With(m *model.Issue) { r.Incidents, n) } - if m.Links != nil { - _ = json.Unmarshal(m.Links, &r.Links) - } - if m.Facts != nil { - _ = json.Unmarshal(m.Facts, &r.Facts) - } - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) + for _, l := range m.Links { + r.Links = append(r.Links, Link(l)) } + r.Facts = m.Facts + r.Labels = m.Labels r.Effort = m.Effort } @@ -2203,9 +2191,11 @@ func (r *Issue) Model() (m *model.Issue) { m.Incidents, *n) } - m.Links, _ = json.Marshal(r.Links) - m.Facts, _ = json.Marshal(r.Facts) - m.Labels, _ = json.Marshal(r.Labels) + for _, l := range r.Links { + m.Links = append(m.Links, model.Link(l)) + } + m.Facts = r.Facts + m.Labels = r.Labels m.Effort = r.Effort return } @@ -2231,9 +2221,7 @@ func (r *TechDependency) With(m *model.TechDependency) { r.Version = m.Version r.Indirect = m.Indirect r.SHA = m.SHA - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) - } + r.Labels = m.Labels } // Model builds a model. @@ -2244,7 +2232,7 @@ func (r *TechDependency) Model() (m *model.TechDependency) { m.Version = r.Version m.Provider = r.Provider m.Indirect = r.Indirect - m.Labels, _ = json.Marshal(r.Labels) + m.Labels = r.Labels m.SHA = r.SHA return } @@ -2252,12 +2240,12 @@ func (r *TechDependency) Model() (m *model.TechDependency) { // Incident REST resource. type Incident struct { Resource `yaml:",inline"` - Issue uint `json:"issue"` - File string `json:"file"` - Line int `json:"line"` - Message string `json:"message"` - CodeSnip string `json:"codeSnip" yaml:"codeSnip"` - Facts FactMap `json:"facts"` + Issue uint `json:"issue"` + File string `json:"file"` + Line int `json:"line"` + Message string `json:"message"` + CodeSnip string `json:"codeSnip" yaml:"codeSnip"` + Facts Map `json:"facts"` } // With updates the resource with the model. @@ -2268,9 +2256,7 @@ func (r *Incident) With(m *model.Incident) { r.Line = m.Line r.Message = m.Message r.CodeSnip = m.CodeSnip - if m.Facts != nil { - _ = json.Unmarshal(m.Facts, &r.Facts) - } + r.Facts = m.Facts } // Model builds a model. @@ -2280,7 +2266,7 @@ func (r *Incident) Model() (m *model.Incident) { m.Line = r.Line m.Message = r.Message m.CodeSnip = r.CodeSnip - m.Facts, _ = json.Marshal(r.Facts) + m.Facts = r.Facts return } @@ -2371,9 +2357,6 @@ type DepAppReport struct { } `json:"dependency"` } -// FactMap map. -type FactMap map[string]any - // IssueWriter used to create a file containing issues. type IssueWriter struct { encoder diff --git a/api/application.go b/api/application.go index 3d4940c89..6b61e3c6a 100644 --- a/api/application.go +++ b/api/application.go @@ -362,7 +362,7 @@ func (h ApplicationHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations, "BucketID") - result = db.Updates(h.fields(m)) + result = db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -722,12 +722,10 @@ func (h ApplicationHandler) FactList(ctx *gin.Context, key FactKey) { return } - facts := FactMap{} + facts := Map{} for i := range list { fact := &list[i] - var v any - _ = json.Unmarshal(fact.Value, &v) - facts[fact.Key] = v + facts[fact.Key] = fact.Value } h.Respond(ctx, http.StatusOK, facts) } @@ -772,9 +770,7 @@ func (h ApplicationHandler) FactGet(ctx *gin.Context) { return } - var v any - _ = json.Unmarshal(list[0].Value, &v) - h.Respond(ctx, http.StatusOK, v) + h.Respond(ctx, http.StatusOK, list[0].Value) } // FactCreate godoc @@ -846,12 +842,11 @@ func (h ApplicationHandler) FactPut(ctx *gin.Context) { return } - value, _ := json.Marshal(f.Value) m := &model.Fact{ Key: key.Name(), Source: key.Source(), ApplicationID: id, - Value: value, + Value: f.Value, } db := h.DB(ctx) result = db.Save(m) @@ -906,7 +901,7 @@ func (h ApplicationHandler) FactDelete(ctx *gin.Context) { // @param factmap body api.FactMap true "Fact map" func (h ApplicationHandler) FactReplace(ctx *gin.Context, key FactKey) { id := h.pk(ctx) - facts := FactMap{} + facts := Map{} err := h.Bind(ctx, &facts) if err != nil { _ = ctx.Error(err) @@ -1145,7 +1140,10 @@ func (r *Application) With(m *model.Application, tags []model.ApplicationTag) { r.Bucket = r.refPtr(m.BucketID, m.Bucket) r.Comments = m.Comments r.Binary = m.Binary - _ = json.Unmarshal(m.Repository, &r.Repository) + if m.Repository != (model.Repository{}) { + repo := Repository(m.Repository) + r.Repository = &repo + } if m.Review != nil { ref := &Ref{} ref.With(m.Review.ID, "") @@ -1246,7 +1244,7 @@ func (r *Application) Model() (m *model.Application) { } m.ID = r.ID if r.Repository != nil { - m.Repository, _ = json.Marshal(r.Repository) + m.Repository = model.Repository(*r.Repository) } if r.BusinessService != nil { m.BusinessServiceID = &r.BusinessService.ID @@ -1307,14 +1305,14 @@ type Fact struct { func (r *Fact) With(m *model.Fact) { r.Key = m.Key r.Source = m.Source - _ = json.Unmarshal(m.Value, &r.Value) + r.Value = m.Value } func (r *Fact) Model() (m *model.Fact) { m = &model.Fact{} m.Key = r.Key m.Source = r.Source - m.Value, _ = json.Marshal(r.Value) + m.Value = r.Value return } diff --git a/api/archetype.go b/api/archetype.go index bcadf05cd..dafd141ef 100644 --- a/api/archetype.go +++ b/api/archetype.go @@ -232,7 +232,7 @@ func (h ArchetypeHandler) Update(ctx *gin.Context) { m.UpdateUser = h.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/assessment.go b/api/assessment.go index d024637eb..438c16eb8 100644 --- a/api/assessment.go +++ b/api/assessment.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" @@ -126,7 +125,7 @@ func (h AssessmentHandler) Update(ctx *gin.Context) { m.UpdateUser = h.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations, "Thresholds", "RiskMessages") - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -148,21 +147,25 @@ func (h AssessmentHandler) Update(ctx *gin.Context) { // Assessment REST resource. type Assessment struct { Resource `yaml:",inline"` - Application *Ref `json:"application,omitempty" yaml:",omitempty" binding:"excluded_with=Archetype"` - Archetype *Ref `json:"archetype,omitempty" yaml:",omitempty" binding:"excluded_with=Application"` - Questionnaire Ref `json:"questionnaire" binding:"required"` - Sections []assessment.Section `json:"sections" binding:"dive"` - Stakeholders []Ref `json:"stakeholders"` - StakeholderGroups []Ref `json:"stakeholderGroups" yaml:"stakeholderGroups"` + Application *Ref `json:"application,omitempty" yaml:",omitempty" binding:"excluded_with=Archetype"` + Archetype *Ref `json:"archetype,omitempty" yaml:",omitempty" binding:"excluded_with=Application"` + Questionnaire Ref `json:"questionnaire" binding:"required"` + Sections []Section `json:"sections" binding:"dive"` + Stakeholders []Ref `json:"stakeholders"` + StakeholderGroups []Ref `json:"stakeholderGroups" yaml:"stakeholderGroups"` // read only - Risk string `json:"risk"` - Confidence int `json:"confidence"` - Status string `json:"status"` - Thresholds assessment.Thresholds `json:"thresholds"` - RiskMessages assessment.RiskMessages `json:"riskMessages" yaml:"riskMessages"` - Required bool `json:"required"` + Risk string `json:"risk"` + Confidence int `json:"confidence"` + Status string `json:"status"` + Thresholds Thresholds `json:"thresholds"` + RiskMessages RiskMessages `json:"riskMessages" yaml:"riskMessages"` + Required bool `json:"required"` } +type Section model.Section +type Thresholds model.Thresholds +type RiskMessages model.RiskMessages + // With updates the resource with the model. func (r *Assessment) With(m *model.Assessment) { r.Resource.With(&m.Model) @@ -186,9 +189,12 @@ func (r *Assessment) With(m *model.Assessment) { r.Required = a.Questionnaire.Required r.Risk = a.Risk() r.Confidence = a.Confidence() - r.RiskMessages = a.RiskMessages - r.Thresholds = a.Thresholds - r.Sections = a.Sections + r.RiskMessages = RiskMessages(a.RiskMessages) + r.Thresholds = Thresholds(a.Thresholds) + r.Sections = []Section{} + for _, s := range a.Sections { + r.Sections = append(r.Sections, Section(s)) + } r.Status = a.Status() } @@ -196,8 +202,8 @@ func (r *Assessment) With(m *model.Assessment) { func (r *Assessment) Model() (m *model.Assessment) { m = &model.Assessment{} m.ID = r.ID - if r.Sections != nil { - m.Sections, _ = json.Marshal(r.Sections) + for _, s := range r.Sections { + m.Sections = append(m.Sections, model.Section(s)) } m.QuestionnaireID = r.Questionnaire.ID if r.Archetype != nil { diff --git a/api/base.go b/api/base.go index 602ac3be4..3e4e713a8 100644 --- a/api/base.go +++ b/api/base.go @@ -84,12 +84,6 @@ func (h *BaseHandler) preLoad(db *gorm.DB, fields ...string) (tx *gorm.DB) { return } -// fields builds a map of fields. -func (h *BaseHandler) fields(m any) (mp map[string]any) { - mp = reflect.Fields(m) - return -} - // pk returns the PK (ID) parameter. func (h *BaseHandler) pk(ctx *gin.Context) (id uint) { s := ctx.Param(ID) diff --git a/api/businessservice.go b/api/businessservice.go index 41caf95b5..dabc03c09 100644 --- a/api/businessservice.go +++ b/api/businessservice.go @@ -153,7 +153,7 @@ func (h BusinessServiceHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/group.go b/api/group.go index dee59ff97..daf876519 100644 --- a/api/group.go +++ b/api/group.go @@ -163,7 +163,7 @@ func (h StakeholderGroupHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/identity.go b/api/identity.go index 8c631c6f7..e01e89b08 100644 --- a/api/identity.go +++ b/api/identity.go @@ -202,7 +202,7 @@ func (h IdentityHandler) Update(ctx *gin.Context) { m.ID = id m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) - err = db.Updates(h.fields(m)).Error + err = db.Save(m).Error if err != nil { _ = ctx.Error(err) return diff --git a/api/jobfunction.go b/api/jobfunction.go index 82e615956..135dd1cb2 100644 --- a/api/jobfunction.go +++ b/api/jobfunction.go @@ -153,7 +153,7 @@ func (h JobFunctionHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/migrationwave.go b/api/migrationwave.go index 128bb485f..b92b23d6b 100644 --- a/api/migrationwave.go +++ b/api/migrationwave.go @@ -145,7 +145,7 @@ func (h MigrationWaveHandler) Update(ctx *gin.Context) { m.UpdateUser = h.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/proxy.go b/api/proxy.go index 72f3dd67b..ccaf1551c 100644 --- a/api/proxy.go +++ b/api/proxy.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" @@ -161,7 +160,7 @@ func (h ProxyHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -189,7 +188,7 @@ func (r *Proxy) With(m *model.Proxy) { r.Host = m.Host r.Port = m.Port r.Identity = r.refPtr(m.IdentityID, m.Identity) - _ = json.Unmarshal(m.Excluded, &r.Excluded) + r.Excluded = m.Excluded if r.Excluded == nil { r.Excluded = []string{} } @@ -205,9 +204,7 @@ func (r *Proxy) Model() (m *model.Proxy) { } m.ID = r.ID m.IdentityID = r.idPtr(r.Identity) - if r.Excluded != nil { - m.Excluded, _ = json.Marshal(r.Excluded) - } + m.Excluded = r.Excluded return } diff --git a/api/questionnaire.go b/api/questionnaire.go index 1dcb8c50d..ba183afe7 100644 --- a/api/questionnaire.go +++ b/api/questionnaire.go @@ -1,11 +1,9 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" - "github.com/konveyor/tackle2-hub/assessment" "github.com/konveyor/tackle2-hub/model" "gorm.io/gorm/clause" ) @@ -167,19 +165,16 @@ func (h QuestionnaireHandler) Update(ctx *gin.Context) { updated := r.Model() updated.ID = id updated.UpdateUser = h.CurrentUser(ctx) - var fields map[string]any if m.Builtin() { - fields = map[string]any{ - "updateUser": updated.UpdateUser, - "required": updated.Required, - } + m.UpdateUser = updated.UpdateUser + m.Required = updated.Required } else { - fields = h.fields(updated) + m = updated } db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result = db.Updates(fields) + result = db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -190,13 +185,13 @@ func (h QuestionnaireHandler) Update(ctx *gin.Context) { type Questionnaire struct { Resource `yaml:",inline"` - Name string `json:"name" yaml:"name" binding:"required"` - Description string `json:"description" yaml:"description"` - Required bool `json:"required" yaml:"required"` - Sections []assessment.Section `json:"sections" yaml:"sections" binding:"required,min=1,dive"` - Thresholds assessment.Thresholds `json:"thresholds" yaml:"thresholds" binding:"required"` - RiskMessages assessment.RiskMessages `json:"riskMessages" yaml:"riskMessages" binding:"required"` - Builtin bool `json:"builtin,omitempty" yaml:"builtin,omitempty"` + Name string `json:"name" yaml:"name" binding:"required"` + Description string `json:"description" yaml:"description"` + Required bool `json:"required" yaml:"required"` + Sections []Section `json:"sections" yaml:"sections" binding:"required,min=1,dive"` + Thresholds Thresholds `json:"thresholds" yaml:"thresholds" binding:"required"` + RiskMessages RiskMessages `json:"riskMessages" yaml:"riskMessages" binding:"required"` + Builtin bool `json:"builtin,omitempty" yaml:"builtin,omitempty"` } // With updates the resource with the model. @@ -206,9 +201,12 @@ func (r *Questionnaire) With(m *model.Questionnaire) { r.Description = m.Description r.Required = m.Required r.Builtin = m.Builtin() - _ = json.Unmarshal(m.Sections, &r.Sections) - _ = json.Unmarshal(m.Thresholds, &r.Thresholds) - _ = json.Unmarshal(m.RiskMessages, &r.RiskMessages) + r.Sections = []Section{} + for _, s := range m.Sections { + r.Sections = append(r.Sections, Section(s)) + } + r.Thresholds = Thresholds(m.Thresholds) + r.RiskMessages = RiskMessages(m.RiskMessages) } // Model builds a model. @@ -219,9 +217,11 @@ func (r *Questionnaire) Model() (m *model.Questionnaire) { Required: r.Required, } m.ID = r.ID - m.Sections, _ = json.Marshal(r.Sections) - m.Thresholds, _ = json.Marshal(r.Thresholds) - m.RiskMessages, _ = json.Marshal(r.RiskMessages) + for _, s := range r.Sections { + m.Sections = append(m.Sections, model.Section(s)) + } + m.Thresholds = model.Thresholds(r.Thresholds) + m.RiskMessages = model.RiskMessages(r.RiskMessages) return } diff --git a/api/review.go b/api/review.go index 85fbbdcb8..db4c65222 100644 --- a/api/review.go +++ b/api/review.go @@ -30,7 +30,7 @@ func (h ReviewHandler) AddRoutes(e *gin.Engine) { routeGroup.GET(ReviewRoot, h.Get) routeGroup.PUT(ReviewRoot, h.Update) routeGroup.DELETE(ReviewRoot, h.Delete) - routeGroup.POST(CopyRoot, h.CopyReview) + routeGroup.POST(CopyRoot, h.CopyReview, Transaction) } // Get godoc @@ -155,7 +155,7 @@ func (h ReviewHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -194,26 +194,15 @@ func (h ReviewHandler) CopyReview(ctx *gin.Context) { Comments: m.Comments, ApplicationID: &id, } - existing := []model.Review{} - result = h.DB(ctx).Find(&existing, "applicationid = ?", id) + result = h.DB(ctx).Delete(&model.Review{}, "applicationid = ?", id) if result.Error != nil { _ = ctx.Error(result.Error) return } - // if the application doesn't already have a review, create one. - if len(existing) == 0 { - result = h.DB(ctx).Create(copied) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } - // if the application already has a review, replace it with the copied review. - } else { - result = h.DB(ctx).Model(&existing[0]).Updates(h.fields(copied)) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } + result = h.DB(ctx).Create(copied) + if result.Error != nil { + _ = ctx.Error(result.Error) + return } } h.Status(ctx, http.StatusNoContent) diff --git a/api/ruleset.go b/api/ruleset.go index 95bfabb58..43d8e4023 100644 --- a/api/ruleset.go +++ b/api/ruleset.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" @@ -249,7 +248,7 @@ func (h *RuleSetHandler) update(ctx *gin.Context, r *RuleSet) (err error) { m.UpdateUser = h.CurrentUser(ctx) db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - err = db.Updates(h.fields(m)).Error + err = db.Save(m).Error if err != nil { return } @@ -268,7 +267,7 @@ func (h *RuleSetHandler) update(ctx *gin.Context, r *RuleSet) (err error) { for i := range m.Rules { m := &m.Rules[i] db = h.DB(ctx).Model(m) - err = db.Updates(h.fields(m)).Error + err = db.Updates(m).Error if err != nil { return } @@ -313,7 +312,10 @@ func (r *RuleSet) With(m *model.RuleSet) { r.Name = m.Name r.Description = m.Description r.Identity = r.refPtr(m.IdentityID, m.Identity) - _ = json.Unmarshal(m.Repository, &r.Repository) + if m.Repository != (model.Repository{}) { + repo := Repository(m.Repository) + r.Repository = &repo + } r.Rules = []Rule{} for i := range m.Rules { rule := Rule{} @@ -344,7 +346,7 @@ func (r *RuleSet) Model() (m *model.RuleSet) { m.Rules = append(m.Rules, *rule.Model()) } if r.Repository != nil { - m.Repository, _ = json.Marshal(r.Repository) + m.Repository = model.Repository(*r.Repository) } for _, ref := range r.DependsOn { m.DependsOn = append( @@ -382,7 +384,7 @@ type Rule struct { func (r *Rule) With(m *model.Rule) { r.Resource.With(&m.Model) r.Name = m.Name - _ = json.Unmarshal(m.Labels, &r.Labels) + r.Labels = m.Labels r.File = r.refPtr(m.FileID, m.File) } @@ -391,9 +393,7 @@ func (r *Rule) Model() (m *model.Rule) { m = &model.Rule{} m.ID = r.ID m.Name = r.Name - if r.Labels != nil { - m.Labels, _ = json.Marshal(r.Labels) - } + m.Labels = r.Labels m.FileID = r.idPtr(r.File) return } diff --git a/api/setting.go b/api/setting.go index a758f4b81..ad5c6997e 100644 --- a/api/setting.go +++ b/api/setting.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "fmt" "net/http" "strings" @@ -179,19 +178,19 @@ func (h SettingHandler) Update(ctx *gin.Context) { return } - updates := Setting{} - updates.Key = key - err := h.Bind(ctx, &updates.Value) + m := &model.Setting{} + result := h.DB(ctx).First(m, "key = ?", key) + if result.Error != nil { + _ = ctx.Error(result.Error) + return + } + err := h.Bind(ctx, &m.Value) if err != nil { _ = ctx.Error(err) return } - - m := updates.Model() m.UpdateUser = h.BaseHandler.CurrentUser(ctx) - db := h.DB(ctx).Model(m) - db = db.Where("key", key) - result := db.Updates(h.fields(m)) + result = h.DB(ctx).Save(m) if result.Error != nil { _ = ctx.Error(result.Error) } @@ -235,12 +234,11 @@ type Setting struct { func (r *Setting) With(m *model.Setting) { r.Key = m.Key - _ = json.Unmarshal(m.Value, &r.Value) - + r.Value = m.Value } func (r *Setting) Model() (m *model.Setting) { m = &model.Setting{Key: r.Key} - m.Value, _ = json.Marshal(r.Value) + m.Value = r.Value return } diff --git a/api/stakeholder.go b/api/stakeholder.go index 1a551cf4d..eead223b9 100644 --- a/api/stakeholder.go +++ b/api/stakeholder.go @@ -173,7 +173,7 @@ func (h StakeholderHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/tag.go b/api/tag.go index 17d472444..648456b5f 100644 --- a/api/tag.go +++ b/api/tag.go @@ -153,7 +153,7 @@ func (h TagHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/tagcategory.go b/api/tagcategory.go index 8ae7799b2..025065095 100644 --- a/api/tagcategory.go +++ b/api/tagcategory.go @@ -160,7 +160,7 @@ func (h TagCategoryHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/target.go b/api/target.go index efccc4d10..c27c8fe00 100644 --- a/api/target.go +++ b/api/target.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "errors" "fmt" "net/http" @@ -225,7 +224,7 @@ func (h TargetHandler) Update(ctx *gin.Context) { } db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result = db.Updates(h.fields(m)) + result = db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -237,20 +236,17 @@ func (h TargetHandler) Update(ctx *gin.Context) { // Target REST resource. type Target struct { Resource `yaml:",inline"` - Name string `json:"name"` - Description string `json:"description"` - Provider string `json:"provider,omitempty" yaml:",omitempty"` - Choice bool `json:"choice,omitempty" yaml:",omitempty"` - Custom bool `json:"custom,omitempty" yaml:",omitempty"` - Labels []Label `json:"labels"` - Image Ref `json:"image"` - RuleSet *RuleSet `json:"ruleset,omitempty" yaml:"ruleset,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Provider string `json:"provider,omitempty" yaml:",omitempty"` + Choice bool `json:"choice,omitempty" yaml:",omitempty"` + Custom bool `json:"custom,omitempty" yaml:",omitempty"` + Labels []TargetLabel `json:"labels"` + Image Ref `json:"image"` + RuleSet *RuleSet `json:"ruleset,omitempty" yaml:"ruleset,omitempty"` } -type Label struct { - Name string `json:"name"` - Label string `json:"label"` -} +type TargetLabel model.TargetLabel // With updates the resource with the model. func (r *Target) With(m *model.Target) { @@ -269,7 +265,10 @@ func (r *Target) With(m *model.Target) { imgRef.Name = m.Image.Name } r.Image = imgRef - _ = json.Unmarshal(m.Labels, &r.Labels) + r.Labels = []TargetLabel{} + for _, l := range m.Labels { + r.Labels = append(r.Labels, TargetLabel(l)) + } } // Model builds a model. @@ -282,6 +281,8 @@ func (r *Target) Model() (m *model.Target) { } m.ID = r.ID m.ImageID = r.Image.ID - m.Labels, _ = json.Marshal(r.Labels) + for _, l := range r.Labels { + m.Labels = append(m.Labels, model.TargetLabel(l)) + } return } diff --git a/api/ticket.go b/api/ticket.go index e25af5611..e716d9161 100644 --- a/api/ticket.go +++ b/api/ticket.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "time" @@ -156,7 +155,7 @@ type Ticket struct { Message string `json:"message"` Status string `json:"status"` LastUpdated time.Time `json:"lastUpdated" yaml:"lastUpdated"` - Fields Fields `json:"fields"` + Fields Map `json:"fields"` Application Ref `json:"application" binding:"required"` Tracker Ref `json:"tracker" binding:"required"` } @@ -174,7 +173,7 @@ func (r *Ticket) With(m *model.Ticket) { r.LastUpdated = m.LastUpdated r.Application = r.ref(m.ApplicationID, m.Application) r.Tracker = r.ref(m.TrackerID, m.Tracker) - _ = json.Unmarshal(m.Fields, &r.Fields) + r.Fields = m.Fields } // Model builds a model. @@ -185,13 +184,8 @@ func (r *Ticket) Model() (m *model.Ticket) { ApplicationID: r.Application.ID, TrackerID: r.Tracker.ID, } - if r.Fields == nil { - r.Fields = Fields{} - } - m.Fields, _ = json.Marshal(r.Fields) + m.Fields = r.Fields m.ID = r.ID return } - -type Fields map[string]any diff --git a/api/tracker.go b/api/tracker.go index 650607bac..96434ed12 100644 --- a/api/tracker.go +++ b/api/tracker.go @@ -180,7 +180,7 @@ func (h TrackerHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/assessment/assessment.go b/assessment/assessment.go index 6a874a519..aa9b56cb4 100644 --- a/assessment/assessment.go +++ b/assessment/assessment.go @@ -1,7 +1,6 @@ package assessment import ( - "encoding/json" "math" "github.com/konveyor/tackle2-hub/model" @@ -10,17 +9,11 @@ import ( // Assessment represents a deserialized Assessment. type Assessment struct { *model.Assessment - Sections []Section `json:"sections"` - Thresholds Thresholds `json:"thresholds"` - RiskMessages RiskMessages `json:"riskMessages"` } // With updates the Assessment with the db model and deserializes its fields. func (r *Assessment) With(m *model.Assessment) { r.Assessment = m - _ = json.Unmarshal(m.Sections, &r.Sections) - _ = json.Unmarshal(m.Thresholds, &r.Thresholds) - _ = json.Unmarshal(m.RiskMessages, &r.RiskMessages) } // Status returns the started status of the assessment. @@ -37,7 +30,7 @@ func (r *Assessment) Status() string { // Complete returns whether all sections have been completed. func (r *Assessment) Complete() bool { for _, s := range r.Sections { - if !s.Complete() { + if !r.sectionComplete(&s) { return false } } @@ -47,7 +40,7 @@ func (r *Assessment) Complete() bool { // Started returns whether any sections have been started. func (r *Assessment) Started() bool { for _, s := range r.Sections { - if s.Started() { + if r.sectionStarted(&s) { return true } } @@ -59,7 +52,7 @@ func (r *Assessment) Risk() string { var total uint colors := make(map[string]uint) for _, s := range r.Sections { - for _, risk := range s.Risks() { + for _, risk := range r.sectionRisks(&s) { colors[risk]++ total++ } @@ -85,8 +78,8 @@ func (r *Assessment) Confidence() (score int) { totalQuestions := 0 riskCounts := make(map[string]int) for _, s := range r.Sections { - for _, r := range s.Risks() { - riskCounts[r]++ + for _, risk := range r.sectionRisks(&s) { + riskCounts[risk]++ totalQuestions++ } } @@ -118,18 +111,73 @@ func (r *Assessment) Confidence() (score int) { return } -// Section represents a group of questions in a questionnaire. -type Section struct { - Order *uint `json:"order" yaml:"order" binding:"required"` - Name string `json:"name" yaml:"name"` - Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` - Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +func (r *Assessment) Prepare(tagResolver *TagResolver, tags Set) { + for i := range r.Sections { + s := &r.Sections[i] + includedQuestions := []model.Question{} + for _, q := range s.Questions { + for j := range q.Answers { + a := &q.Answers[j] + autoAnswerTags := NewSet() + for _, t := range a.AutoAnswerFor { + tag, found := tagResolver.Resolve(t.Category, t.Tag) + if found { + autoAnswerTags.Add(tag.ID) + } + } + if tags.Intersects(autoAnswerTags) { + a.AutoAnswered = true + a.Selected = true + break + } + } + + if len(q.IncludeFor) > 0 { + includeForTags := NewSet() + for _, t := range q.IncludeFor { + tag, found := tagResolver.Resolve(t.Category, t.Tag) + if found { + includeForTags.Add(tag.ID) + } + } + if tags.Intersects(includeForTags) { + includedQuestions = append(includedQuestions, q) + } + continue + } + + if len(q.ExcludeFor) > 0 { + excludeForTags := NewSet() + for _, t := range q.ExcludeFor { + tag, found := tagResolver.Resolve(t.Category, t.Tag) + if found { + excludeForTags.Add(tag.ID) + } + } + if tags.Intersects(excludeForTags) { + continue + } + } + includedQuestions = append(includedQuestions, q) + } + s.Questions = includedQuestions + } + return +} + +func (r *Assessment) Tags() (tags []model.CategorizedTag) { + for _, s := range r.Sections { + for _, t := range r.sectionTags(&s) { + tags = append(tags, t) + } + } + return } // Complete returns whether all questions in the section have been answered. -func (r *Section) Complete() bool { - for _, q := range r.Questions { - if !q.Answered() { +func (r *Assessment) sectionComplete(s *model.Section) bool { + for _, q := range s.Questions { + if !r.questionAnswered(&q) { return false } } @@ -137,9 +185,10 @@ func (r *Section) Complete() bool { } // Started returns whether any questions in the section have been answered. -func (r *Section) Started() bool { - for _, q := range r.Questions { - if q.Answered() && !q.AutoAnswered() { + +func (r *Assessment) sectionStarted(s *model.Section) bool { + for _, q := range s.Questions { + if r.questionAnswered(&q) && !r.questionAutoAnswered(&q) { return true } } @@ -147,36 +196,26 @@ func (r *Section) Started() bool { } // Risks returns a slice of the risks of each of its questions. -func (r *Section) Risks() []string { +func (r *Assessment) sectionRisks(s *model.Section) []string { risks := []string{} - for _, q := range r.Questions { - risks = append(risks, q.Risk()) + for _, q := range s.Questions { + risks = append(risks, r.questionRisk(&q)) } return risks } // Tags returns all the tags that should be applied based on how // the questions in the section have been answered. -func (r *Section) Tags() (tags []CategorizedTag) { - for _, q := range r.Questions { - tags = append(tags, q.Tags()...) +func (r *Assessment) sectionTags(s *model.Section) (tags []model.CategorizedTag) { + for _, q := range s.Questions { + tags = append(tags, r.questionTags(&q)...) } return } -// Question represents a question in a questionnaire. -type Question struct { - Order *uint `json:"order" yaml:"order" binding:"required"` - Text string `json:"text" yaml:"text"` - Explanation string `json:"explanation" yaml:"explanation"` - IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` - ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` - Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` -} - // Risk returns the risk level for the question based on how it has been answered. -func (r *Question) Risk() string { - for _, a := range r.Answers { +func (r *Assessment) questionRisk(q *model.Question) string { + for _, a := range q.Answers { if a.Selected { return a.Risk } @@ -185,8 +224,8 @@ func (r *Question) Risk() string { } // Answered returns whether the question has had an answer selected. -func (r *Question) Answered() bool { - for _, a := range r.Answers { +func (r *Assessment) questionAnswered(q *model.Question) bool { + for _, a := range q.Answers { if a.Selected { return true } @@ -196,8 +235,8 @@ func (r *Question) Answered() bool { // AutoAnswered returns whether the question has had an // answer pre-selected by the system. -func (r *Question) AutoAnswered() bool { - for _, a := range r.Answers { +func (r *Assessment) questionAutoAnswered(q *model.Question) bool { + for _, a := range q.Answers { if a.AutoAnswered { return true } @@ -206,8 +245,8 @@ func (r *Question) AutoAnswered() bool { } // Tags returns any tags to be applied based on how the question is answered. -func (r *Question) Tags() (tags []CategorizedTag) { - for _, answer := range r.Answers { +func (r *Assessment) questionTags(q *model.Question) (tags []model.CategorizedTag) { + for _, answer := range q.Answers { if answer.Selected { tags = answer.ApplyTags return @@ -215,37 +254,3 @@ func (r *Question) Tags() (tags []CategorizedTag) { } return } - -// Answer represents an answer to a question in a questionnaire. -type Answer struct { - Order *uint `json:"order" yaml:"order" binding:"required"` - Text string `json:"text" yaml:"text"` - Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` - Rationale string `json:"rationale" yaml:"rationale"` - Mitigation string `json:"mitigation" yaml:"mitigation"` - ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` - AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` - Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` - AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` -} - -// CategorizedTag represents a human-readable pair of category and tag. -type CategorizedTag struct { - Category string `json:"category" yaml:"category"` - Tag string `json:"tag" yaml:"tag"` -} - -// RiskMessages contains messages to display for each risk level. -type RiskMessages struct { - Red string `json:"red" yaml:"red"` - Yellow string `json:"yellow" yaml:"yellow"` - Green string `json:"green" yaml:"green"` - Unknown string `json:"unknown" yaml:"unknown"` -} - -// Thresholds contains the threshold values for determining risk for the questionnaire. -type Thresholds struct { - Red uint `json:"red" yaml:"red"` - Yellow uint `json:"yellow" yaml:"yellow"` - Unknown uint `json:"unknown" yaml:"unknown"` -} diff --git a/assessment/assessment_test.go b/assessment/assessment_test.go index 28efe1154..69df6702b 100644 --- a/assessment/assessment_test.go +++ b/assessment/assessment_test.go @@ -7,15 +7,16 @@ import ( "github.com/onsi/gomega" ) -func TestPrepareSections(t *testing.T) { +func TestPrepare(t *testing.T) { g := gomega.NewGomegaWithT(t) - sections := []Section{ + assessment := model.Assessment{} + assessment.Sections = []model.Section{ { - Questions: []Question{ + Questions: []model.Question{ { Text: "Default", - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", }, @@ -26,10 +27,10 @@ func TestPrepareSections(t *testing.T) { }, { Text: "Should Include", - IncludeFor: []CategorizedTag{ + IncludeFor: []model.CategorizedTag{ {Category: "Category", Tag: "Tag"}, }, - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", }, @@ -40,10 +41,10 @@ func TestPrepareSections(t *testing.T) { }, { Text: "Should Exclude", - ExcludeFor: []CategorizedTag{ + ExcludeFor: []model.CategorizedTag{ {Category: "Category", Tag: "Tag"}, }, - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", }, @@ -54,10 +55,10 @@ func TestPrepareSections(t *testing.T) { }, { Text: "AutoAnswer", - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", - AutoAnswerFor: []CategorizedTag{ + AutoAnswerFor: []model.CategorizedTag{ {Category: "Category", Tag: "Tag"}, }, }, @@ -69,6 +70,9 @@ func TestPrepareSections(t *testing.T) { }, }, } + a := Assessment{} + a.With(&assessment) + tagResolver := TagResolver{ cache: map[string]map[string]*model.Tag{ "Category": {"Tag": {Model: model.Model{ID: 1}}}, @@ -77,16 +81,16 @@ func TestPrepareSections(t *testing.T) { tags := NewSet() tags.Add(1) - preparedSections := prepareSections(&tagResolver, tags, sections) - questions := preparedSections[0].Questions + a.Prepare(&tagResolver, tags) + questions := a.Sections[0].Questions g.Expect(len(questions)).To(gomega.Equal(3)) g.Expect(questions[0].Text).To(gomega.Equal("Default")) - g.Expect(questions[0].Answered()).To(gomega.BeFalse()) + g.Expect(a.questionAnswered(&questions[0])).To(gomega.BeFalse()) g.Expect(questions[1].Text).To(gomega.Equal("Should Include")) - g.Expect(questions[1].Answered()).To(gomega.BeFalse()) + g.Expect(a.questionAnswered(&questions[1])).To(gomega.BeFalse()) g.Expect(questions[2].Text).To(gomega.Equal("AutoAnswer")) - g.Expect(questions[2].Answered()).To(gomega.BeTrue()) + g.Expect(a.questionAnswered(&questions[2])).To(gomega.BeTrue()) g.Expect(questions[2].Answers[0].Text).To(gomega.Equal("Answer1")) g.Expect(questions[2].Answers[0].AutoAnswered).To(gomega.BeTrue()) g.Expect(questions[2].Answers[0].Selected).To(gomega.BeTrue()) @@ -95,113 +99,117 @@ func TestPrepareSections(t *testing.T) { func TestAssessmentStarted(t *testing.T) { g := gomega.NewGomegaWithT(t) - assessment := Assessment{ - Sections: []Section{ - { - Questions: []Question{ - { - Text: "S1Q1", - Answers: []Answer{ - { - Text: "A1", - Selected: true, - }, - { - Text: "A2", - }, + assessment := model.Assessment{} + assessment.Sections = []model.Section{ + { + Questions: []model.Question{ + { + Text: "S1Q1", + Answers: []model.Answer{ + { + Text: "A1", + Selected: true, + }, + { + Text: "A2", }, }, - { - Text: "S1Q2", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - }, + }, + { + Text: "S1Q2", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", }, }, }, }, - { - Questions: []Question{ - { - Text: "S2Q1", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - }, + }, + { + Questions: []model.Question{ + { + Text: "S2Q1", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", }, }, }, }, }, } - g.Expect(assessment.Started()).To(gomega.BeTrue()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusStarted)) - assessment.Sections[0].Questions[0].Answers[0].AutoAnswered = true - g.Expect(assessment.Started()).To(gomega.BeFalse()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusEmpty)) + + a := Assessment{} + a.With(&assessment) + g.Expect(a.Started()).To(gomega.BeTrue()) + g.Expect(a.Status()).To(gomega.Equal(StatusStarted)) + a.Sections[0].Questions[0].Answers[0].AutoAnswered = true + g.Expect(a.Started()).To(gomega.BeFalse()) + g.Expect(a.Status()).To(gomega.Equal(StatusEmpty)) } func TestAssessmentComplete(t *testing.T) { g := gomega.NewGomegaWithT(t) - assessment := Assessment{ - Sections: []Section{ - { - Questions: []Question{ - { - Text: "S1Q1", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - }, + assessment := model.Assessment{} + assessment.Sections = []model.Section{ + { + Questions: []model.Question{ + { + Text: "S1Q1", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", }, }, - { - Text: "S1Q2", - Answers: []Answer{ - { - Text: "A1", - Selected: true, - }, - { - Text: "A2", - }, + }, + { + Text: "S1Q2", + Answers: []model.Answer{ + { + Text: "A1", + Selected: true, + }, + { + Text: "A2", }, }, }, }, - { - Questions: []Question{ - { - Text: "S2Q1", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - Selected: true, - AutoAnswered: true, - }, + }, + { + Questions: []model.Question{ + { + Text: "S2Q1", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", + Selected: true, + AutoAnswered: true, }, }, }, }, }, } - g.Expect(assessment.Complete()).To(gomega.BeFalse()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusStarted)) - assessment.Sections[0].Questions[0].Answers[0].Selected = true - g.Expect(assessment.Complete()).To(gomega.BeTrue()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusComplete)) + + a := Assessment{} + a.With(&assessment) + g.Expect(a.Complete()).To(gomega.BeFalse()) + g.Expect(a.Status()).To(gomega.Equal(StatusStarted)) + a.Sections[0].Questions[0].Answers[0].Selected = true + g.Expect(a.Complete()).To(gomega.BeTrue()) + g.Expect(a.Status()).To(gomega.Equal(StatusComplete)) } diff --git a/assessment/membership.go b/assessment/membership.go index b6180c5f3..71c74fad7 100644 --- a/assessment/membership.go +++ b/assessment/membership.go @@ -22,6 +22,7 @@ type MembershipResolver struct { tagSets map[uint]Set archetypeMembers map[uint][]Application membersCached bool + archetypesCached bool } // Applications returns the list of applications that are members of the given archetype. @@ -77,7 +78,7 @@ loop: } func (r *MembershipResolver) cacheArchetypes() (err error) { - if len(r.archetypes) > 0 { + if r.archetypesCached { return } @@ -101,6 +102,7 @@ func (r *MembershipResolver) cacheArchetypes() (err error) { } r.tagSets[a.ID] = set } + r.archetypesCached = true return } diff --git a/assessment/pkg.go b/assessment/pkg.go index 263eca7a0..c0268833e 100644 --- a/assessment/pkg.go +++ b/assessment/pkg.go @@ -1,8 +1,6 @@ package assessment import ( - "encoding/json" - "github.com/konveyor/tackle2-hub/model" ) @@ -93,25 +91,19 @@ func Confidence(assessments []Assessment) (confidence int) { // PrepareForApplication prepares the sections of an assessment by including, excluding, // or auto-answering questions based on a set of tags. func PrepareForApplication(tagResolver *TagResolver, application *model.Application, assessment *model.Assessment) { - sections := []Section{} - _ = json.Unmarshal(assessment.Sections, §ions) - tagSet := NewSet() for _, t := range application.Tags { tagSet.Add(t.ID) } - - assessment.Sections, _ = json.Marshal(prepareSections(tagResolver, tagSet, sections)) - + a := Assessment{} + a.With(assessment) + a.Prepare(tagResolver, tagSet) return } // PrepareForArchetype prepares the sections of an assessment by including, excluding, // or auto-answering questions based on a set of tags. func PrepareForArchetype(tagResolver *TagResolver, archetype *model.Archetype, assessment *model.Assessment) { - sections := []Section{} - _ = json.Unmarshal(assessment.Sections, §ions) - tagSet := NewSet() for _, t := range archetype.CriteriaTags { tagSet.Add(t.ID) @@ -119,63 +111,8 @@ func PrepareForArchetype(tagResolver *TagResolver, archetype *model.Archetype, a for _, t := range archetype.Tags { tagSet.Add(t.ID) } - - assessment.Sections, _ = json.Marshal(prepareSections(tagResolver, tagSet, sections)) - - return -} - -func prepareSections(tagResolver *TagResolver, tags Set, sections []Section) (preparedSections []Section) { - for i := range sections { - s := §ions[i] - includedQuestions := []Question{} - for _, q := range s.Questions { - for j := range q.Answers { - a := &q.Answers[j] - autoAnswerTags := NewSet() - for _, t := range a.AutoAnswerFor { - tag, found := tagResolver.Resolve(t.Category, t.Tag) - if found { - autoAnswerTags.Add(tag.ID) - } - } - if tags.Intersects(autoAnswerTags) { - a.AutoAnswered = true - a.Selected = true - break - } - } - - if len(q.IncludeFor) > 0 { - includeForTags := NewSet() - for _, t := range q.IncludeFor { - tag, found := tagResolver.Resolve(t.Category, t.Tag) - if found { - includeForTags.Add(tag.ID) - } - } - if tags.Intersects(includeForTags) { - includedQuestions = append(includedQuestions, q) - } - continue - } - - if len(q.ExcludeFor) > 0 { - excludeForTags := NewSet() - for _, t := range q.ExcludeFor { - tag, found := tagResolver.Resolve(t.Category, t.Tag) - if found { - excludeForTags.Add(tag.ID) - } - } - if tags.Intersects(excludeForTags) { - continue - } - } - includedQuestions = append(includedQuestions, q) - } - s.Questions = includedQuestions - } - preparedSections = sections + a := Assessment{} + a.With(assessment) + a.Prepare(tagResolver, tagSet) return } diff --git a/assessment/tag.go b/assessment/tag.go index 930ed9b31..39eb44658 100644 --- a/assessment/tag.go +++ b/assessment/tag.go @@ -30,12 +30,10 @@ func (r *TagResolver) Resolve(category string, tag string) (t *model.Tag, found // Assessment returns all the Tag models that should be applied from the assessment. func (r *TagResolver) Assessment(assessment Assessment) (tags []model.Tag) { - for _, s := range assessment.Sections { - for _, t := range s.Tags() { - tag, found := r.Resolve(t.Category, t.Tag) - if found { - tags = append(tags, *tag) - } + for _, t := range assessment.Tags() { + tag, found := r.Resolve(t.Category, t.Tag) + if found { + tags = append(tags, *tag) } } return diff --git a/binding/application.go b/binding/application.go index b8b02fd62..ec7406f43 100644 --- a/binding/application.go +++ b/binding/application.go @@ -229,8 +229,8 @@ func (h *AppFacts) Source(source string) { } // List facts. -func (h *AppFacts) List() (facts api.FactMap, err error) { - facts = api.FactMap{} +func (h *AppFacts) List() (facts api.Map, err error) { + facts = api.Map{} key := api.FactKey("") key.Qualify(h.source) path := Path(api.ApplicationFactsRoot).Inject(Params{api.ID: h.appId, api.Key: key}) @@ -278,7 +278,7 @@ func (h *AppFacts) Delete(name string) (err error) { } // Replace facts. -func (h *AppFacts) Replace(facts api.FactMap) (err error) { +func (h *AppFacts) Replace(facts api.Map) (err error) { key := api.FactKey("") key.Qualify(h.source) path := Path(api.ApplicationFactsRoot).Inject(Params{api.ID: h.appId, api.Key: key}) diff --git a/hack/cmd/addon/main.go b/hack/cmd/addon/main.go index 3b5ddb5d3..88cb13333 100644 --- a/hack/cmd/addon/main.go +++ b/hack/cmd/addon/main.go @@ -9,16 +9,17 @@ package main import ( "bytes" "errors" - hub "github.com/konveyor/tackle2-hub/addon" - "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/nas" - "k8s.io/apimachinery/pkg/util/rand" "os" "os/exec" pathlib "path" "strconv" "strings" "time" + + hub "github.com/konveyor/tackle2-hub/addon" + "github.com/konveyor/tackle2-hub/api" + "github.com/konveyor/tackle2-hub/nas" + "k8s.io/apimachinery/pkg/util/rand" ) var ( @@ -32,7 +33,6 @@ const ( TmpDir = "/tmp/list" ) -// // main func main() { addon.Run(func() (err error) { @@ -76,7 +76,7 @@ func main() { // // Replace facts. err = facts.Replace( - api.FactMap{ + api.Map{ "Listed": true, "Color": "blue", "Length": 100, @@ -100,7 +100,6 @@ func main() { }) } -// // listDir builds and populates the bucket. func listDir(d *Data, application *api.Application, paths []string) (err error) { // @@ -178,7 +177,6 @@ func listDir(d *Data, application *api.Application, paths []string) (err error) return } -// // playWithBucket func playWithBucket(bucket *hub.BucketContent) (err error) { tmpDir := tmpDir() @@ -215,7 +213,6 @@ func playWithBucket(bucket *hub.BucketContent) (err error) { return } -// // Build index.html func buildIndex(output string) (err error) { addon.Activity("Building index.") @@ -247,7 +244,6 @@ func buildIndex(output string) (err error) { return } -// // find files. func find(path string, max int) (paths []string, err error) { Log.Info("Listing.", "path", path) @@ -279,7 +275,6 @@ func find(path string, max int) (paths []string, err error) { return } -// // Play with files. func playWithFiles() (err error) { f, err := addon.File.Put("/etc/hosts") @@ -301,7 +296,6 @@ func playWithFiles() (err error) { return } -// // addTags ensure tags created and associated with application. // Ensure tag exists and associated with the application. func addTags(application *api.Application, source string, names ...string) (err error) { @@ -337,7 +331,6 @@ func addTags(application *api.Application, source string, names ...string) (err return } -// // replaceTags replaces current set of tags for the source with a new set. // Ensures desired tags exist before replacing. func replaceTags(application *api.Application, source string, names ...string) (err error) { @@ -395,7 +388,6 @@ func tmpDir() (p string) { return } -// // Data Addon input. type Data struct { // Path to be listed. diff --git a/importer/manager.go b/importer/manager.go index b11f45e4b..540214e2c 100644 --- a/importer/manager.go +++ b/importer/manager.go @@ -2,7 +2,6 @@ package importer import ( "context" - "encoding/json" "fmt" "regexp" @@ -136,7 +135,7 @@ func (m *Manager) createApplication(imp *model.Import) (ok bool) { return } - repository := api.Repository{ + repository := model.Repository{ Kind: imp.RepositoryKind, URL: imp.RepositoryURL, Branch: imp.RepositoryBranch, @@ -148,8 +147,7 @@ func (m *Manager) createApplication(imp *model.Import) (ok bool) { repository.Kind = "git" } - app.Repository, _ = json.Marshal(repository) - + app.Repository = repository // Validate Binary-related fields (allow all 3 empty or present) if imp.BinaryGroup != "" || imp.BinaryArtifact != "" || imp.BinaryVersion != "" { if imp.BinaryGroup == "" || imp.BinaryArtifact == "" || imp.BinaryVersion == "" { diff --git a/migration/json/fields.go b/migration/json/fields.go index 926aa40b4..b79cd6529 100644 --- a/migration/json/fields.go +++ b/migration/json/fields.go @@ -1,6 +1,8 @@ package json -import "gopkg.in/yaml.v2" +import ( + "gopkg.in/yaml.v2" +) // Ref represents a FK. type Ref struct { diff --git a/migration/migrate.go b/migration/migrate.go index 4c8ec8b71..43f582a79 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -1,7 +1,6 @@ package migration import ( - "encoding/json" "errors" "os" "path" @@ -44,12 +43,9 @@ func Migrate(migrations []Migration) (err error) { } var v Version - if setting.Value != nil { - err = json.Unmarshal(setting.Value, &v) - if err != nil { - err = liberr.Wrap(err) - return - } + err = setting.As(&v) + if err != nil { + return } var start = v.Version if start != 0 && start < MinimumVersion { @@ -112,9 +108,7 @@ func Migrate(migrations []Migration) (err error) { // Set the version record. func setVersion(db *gorm.DB, version int) (err error) { setting := &model.Setting{Key: VersionKey} - v := Version{Version: version} - value, _ := json.Marshal(v) - setting.Value = value + setting.Value = Version{Version: version} result := db.Where("key", VersionKey).Updates(setting) if result.Error != nil { err = liberr.Wrap(result.Error) diff --git a/migration/migrate_test.go b/migration/migrate_test.go index 943a85a1a..a82e59fa3 100644 --- a/migration/migrate_test.go +++ b/migration/migrate_test.go @@ -1,7 +1,6 @@ package migration import ( - "encoding/json" "os" "testing" @@ -132,7 +131,7 @@ func expectVersion(g *gomega.GomegaWithT, version int) { result := db.Find(setting, "key", VersionKey) g.Expect(result.Error).To(gomega.BeNil()) var v Version - _ = json.Unmarshal(setting.Value, &v) + _ = setting.As(&v) g.Expect(v.Version).To(gomega.Equal(version)) _ = database.Close(db) } diff --git a/migration/v14/model/analysis.go b/migration/v14/model/analysis.go index 4b61e1f40..bc33da893 100644 --- a/migration/v14/model/analysis.go +++ b/migration/v14/model/analysis.go @@ -1,6 +1,9 @@ package model -import "gorm.io/gorm" +import ( + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) // Analysis report. type Analysis struct { @@ -8,7 +11,7 @@ type Analysis struct { Effort int Commit string Archived bool - Summary JSON `gorm:"type:json"` + Summary []ArchivedIssue `gorm:"type:json;serializer:json"` Issues []Issue `gorm:"constraint:OnDelete:CASCADE"` Dependencies []TechDependency `gorm:"constraint:OnDelete:CASCADE"` ApplicationID uint `gorm:"index;not null"` @@ -23,8 +26,8 @@ type TechDependency struct { Version string `gorm:"uniqueIndex:depA"` SHA string `gorm:"uniqueIndex:depA"` Indirect bool - Labels JSON `gorm:"type:json"` - AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` + Labels []string `gorm:"type:json;serializer:json"` + AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` Analysis *Analysis } @@ -37,9 +40,9 @@ type Issue struct { Description string Category string `gorm:"index;not null"` Incidents []Incident `gorm:"foreignKey:IssueID;constraint:OnDelete:CASCADE"` - Links JSON `gorm:"type:json"` - Facts JSON `gorm:"type:json"` - Labels JSON `gorm:"type:json"` + Links []Link `gorm:"type:json;serializer:json"` + Facts json.Map `gorm:"type:json;serializer:json"` + Labels []string `gorm:"type:json;serializer:json"` Effort int `gorm:"index;not null"` AnalysisID uint `gorm:"index;uniqueIndex:issueA;not null"` Analysis *Analysis @@ -52,28 +55,11 @@ type Incident struct { Line int Message string CodeSnip string - Facts JSON `gorm:"type:json"` - IssueID uint `gorm:"index;not null"` + Facts json.Map `gorm:"type:json;serializer:json"` + IssueID uint `gorm:"index;not null"` Issue *Issue } -// Link URL link. -type Link struct { - URL string `json:"url"` - Title string `json:"title,omitempty"` -} - -// ArchivedIssue resource created when issues are archived. -type ArchivedIssue struct { - RuleSet string `json:"ruleSet"` - Rule string `json:"rule"` - Name string `json:"name,omitempty" yaml:",omitempty"` - Description string `json:"description,omitempty" yaml:",omitempty"` - Category string `json:"category"` - Effort int `json:"effort"` - Incidents int `json:"incidents"` -} - // RuleSet - Analysis ruleset. type RuleSet struct { Model @@ -81,8 +67,8 @@ type RuleSet struct { Kind string Name string `gorm:"uniqueIndex;not null"` Description string - Repository JSON `gorm:"type:json"` - IdentityID *uint `gorm:"index"` + Repository Repository `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` Identity *Identity Rules []Rule `gorm:"constraint:OnDelete:CASCADE"` DependsOn []RuleSet `gorm:"many2many:RuleSetDependencies;constraint:OnDelete:CASCADE"` @@ -130,8 +116,8 @@ type Rule struct { Model Name string Description string - Labels JSON `gorm:"type:json"` - RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` + Labels []string `gorm:"type:json;serializer:json"` + RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` RuleSet *RuleSet FileID *uint `gorm:"uniqueIndex:RuleA" ref:"file"` File *File @@ -145,8 +131,8 @@ type Target struct { Description string Provider string Choice bool - Labels JSON `gorm:"type:json"` - ImageID uint `gorm:"index" ref:"file"` + Labels []TargetLabel `gorm:"type:json;serializer:json"` + ImageID uint `gorm:"index" ref:"file"` Image *File RuleSetID *uint `gorm:"index"` RuleSet *RuleSet @@ -155,3 +141,30 @@ type Target struct { func (r *Target) Builtin() bool { return r.UUID != nil } + +// +// JSON Fields. +// + +// ArchivedIssue resource created when issues are archived. +type ArchivedIssue struct { + RuleSet string `json:"ruleSet"` + Rule string `json:"rule"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Description string `json:"description,omitempty" yaml:",omitempty"` + Category string `json:"category"` + Effort int `json:"effort"` + Incidents int `json:"incidents"` +} + +// Link URL link. +type Link struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` +} + +// TargetLabel - label format specific to Targets +type TargetLabel struct { + Name string `json:"name"` + Label string `json:"label"` +} diff --git a/migration/v14/model/application.go b/migration/v14/model/application.go index 49a9a959a..cb363b78d 100644 --- a/migration/v14/model/application.go +++ b/migration/v14/model/application.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/konveyor/tackle2-hub/migration/json" "gorm.io/gorm" ) @@ -13,8 +14,8 @@ type Application struct { BucketOwner Name string `gorm:"index;unique;not null"` Description string - Review *Review `gorm:"constraint:OnDelete:CASCADE"` - Repository JSON `gorm:"type:json"` + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Repository Repository `gorm:"type:json;serializer:json"` Binary string Facts []Fact `gorm:"constraint:OnDelete:CASCADE"` Comments string @@ -37,7 +38,7 @@ type Fact struct { ApplicationID uint `gorm:"<-:create;primaryKey"` Key string `gorm:"<-:create;primaryKey"` Source string `gorm:"<-:create;primaryKey;not null"` - Value JSON `gorm:"type:json;not null"` + Value any `gorm:"type:json;not null;serializer:json"` Application *Application } @@ -195,7 +196,7 @@ type Ticket struct { // Parent resource that this ticket should belong to in the tracker. (e.g. Jira project) Parent string `gorm:"not null"` // Custom fields to send to the tracker when creating the ticket - Fields JSON `gorm:"type:json"` + Fields json.Map `gorm:"type:json;serializer:json"` // Whether the last attempt to do something with the ticket reported an error Error bool // Error message, if any @@ -297,6 +298,11 @@ type ImportTag struct { Import *Import } +// +// JSON Fields. +// + +// Repository represents an SCM repository. type Repository struct { Kind string `json:"kind"` URL string `json:"url"` diff --git a/migration/v14/model/assessment.go b/migration/v14/model/assessment.go index 3a734e86e..0b51e714d 100644 --- a/migration/v14/model/assessment.go +++ b/migration/v14/model/assessment.go @@ -6,9 +6,9 @@ type Questionnaire struct { Name string `gorm:"unique"` Description string Required bool - Sections JSON `gorm:"type:json"` - Thresholds JSON `gorm:"type:json"` - RiskMessages JSON `gorm:"type:json"` + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` } @@ -25,9 +25,9 @@ type Assessment struct { Archetype *Archetype QuestionnaireID uint `gorm:"uniqueIndex:AssessmentA;uniqueIndex:AssessmentB"` Questionnaire Questionnaire - Sections JSON `gorm:"type:json"` - Thresholds JSON `gorm:"type:json"` - RiskMessages JSON `gorm:"type:json"` + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` Stakeholders []Stakeholder `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` StakeholderGroups []StakeholderGroup `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` } @@ -44,3 +44,59 @@ type Review struct { ArchetypeID *uint `gorm:"uniqueIndex"` Archetype *Archetype } + +// +// JSON Fields. +// + +// Section represents a group of questions in a questionnaire. +type Section struct { + Order uint `json:"order" yaml:"order"` + Name string `json:"name" yaml:"name"` + Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` + Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +} + +// Question represents a question in a questionnaire. +type Question struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Explanation string `json:"explanation" yaml:"explanation"` + IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` + ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` + Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` +} + +// Answer represents an answer to a question in a questionnaire. +type Answer struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` + Rationale string `json:"rationale" yaml:"rationale"` + Mitigation string `json:"mitigation" yaml:"mitigation"` + ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` + AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` + Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` + AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` +} + +// CategorizedTag represents a human-readable pair of category and tag. +type CategorizedTag struct { + Category string `json:"category" yaml:"category"` + Tag string `json:"tag" yaml:"tag"` +} + +// RiskMessages contains messages to display for each risk level. +type RiskMessages struct { + Red string `json:"red" yaml:"red"` + Yellow string `json:"yellow" yaml:"yellow"` + Green string `json:"green" yaml:"green"` + Unknown string `json:"unknown" yaml:"unknown"` +} + +// Thresholds contains the threshold values for determining risk for the questionnaire. +type Thresholds struct { + Red uint `json:"red" yaml:"red"` + Yellow uint `json:"yellow" yaml:"yellow"` + Unknown uint `json:"unknown" yaml:"unknown"` +} diff --git a/migration/v14/model/core.go b/migration/v14/model/core.go index 06c02bd6d..a5d6a5ab1 100644 --- a/migration/v14/model/core.go +++ b/migration/v14/model/core.go @@ -23,22 +23,16 @@ type Model struct { type Setting struct { Model Key string `gorm:"<-:create;uniqueIndex"` - Value JSON `gorm:"type:json"` + Value any `gorm:"type:json;serializer:json"` } -// With updates the value of the Setting with the json representation -// of the `value` parameter. -func (r *Setting) With(value any) (err error) { - r.Value, err = json.Marshal(value) +// As unmarshalls the value of the Setting into the `ptr` parameter. +func (r *Setting) As(ptr any) (err error) { + bytes, err := json.Marshal(r.Value) if err != nil { err = liberr.Wrap(err) } - return -} - -// As unmarshalls the value of the Setting into the `ptr` parameter. -func (r *Setting) As(ptr any) (err error) { - err = json.Unmarshal(r.Value, ptr) + err = json.Unmarshal(bytes, ptr) if err != nil { err = liberr.Wrap(err) } @@ -146,26 +140,6 @@ func (m *Task) BeforeCreate(db *gorm.DB) (err error) { return } -// TaskError used in Task.Errors. -type TaskError struct { - Severity string `json:"severity"` - Description string `json:"description"` -} - -// TaskPolicy scheduling policy. -type TaskPolicy struct { - Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` - PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` - PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` -} - -// Attachment file attachment. -type Attachment struct { - ID uint `json:"id" binding:"required"` - Name string `json:"name,omitempty" yaml:",omitempty"` - Activity int `json:"activity,omitempty" yaml:",omitempty"` -} - type TaskReport struct { Model Status string @@ -202,8 +176,8 @@ type Proxy struct { Kind string `gorm:"uniqueIndex"` Host string `gorm:"not null"` Port int - Excluded JSON `gorm:"type:json"` - IdentityID *uint `gorm:"index"` + Excluded []string `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` Identity *Identity } @@ -289,6 +263,19 @@ func (r *Identity) Decrypt() (err error) { // JSON Fields. // +// Attachment file attachment. +type Attachment struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Activity int `json:"activity,omitempty" yaml:",omitempty"` +} + +// TaskError used in Task.Errors. +type TaskError struct { + Severity string `json:"severity"` + Description string `json:"description"` +} + // TaskEvent task event. type TaskEvent struct { Kind string `json:"kind"` @@ -297,6 +284,13 @@ type TaskEvent struct { Last time.Time `json:"last"` } +// TaskPolicy scheduling policy. +type TaskPolicy struct { + Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` + PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` + PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` +} + // TTL time-to-live. type TTL struct { Created int `json:"created,omitempty" yaml:",omitempty"` diff --git a/model/pkg.go b/model/pkg.go index 2cb906e60..5cf00cd5a 100644 --- a/model/pkg.go +++ b/model/pkg.go @@ -16,7 +16,6 @@ type Assessment = model.Assessment type TechDependency = model.TechDependency type Incident = model.Incident type Analysis = model.Analysis -type ArchivedIssue = model.ArchivedIssue type Issue = model.Issue type Bucket = model.Bucket type BucketOwner = model.BucketOwner @@ -47,16 +46,27 @@ type TaskReport = model.TaskReport type Ticket = model.Ticket type Tracker = model.Tracker -type TTL = model.TTL +// JSON fields type Ref = json.Ref type Map = json.Map type Data = json.Data - +type ArchivedIssue = model.ArchivedIssue +type Attachment = model.Attachment +type Link = model.Link +type Repository = model.Repository +type TargetLabel = model.TargetLabel type TaskError = model.TaskError type TaskEvent = model.TaskEvent type TaskPolicy = model.TaskPolicy -type Attachment = model.Attachment -type Repository = model.Repository +type TTL = model.TTL + +// Assessment JSON fields +type Section = model.Section +type Question = model.Question +type Answer = model.Answer +type Thresholds = model.Thresholds +type RiskMessages = model.RiskMessages +type CategorizedTag = model.CategorizedTag // Join tables type ApplicationTag = model.ApplicationTag diff --git a/seed/questionnaire.go b/seed/questionnaire.go index 87e419817..134e63c7f 100644 --- a/seed/questionnaire.go +++ b/seed/questionnaire.go @@ -77,9 +77,14 @@ func (r *Questionnaire) Apply(db *gorm.DB) (err error) { questionnaire.Name = q.Name questionnaire.UUID = &q.UUID questionnaire.Description = q.Description - questionnaire.Sections, _ = json.Marshal(q.Sections) - questionnaire.RiskMessages, _ = json.Marshal(q.RiskMessages) - questionnaire.Thresholds, _ = json.Marshal(q.Thresholds) + questionnaire.RiskMessages = model.RiskMessages(q.RiskMessages) + questionnaire.Thresholds = model.Thresholds(q.Thresholds) + bytes, jErr := json.Marshal(q.Sections) + if jErr != nil { + err = liberr.Wrap(jErr) + return + } + err = json.Unmarshal(bytes, &questionnaire.Sections) result := db.Save(&questionnaire) if result.Error != nil { err = liberr.Wrap(result.Error) diff --git a/seed/ruleset.go b/seed/ruleset.go index 3354b6aea..f746d1f08 100644 --- a/seed/ruleset.go +++ b/seed/ruleset.go @@ -128,14 +128,13 @@ func (r *RuleSet) applyRules(db *gorm.DB, ruleSet *model.RuleSet, rs libseed.Rul return } for _, rl := range rs.Rules { - labels, _ := json.Marshal(rl.Labels()) f, fErr := file(db, rl.Path) if fErr != nil { err = liberr.Wrap(fErr) return } rule := model.Rule{ - Labels: labels, + Labels: rl.Labels(), RuleSetID: ruleSet.ID, FileID: &f.ID, } diff --git a/seed/seed.go b/seed/seed.go index 1ae605a9d..a674e1c73 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -1,7 +1,6 @@ package seed import ( - "encoding/json" "fmt" "strings" @@ -75,12 +74,10 @@ func compareChecksum(db *gorm.DB, checksum []byte) (match bool, err error) { return } var seededChecksum string - if setting.Value != nil { - err = json.Unmarshal(setting.Value, &seededChecksum) - if err != nil { - err = liberr.Wrap(err) - return - } + err = setting.As(&seededChecksum) + if err != nil { + err = liberr.Wrap(err) + return } match = seededChecksum == fmt.Sprintf("%x", checksum) @@ -90,8 +87,7 @@ func compareChecksum(db *gorm.DB, checksum []byte) (match bool, err error) { // saveChecksum saves the seed checksum to the setting specified by SeedKey. func saveChecksum(db *gorm.DB, checksum []byte) (err error) { setting := &model.Setting{Key: SeedKey} - value, _ := json.Marshal(fmt.Sprintf("%x", checksum)) - setting.Value = value + setting.Value = fmt.Sprintf("%x", checksum) result := db.Where("key", SeedKey).Updates(setting) if result.Error != nil { err = liberr.Wrap(result.Error) @@ -110,12 +106,10 @@ func migrationVersion(db *gorm.DB) (version uint, err error) { } var v migration.Version - if setting.Value != nil { - err = json.Unmarshal(setting.Value, &v) - if err != nil { - err = liberr.Wrap(err) - return - } + err = setting.As(&v) + if err != nil { + err = liberr.Wrap(err) + return } version = uint(v.Version) diff --git a/seed/target.go b/seed/target.go index 6e2888091..9f665ee8d 100644 --- a/seed/target.go +++ b/seed/target.go @@ -2,7 +2,6 @@ package seed import ( "container/list" - "encoding/json" "errors" "fmt" @@ -80,7 +79,6 @@ func (r *Target) Apply(db *gorm.DB) (err error) { err = liberr.Wrap(fErr) return } - labels, _ := json.Marshal(t.Labels) target.UUID = &t.UUID target.Name = t.Name @@ -88,7 +86,9 @@ func (r *Target) Apply(db *gorm.DB) (err error) { target.Provider = t.Provider target.Choice = t.Choice target.ImageID = f.ID - target.Labels = labels + for _, l := range t.Labels { + target.Labels = append(target.Labels, model.TargetLabel(l)) + } result := db.Save(&target) if result.Error != nil { err = liberr.Wrap(result.Error) @@ -126,7 +126,7 @@ func (r *Target) reorder(db *gorm.DB, seedIds []uint) (err error) { } userOrder := []uint{} _ = s.As(&userOrder) - _ = s.With(merge(userOrder, seedIds, targetIds)) + s.Value = merge(userOrder, seedIds, targetIds) result = db.Where("key", UITargetOrder).Updates(s) if result.Error != nil { diff --git a/test/api/application/facts_test.go b/test/api/application/facts_test.go index d6d487bf1..f294e9304 100644 --- a/test/api/application/facts_test.go +++ b/test/api/application/facts_test.go @@ -108,7 +108,7 @@ func TestApplicationFactsList(t *testing.T) { factsPathSuffix := []string{"facts/test:", "facts/test:/"} for _, pathSuffix := range factsPathSuffix { t.Run(fmt.Sprintf("Fact list application %s with %s", application.Name, pathSuffix), func(t *testing.T) { - got := api.FactMap{} + got := api.Map{} err := Client.Get(fmt.Sprintf("%s/%s", binding.Path(api.ApplicationRoot).Inject(binding.Params{api.ID: application.ID}), pathSuffix), &got) if err != nil { t.Errorf("Get list error: %v", err.Error()) diff --git a/test/api/assessment/samples.go b/test/api/assessment/samples.go index 1a1ec5493..ccfb779e1 100644 --- a/test/api/assessment/samples.go +++ b/test/api/assessment/samples.go @@ -2,7 +2,7 @@ package assessment import ( "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/assessment" + "github.com/konveyor/tackle2-hub/model" "github.com/konveyor/tackle2-hub/test/api/application" "github.com/konveyor/tackle2-hub/test/api/questionnaire" ) @@ -17,28 +17,28 @@ var ( Questionnaire: api.Ref{ Name: questionnaire.Questionnaire1.Name, }, - Sections: []assessment.Section{ + Sections: []api.Section{ { - Order: uint2ptr(1), + Order: 1, Name: "Section 1", - Questions: []assessment.Question{ + Questions: []model.Question{ { - Order: uint2ptr(1), + Order: 1, Text: "What is your favorite color?", Explanation: "Please tell us your favorite color.", - Answers: []assessment.Answer{ + Answers: []model.Answer{ { - Order: uint2ptr(1), + Order: 1, Text: "Red", Risk: "red", }, { - Order: uint2ptr(2), + Order: 2, Text: "Green", Risk: "green", }, { - Order: uint2ptr(3), + Order: 3, Text: "Blue", Risk: "yellow", Selected: true, diff --git a/test/api/questionnaire/samples.go b/test/api/questionnaire/samples.go index 988dbf0c9..b1792f187 100644 --- a/test/api/questionnaire/samples.go +++ b/test/api/questionnaire/samples.go @@ -2,7 +2,7 @@ package questionnaire import ( "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/assessment" + "github.com/konveyor/tackle2-hub/model" ) // Set of valid resources for tests and reuse. @@ -11,30 +11,30 @@ var ( Name: "Questionnaire1", Description: "Questionnaire minimal sample 1", Required: true, - Thresholds: assessment.Thresholds{}, - RiskMessages: assessment.RiskMessages{}, - Sections: []assessment.Section{ + Thresholds: api.Thresholds{}, + RiskMessages: api.RiskMessages{}, + Sections: []api.Section{ { - Order: uint2ptr(1), + Order: 1, Name: "Section 1", - Questions: []assessment.Question{ + Questions: []model.Question{ { - Order: uint2ptr(1), + Order: 1, Text: "What is your favorite color?", Explanation: "Please tell us your favorite color.", - Answers: []assessment.Answer{ + Answers: []model.Answer{ { - Order: uint2ptr(1), + Order: 1, Text: "Red", Risk: "red", }, { - Order: uint2ptr(2), + Order: 2, Text: "Green", Risk: "green", }, { - Order: uint2ptr(3), + Order: 3, Text: "Blue", Risk: "yellow", Selected: true, @@ -47,7 +47,3 @@ var ( } Samples = []api.Questionnaire{Questionnaire1} ) - -func uint2ptr(u uint) *uint { - return &u -} diff --git a/trigger/application.go b/trigger/application.go index f40c8d297..607f368bf 100644 --- a/trigger/application.go +++ b/trigger/application.go @@ -23,7 +23,7 @@ func (r *Application) Updated(m *model.Application) (err error) { if !Settings.Discovery.Enabled { return } - if len(m.Repository) == 0 || string(m.Repository) == "null" { + if m.Repository == (model.Repository{}) { return } kinds, err := r.FindTasks(Settings.Discovery.Label) From e50b49a4c555d003d82464cb28bb83b70d51798a Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Mon, 19 Aug 2024 17:26:26 -0500 Subject: [PATCH 05/15] :sparkles: DB reader/writer mutex. (#711) Support multiple DB connections. Using a custom sqlite driver that is a wrapper around the [_standard_](https://pkg.go.dev/github.com/mattn/go-sqlite3) driver but with a mutex around driver.TX and driver.Stmt as needed. --------- Signed-off-by: Jeff Ortel --- database/driver.go | 240 +++++++++++++++++++++++++++++++++++++++++++++ database/pkg.go | 14 +-- 2 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 database/driver.go diff --git a/database/driver.go b/database/driver.go new file mode 100644 index 000000000..708228530 --- /dev/null +++ b/database/driver.go @@ -0,0 +1,240 @@ +package database + +import ( + "context" + "database/sql/driver" + "strings" + "sync" + + "github.com/mattn/go-sqlite3" +) + +type Driver struct { + mutex sync.Mutex + wrapped driver.Driver + dsn string +} + +func (d *Driver) Open(dsn string) (conn driver.Conn, err error) { + d.wrapped = &sqlite3.SQLiteDriver{} + conn, err = d.wrapped.Open(dsn) + if err != nil { + return + } + conn = &Conn{ + mutex: &d.mutex, + wrapped: conn, + } + return +} + +func (d *Driver) OpenConnector(dsn string) (dc driver.Connector, err error) { + d.dsn = dsn + dc = d + return +} + +func (d *Driver) Connect(context.Context) (conn driver.Conn, err error) { + conn, err = d.Open(d.dsn) + return +} + +func (d *Driver) Driver() driver.Driver { + return d +} + +type Conn struct { + mutex *sync.Mutex + wrapped driver.Conn + tx driver.Tx +} + +func (c *Conn) Ping(ctx context.Context) (err error) { + if p, cast := c.wrapped.(driver.Pinger); cast { + err = p.Ping(ctx) + } + return +} + +func (c *Conn) ResetSession(ctx context.Context) (err error) { + if p, cast := c.wrapped.(driver.SessionResetter); cast { + err = p.ResetSession(ctx) + } + return +} +func (c *Conn) IsValid() (b bool) { + if p, cast := c.wrapped.(driver.Validator); cast { + b = p.IsValid() + } + return +} + +func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Rows, err error) { + if c.tx == nil { + c.mutex.Lock() + defer c.mutex.Unlock() + } + if p, cast := c.wrapped.(driver.QueryerContext); cast { + r, err = p.QueryContext(ctx, query, args) + } + return +} + +func (c *Conn) PrepareContext(ctx context.Context, query string) (s driver.Stmt, err error) { + if p, cast := c.wrapped.(driver.ConnPrepareContext); cast { + s, err = p.PrepareContext(ctx, query) + } + if err != nil { + return + } + stmtLocked := c.stmtLocked(query) + s = &Stmt{ + mutex: c.mutex, + locked: stmtLocked, + wrapped: s, + } + if stmtLocked { + c.mutex.Lock() + } + return +} + +func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Result, err error) { + if c.tx == nil { + c.mutex.Lock() + defer c.mutex.Unlock() + } + if p, cast := c.wrapped.(driver.ExecerContext); cast { + r, err = p.ExecContext(ctx, query, args) + } + return +} + +func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { + if p, cast := c.wrapped.(driver.ConnBeginTx); cast { + tx, err = p.BeginTx(ctx, opts) + } else { + tx, err = c.wrapped.Begin() + } + if err != nil { + return + } + tx = &Tx{ + mutex: c.mutex, + wrapped: tx, + } + c.tx = tx + c.mutex.Lock() + return +} + +func (c *Conn) Prepare(query string) (s driver.Stmt, err error) { + s, err = c.wrapped.Prepare(query) + if err != nil { + return + } + stmtLocked := c.stmtLocked(query) + s = &Stmt{ + mutex: c.mutex, + locked: stmtLocked, + wrapped: s, + } + if stmtLocked { + c.mutex.Lock() + } + return +} + +func (c *Conn) Close() (err error) { + err = c.wrapped.Close() + return +} + +func (c *Conn) Begin() (tx driver.Tx, err error) { + tx, err = c.wrapped.Begin() + if err != nil { + return + } + tx = &Tx{ + mutex: c.mutex, + wrapped: tx, + } + c.tx = tx + c.mutex.Lock() + return +} + +func (c *Conn) stmtLocked(query string) (matched bool) { + if c.tx != nil || query == "" { + return + } + query = strings.ToUpper(query) + action := strings.Fields(query)[0] + action = strings.ToUpper(action) + matched = action == "CREATE" || + action == "INSERT" || + action == "UPDATE" || + action == "DELETE" + return +} + +type Tx struct { + mutex *sync.Mutex + wrapped driver.Tx +} + +func (tx *Tx) Commit() (err error) { + defer func() { + tx.mutex.Unlock() + }() + err = tx.wrapped.Commit() + return +} +func (tx *Tx) Rollback() (err error) { + defer func() { + tx.mutex.Unlock() + }() + err = tx.wrapped.Rollback() + return +} + +type Stmt struct { + mutex *sync.Mutex + wrapped driver.Stmt + locked bool +} + +func (s *Stmt) Close() (err error) { + if s.locked { + s.mutex.Unlock() + } + err = s.wrapped.Close() + return +} +func (s *Stmt) NumInput() (n int) { + n = s.wrapped.NumInput() + return +} +func (s *Stmt) Exec(args []driver.Value) (r driver.Result, err error) { + r, err = s.wrapped.Exec(args) + return +} + +func (s *Stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (r driver.Result, err error) { + if p, cast := s.wrapped.(driver.StmtExecContext); cast { + r, err = p.ExecContext(ctx, args) + } + return +} + +func (s *Stmt) Query(args []driver.Value) (r driver.Rows, err error) { + r, err = s.wrapped.Query(args) + return +} + +func (s *Stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (r driver.Rows, err error) { + if p, cast := s.wrapped.(driver.StmtQueryContext); cast { + r, err = p.QueryContext(ctx, args) + } + return +} diff --git a/database/pkg.go b/database/pkg.go index 9f0b29cb2..3e9adfdae 100644 --- a/database/pkg.go +++ b/database/pkg.go @@ -23,6 +23,10 @@ const ( FKsOff = "&_foreign_keys=no" ) +func init() { + sql.Register("sqlite3x", &Driver{}) +} + // Open and automigrate the DB. func Open(enforceFKs bool) (db *gorm.DB, err error) { connStr := fmt.Sprintf(ConnectionString, Settings.DB.Path) @@ -31,8 +35,10 @@ func Open(enforceFKs bool) (db *gorm.DB, err error) { } else { connStr += FKsOff } + dialector := sqlite.Open(connStr).(*sqlite.Dialector) + dialector.DriverName = "sqlite3x" db, err = gorm.Open( - sqlite.Open(connStr), + dialector, &gorm.Config{ PrepareStmt: true, CreateBatchSize: 500, @@ -45,12 +51,6 @@ func Open(enforceFKs bool) (db *gorm.DB, err error) { err = liberr.Wrap(err) return } - sqlDB, err := db.DB() - if err != nil { - err = liberr.Wrap(err) - return - } - sqlDB.SetMaxOpenConns(1) err = db.AutoMigrate(model.Setting{}) if err != nil { err = liberr.Wrap(err) From 69bff4b640b7b58cde9ba6818a86de5943aa10a3 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Mon, 19 Aug 2024 18:03:58 -0500 Subject: [PATCH 06/15] :sparkles: pod retention. (#737) Support pod retention settings. The current policy is to delete pods as soon as completed (succeed or failed). Tackle users and support are used to troubleshooting by `oc debug` of the task pods. To support this, the task manager can terminate containers in pods as needed and defer to the reaper to delete the pods. This would be controlled new settings. By default succeeded tasks would be retained their pods for 1 minute; failed tasks for 72 hours. In all cases, failure to terminate running container will fallback to deleting the pod immediately. The retention is best effort. Running containers are terminated by `kill -p 1` This will only work for linux containers. --------- Signed-off-by: Jeff Ortel --- go.mod | 1 + go.sum | 5 ++ reaper/task.go | 49 +++++++----- settings/hub.go | 22 ++++++ task/manager.go | 199 +++++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 238 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 75022634b..6ac197c8d 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index 88f63ca3c..8478cb168 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -29,6 +30,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -121,6 +123,7 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -162,6 +165,8 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/reaper/task.go b/reaper/task.go index 3f73432f1..6f64ec41c 100644 --- a/reaper/task.go +++ b/reaper/task.go @@ -44,7 +44,7 @@ type TaskReaper struct { // - Pod is deleted after the defined period. func (r *TaskReaper) Run() { Log.V(1).Info("Reaping tasks.") - list := []model.Task{} + list := []task.Task{} result := r.DB.Find( &list, "state IN ?", @@ -52,6 +52,7 @@ func (r *TaskReaper) Run() { task.Created, task.Succeeded, task.Failed, + task.Canceled, }) Log.Error(result.Error, "") if result.Error != nil { @@ -108,6 +109,10 @@ func (r *TaskReaper) Run() { r.release(m) } } + d := time.Duration(Settings.Hub.Task.Pod.Retention.Succeeded) * Unit + if time.Since(mark) > d { + r.podDelete(m) + } case task.Failed: mark := m.CreateTime if m.Terminated != nil { @@ -124,23 +129,17 @@ func (r *TaskReaper) Run() { r.release(m) } } + d := time.Duration(Settings.Hub.Task.Pod.Retention.Failed) * Unit + if time.Since(mark) > d { + r.podDelete(m) + } } } } -// release resources. -func (r *TaskReaper) release(m *model.Task) { +// release bucket and file resources. +func (r *TaskReaper) release(m *task.Task) { nChanged := 0 - if m.Pod != "" { - rt := Task{Task: m} - err := rt.Delete(r.Client) - if err == nil { - m.Pod = "" - nChanged++ - } else { - Log.Error(err, "") - } - } if m.HasBucket() { Log.Info("Task bucket released.", "id", m.ID) m.SetBucket(nil) @@ -151,8 +150,7 @@ func (r *TaskReaper) release(m *model.Task) { nChanged++ } if nChanged > 0 { - rt := task.Task{Task: m} - rt.Event(task.Released) + m.Event(task.Released) err := r.DB.Save(m).Error if err != nil { Log.Error(err, "") @@ -161,10 +159,25 @@ func (r *TaskReaper) release(m *model.Task) { return } +// podDelete deletes the task pod. +func (r *TaskReaper) podDelete(m *task.Task) { + if m.Pod == "" { + return + } + err := m.Delete(r.Client) + if err != nil { + Log.Error(err, "") + return + } + err = r.DB.Save(m).Error + if err != nil { + Log.Error(err, "") + } +} + // delete task. -func (r *TaskReaper) delete(m *model.Task) { - rt := Task{Task: m} - err := rt.Delete(r.Client) +func (r *TaskReaper) delete(m *task.Task) { + err := m.Delete(r.Client) if err != nil { Log.Error(err, "") } diff --git a/settings/hub.go b/settings/hub.go index d21e14813..a215177e0 100644 --- a/settings/hub.go +++ b/settings/hub.go @@ -19,6 +19,8 @@ const ( EnvTaskReapCreated = "TASK_REAP_CREATED" EnvTaskReapSucceeded = "TASK_REAP_SUCCEEDED" EnvTaskReapFailed = "TASK_REAP_FAILED" + EnvTaskPodRetainSucceeded = "TASK_POD_RETAIN_SUCCEEDED" + EnvTaskPodRetainFailed = "TASK_POD_RETAIN_FAILED" EnvTaskSA = "TASK_SA" EnvTaskRetries = "TASK_RETRIES" EnvTaskPreemptEnabled = "TASK_PREEMPT_ENABLED" @@ -84,6 +86,12 @@ type Hub struct { Postponed time.Duration Rate int } + Pod struct { + Retention struct { + Succeeded int + Failed int + } + } } // Frequency Frequency struct { @@ -169,6 +177,20 @@ func (r *Hub) Load() (err error) { } else { r.Task.Reaper.Failed = 43200 // 720 hours (30 days). } + s, found = os.LookupEnv(EnvTaskPodRetainSucceeded) + if found { + n, _ := strconv.Atoi(s) + r.Task.Pod.Retention.Succeeded = n + } else { + r.Task.Pod.Retention.Succeeded = 1 + } + s, found = os.LookupEnv(EnvTaskPodRetainFailed) + if found { + n, _ := strconv.Atoi(s) + r.Task.Pod.Retention.Failed = n + } else { + r.Task.Pod.Retention.Failed = 4320 // 72 hours. + } r.Task.SA, found = os.LookupEnv(EnvTaskSA) if !found { r.Task.SA = "tackle-hub" diff --git a/task/manager.go b/task/manager.go index 914c23999..5bd34fcf4 100644 --- a/task/manager.go +++ b/task/manager.go @@ -1,6 +1,7 @@ package task import ( + "bytes" "context" "errors" "fmt" @@ -30,7 +31,10 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" k8s "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) // States @@ -51,19 +55,20 @@ const ( // Events const ( - AddonSelected = "AddonSelected" - ExtSelected = "ExtensionSelected" - ImageError = "ImageError" - PodNotFound = "PodNotFound" - PodCreated = "PodCreated" - PodPending = "PodPending" - PodRunning = "PodRunning" - Preempted = "Preempted" - PodSucceeded = "PodSucceeded" - PodFailed = "PodFailed" - PodDeleted = "PodDeleted" - Escalated = "Escalated" - Released = "Released" + AddonSelected = "AddonSelected" + ExtSelected = "ExtensionSelected" + ImageError = "ImageError" + PodNotFound = "PodNotFound" + PodCreated = "PodCreated" + PodPending = "PodPending" + PodRunning = "PodRunning" + Preempted = "Preempted" + PodSucceeded = "PodSucceeded" + PodFailed = "PodFailed" + PodDeleted = "PodDeleted" + Escalated = "Escalated" + Released = "Released" + ContainerKilled = "ContainerKilled" ) // k8s labels. @@ -123,6 +128,7 @@ func (m *Manager) Run(ctx context.Context) { m.deleteOrphanPods() m.runActions() m.updateRunning() + m.deleteZombies() m.startReady() m.pause() } else { @@ -860,9 +866,6 @@ func (m *Manager) updateRunning() { list = append(list, &Task{task}) } for _, task := range list { - if !task.StateIn(Running, Pending) { - continue - } running := task pod, found := running.Reflect(&m.cluster) if found { @@ -872,10 +875,24 @@ func (m *Manager) updateRunning() { Log.Error(err, "") continue } - err = running.Delete(m.Client) - if err != nil { - Log.Error(err, "") - continue + podRetention := 0 + if running.State == Succeeded { + podRetention = Settings.Hub.Task.Pod.Retention.Succeeded + } else { + podRetention = Settings.Hub.Task.Pod.Retention.Failed + } + if podRetention > 0 { + err = m.ensureTerminated(running, pod) + if err != nil { + podRetention = 0 + } + } + if podRetention == 0 { + err = running.Delete(m.Client) + if err != nil { + Log.Error(err, "") + continue + } } } } @@ -888,6 +905,56 @@ func (m *Manager) updateRunning() { } } +// deleteZombies - detect and delete zombie pods. +// A zombie is a (succeed|failed) task with a running pod that +// the manager has previously tried to kill. +func (m *Manager) deleteZombies() { + var err error + defer func() { + Log.Error(err, "") + }() + var pods []string + for _, pod := range m.cluster.Pods() { + if pod.Status.Phase == core.PodRunning { + ref := path.Join(pod.Namespace, pod.Name) + pods = append( + pods, + ref) + } + } + fetched := []*Task{} + db := m.DB.Select("Events") + db = db.Where("Pod", pods) + db = db.Where("state IN ?", + []string{ + Succeeded, + Failed, + }) + err = db.Find(&fetched).Error + if err != nil { + err = liberr.Wrap(err) + return + } + for _, task := range fetched { + event, found := task.LastEvent(ContainerKilled) + if !found { + continue + } + if time.Since(event.Last) > time.Minute { + Log.Info( + "Zombie detected.", + "task", + task.ID, + "pod", + task.Pod) + err = task.Delete(m.Client) + if err != nil { + Log.Error(err, "") + } + } + } +} + // deleteOrphanPods finds and deletes task pods not referenced by a task. func (m *Manager) deleteOrphanPods() { var err error @@ -1068,6 +1135,87 @@ func (m *Manager) containerLog(pod *core.Pod, container string) (file *model.Fil return } +// ensureTerminated - Terminate running containers. +func (m *Manager) ensureTerminated(task *Task, pod *core.Pod) (err error) { + for _, status := range pod.Status.ContainerStatuses { + if status.State.Terminated != nil { + continue + } + if status.Started == nil || !*status.Started { + continue + } + err = m.terminateContainer(task, pod, status.Name) + if err != nil { + return + } + } + return +} + +// terminateContainer - Terminate container as needed. +// The container is killed. +// Should the container continue to run after (1) minute, +// it is reported as an error. +func (m *Manager) terminateContainer(task *Task, pod *core.Pod, container string) (err error) { + Log.V(1).Info("KILL container", "container", container) + clientSet, err := k8s2.NewClientSet() + if err != nil { + return + } + cmd := []string{ + "sh", + "-c", + "kill 1", + } + req := clientSet.CoreV1().RESTClient().Post() + req = req.Resource("pods") + req = req.Name(pod.Name) + req = req.Namespace(pod.Namespace) + req = req.SubResource("exec") + option := &core.PodExecOptions{ + Command: cmd, + Container: container, + Stdout: true, + Stderr: true, + TTY: true, + } + req.VersionedParams( + option, + scheme.ParameterCodec, + ) + cfg, _ := config.GetConfig() + exec, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return + } + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + Log.Info( + "Container KILL failed.", + "name", + container, + "err", + err.Error(), + "stderr", + stderr.String()) + } else { + task.Event( + ContainerKilled, + "container: '%s' had not terminated.", + container) + Log.Info( + "Container KILLED.", + "name", + container) + } + return +} + // Task is an runtime task. type Task struct { // model. @@ -1140,6 +1288,17 @@ func (r *Task) LastEvent(kind string) (event *model.TaskEvent, found bool) { return } +// FindEvent returns the matched events by kind. +func (r *Task) FindEvent(kind string) (matched []*model.TaskEvent) { + for i := 0; i < len(r.Events); i++ { + event := &r.Events[i] + if kind == event.Kind { + matched = append(matched, event) + } + } + return +} + // Run the specified task. func (r *Task) Run(cluster *Cluster) (started bool, err error) { mark := time.Now() From 055cb2f7df40df48d78bbda7b1aebd74acb4d2d8 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Wed, 21 Aug 2024 15:54:19 -0500 Subject: [PATCH 07/15] :bug: Fixes database-locked. (#747) Seems the underlying sqlite driver keeps the lock until the connection is closed. The `Conn` acquires the mutex and holds it until the connection is closed. The `Tx` and `Stmt` are no longer necessary. --------- Signed-off-by: Jeff Ortel --- database/db_test.go | 45 ++++++++++++--- database/driver.go | 131 +++++++++----------------------------------- database/pkg.go | 2 +- 3 files changed, 64 insertions(+), 114 deletions(-) diff --git a/database/db_test.go b/database/db_test.go index 36555f21b..37102df91 100644 --- a/database/db_test.go +++ b/database/db_test.go @@ -1,19 +1,24 @@ package database import ( - "encoding/json" "fmt" "os" "testing" + "time" "github.com/konveyor/tackle2-hub/model" + "gorm.io/gorm" + "k8s.io/utils/env" ) -var N = 800 +var N, _ = env.GetInt("TEST_CONCURRENT", 10) func TestConcurrent(t *testing.T) { - Settings.DB.Path = "/tmp/concurrent.db" - _ = os.Remove(Settings.DB.Path) + pid := os.Getpid() + Settings.DB.Path = fmt.Sprintf("/tmp/concurrent-%d.db", pid) + defer func() { + _ = os.Remove(Settings.DB.Path) + }() db, err := Open(true) if err != nil { panic(err) @@ -22,13 +27,39 @@ func TestConcurrent(t *testing.T) { for w := 0; w < N; w++ { go func(id int) { fmt.Printf("Started %d\n", id) - for n := 0; n < N; n++ { - v, _ := json.Marshal(fmt.Sprintf("Test-%d", n)) - m := &model.Setting{Key: fmt.Sprintf("key-%d-%d", id, n), Value: v} + for n := 0; n < N*10; n++ { + m := &model.Setting{Key: fmt.Sprintf("key-%d-%d", id, n), Value: n} + fmt.Printf("(%.4d) CREATE: %.4d\n", id, n) uErr := db.Create(m).Error if uErr != nil { panic(uErr) } + uErr = db.Save(m).Error + if uErr != nil { + panic(uErr) + } + for i := 0; i < 10; i++ { + fmt.Printf("(%.4d) READ: %.4d/%.4d\n", id, n, i) + uErr = db.First(m).Error + if uErr != nil { + panic(uErr) + } + } + for i := 0; i < 4; i++ { + uErr = db.Transaction(func(tx *gorm.DB) (err error) { + time.Sleep(time.Millisecond * 10) + for i := 0; i < 3; i++ { + err = tx.Save(m).Error + if err != nil { + break + } + } + return + }) + if uErr != nil { + panic(uErr) + } + } } dq <- id }(w) diff --git a/database/driver.go b/database/driver.go index 708228530..14bfbb1fa 100644 --- a/database/driver.go +++ b/database/driver.go @@ -44,9 +44,9 @@ func (d *Driver) Driver() driver.Driver { } type Conn struct { - mutex *sync.Mutex - wrapped driver.Conn - tx driver.Tx + mutex *sync.Mutex + wrapped driver.Conn + hasMutex bool } func (c *Conn) Ping(ctx context.Context) (err error) { @@ -70,9 +70,8 @@ func (c *Conn) IsValid() (b bool) { } func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Rows, err error) { - if c.tx == nil { - c.mutex.Lock() - defer c.mutex.Unlock() + if c.needsMutex(query) { + c.acquire() } if p, cast := c.wrapped.(driver.QueryerContext); cast { r, err = p.QueryContext(ctx, query, args) @@ -81,29 +80,17 @@ func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.Nam } func (c *Conn) PrepareContext(ctx context.Context, query string) (s driver.Stmt, err error) { + if c.needsMutex(query) { + c.acquire() + } if p, cast := c.wrapped.(driver.ConnPrepareContext); cast { s, err = p.PrepareContext(ctx, query) } - if err != nil { - return - } - stmtLocked := c.stmtLocked(query) - s = &Stmt{ - mutex: c.mutex, - locked: stmtLocked, - wrapped: s, - } - if stmtLocked { - c.mutex.Lock() - } return } func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Result, err error) { - if c.tx == nil { - c.mutex.Lock() - defer c.mutex.Unlock() - } + c.acquire() if p, cast := c.wrapped.(driver.ExecerContext); cast { r, err = p.ExecContext(ctx, query, args) } @@ -111,61 +98,40 @@ func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.Name } func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { + c.acquire() if p, cast := c.wrapped.(driver.ConnBeginTx); cast { tx, err = p.BeginTx(ctx, opts) } else { tx, err = c.wrapped.Begin() } - if err != nil { - return - } - tx = &Tx{ - mutex: c.mutex, - wrapped: tx, - } - c.tx = tx - c.mutex.Lock() return } func (c *Conn) Prepare(query string) (s driver.Stmt, err error) { - s, err = c.wrapped.Prepare(query) - if err != nil { - return - } - stmtLocked := c.stmtLocked(query) - s = &Stmt{ - mutex: c.mutex, - locked: stmtLocked, - wrapped: s, - } - if stmtLocked { - c.mutex.Lock() + if c.needsMutex(query) { + c.acquire() } + s, err = c.wrapped.Prepare(query) return } func (c *Conn) Close() (err error) { err = c.wrapped.Close() + c.release() return } func (c *Conn) Begin() (tx driver.Tx, err error) { + c.acquire() tx, err = c.wrapped.Begin() if err != nil { return } - tx = &Tx{ - mutex: c.mutex, - wrapped: tx, - } - c.tx = tx - c.mutex.Lock() return } -func (c *Conn) stmtLocked(query string) (matched bool) { - if c.tx != nil || query == "" { +func (c *Conn) needsMutex(query string) (matched bool) { + if query == "" { return } query = strings.ToUpper(query) @@ -178,63 +144,16 @@ func (c *Conn) stmtLocked(query string) (matched bool) { return } -type Tx struct { - mutex *sync.Mutex - wrapped driver.Tx -} - -func (tx *Tx) Commit() (err error) { - defer func() { - tx.mutex.Unlock() - }() - err = tx.wrapped.Commit() - return -} -func (tx *Tx) Rollback() (err error) { - defer func() { - tx.mutex.Unlock() - }() - err = tx.wrapped.Rollback() - return -} - -type Stmt struct { - mutex *sync.Mutex - wrapped driver.Stmt - locked bool -} - -func (s *Stmt) Close() (err error) { - if s.locked { - s.mutex.Unlock() - } - err = s.wrapped.Close() - return -} -func (s *Stmt) NumInput() (n int) { - n = s.wrapped.NumInput() - return -} -func (s *Stmt) Exec(args []driver.Value) (r driver.Result, err error) { - r, err = s.wrapped.Exec(args) - return -} - -func (s *Stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (r driver.Result, err error) { - if p, cast := s.wrapped.(driver.StmtExecContext); cast { - r, err = p.ExecContext(ctx, args) +func (c *Conn) acquire() { + if !c.hasMutex { + c.mutex.Lock() + c.hasMutex = true } - return } -func (s *Stmt) Query(args []driver.Value) (r driver.Rows, err error) { - r, err = s.wrapped.Query(args) - return -} - -func (s *Stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (r driver.Rows, err error) { - if p, cast := s.wrapped.(driver.StmtQueryContext); cast { - r, err = p.QueryContext(ctx, args) +func (c *Conn) release() { + if c.hasMutex { + c.mutex.Unlock() + c.hasMutex = false } - return } diff --git a/database/pkg.go b/database/pkg.go index 3e9adfdae..b513c1c6c 100644 --- a/database/pkg.go +++ b/database/pkg.go @@ -18,7 +18,7 @@ var log = logr.WithName("db") var Settings = &settings.Settings const ( - ConnectionString = "file:%s?_journal=WAL" + ConnectionString = "file:%s?_journal=WAL&_timeout=100" FKsOn = "&_foreign_keys=yes" FKsOff = "&_foreign_keys=no" ) From d3871f8dad4d8db1d2658d944738da066a28b40c Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Thu, 22 Aug 2024 14:21:33 -0500 Subject: [PATCH 08/15] :ghost: API-test action (#749) Signed-off-by: Jeff Ortel --- .github/workflows/main.yml | 10 +++++++--- Makefile | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e99967c4..45642a6fa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,16 +52,20 @@ jobs: - build - test-unit runs-on: ubuntu-latest + env: + DISCONNECTED: 1 + HUB_BASE_URL: http://localhost:8080 + DB_PATH: /tmp/hub.db steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: go-version: '1.19' - run: | - DISCONNECTED=1 make run & + rm -f $DB_PATH + make run & sleep 15 # probably a dirty solution - DISCONNECTED=1 HUB_BASE_URL=http://localhost:8080 make test-api - DISCONNECTED=1 HUB_BASE_URL=http://localhost:8080 make test-api # Intentionaly run 2x to catch data left in Hub DB. + make test-api build-image: needs: diff --git a/Makefile b/Makefile index cc27ae466..e8463bc85 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ test: # Run Hub REST API tests. test-api: - HUB_BASE_URL=$(HUB_BASE_URL) go test -count=1 -p=1 -v ./test/api/... + HUB_BASE_URL=$(HUB_BASE_URL) go test -count=1 -p=1 -v -failfast ./test/api/... # Run Hub test suite. test-all: test-unit test-api From 384d7f8a65dfae854f5dadbf055dc8976ab208cb Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Mon, 9 Sep 2024 11:53:21 -0500 Subject: [PATCH 09/15] :sparkles: Generate Primary Keys. (#635) Generate primary keys instead of GORM. This fixes the issue of GORM reusing the highest key after the model with that ID is deleted. When the PK is 0, GORM assigns the next (highest) ID. This approach is to assign the ID ahead of time using a pool managed by tackle. --------- Signed-off-by: Jeff Ortel --- api/migrationwave.go | 1 + cmd/main.go | 5 + database/db_test.go | 53 ++++++++++ database/pk.go | 160 +++++++++++++++++++++++++++++ database/pkg.go | 11 +- migration/v14/model/core.go | 7 ++ migration/v14/model/pkg.go | 1 + model/pkg.go | 3 + test/api/migrationwave/api_test.go | 6 ++ test/api/migrationwave/samples.go | 3 - test/api/review/api_test.go | 2 +- test/api/review/samples.go | 2 - test/api/ticket/api_test.go | 10 +- test/api/ticket/samples.go | 5 +- test/api/tracker/samples.go | 2 - 15 files changed, 257 insertions(+), 14 deletions(-) create mode 100644 database/pk.go diff --git a/api/migrationwave.go b/api/migrationwave.go index b92b23d6b..df75eb7d0 100644 --- a/api/migrationwave.go +++ b/api/migrationwave.go @@ -118,6 +118,7 @@ func (h MigrationWaveHandler) Create(ctx *gin.Context) { _ = ctx.Error(err) return } + r.With(m) h.Respond(ctx, http.StatusCreated, r) diff --git a/cmd/main.go b/cmd/main.go index 8be4667bb..91a6adcb3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,6 +17,7 @@ import ( crd "github.com/konveyor/tackle2-hub/k8s/api" "github.com/konveyor/tackle2-hub/metrics" "github.com/konveyor/tackle2-hub/migration" + "github.com/konveyor/tackle2-hub/model" "github.com/konveyor/tackle2-hub/reaper" "github.com/konveyor/tackle2-hub/seed" "github.com/konveyor/tackle2-hub/settings" @@ -53,6 +54,10 @@ func Setup() (db *gorm.DB, err error) { if err != nil { return } + err = database.PK.Load(db, model.ALL) + if err != nil { + return + } return } diff --git a/database/db_test.go b/database/db_test.go index 37102df91..dde8c6fae 100644 --- a/database/db_test.go +++ b/database/db_test.go @@ -69,3 +69,56 @@ func TestConcurrent(t *testing.T) { fmt.Printf("Done %d\n", id) } } + +func TestKeyGen(t *testing.T) { + pid := os.Getpid() + Settings.DB.Path = fmt.Sprintf("/tmp/keygen-%d.db", pid) + defer func() { + _ = os.Remove(Settings.DB.Path) + }() + db, err := Open(true) + if err != nil { + panic(err) + } + // ids 1-7 created. + N = 8 + for n := 1; n < N; n++ { + m := &model.Setting{Key: fmt.Sprintf("key-%d", n), Value: n} + err := db.Create(m).Error + if err != nil { + panic(err) + } + fmt.Printf("CREATED: %d/%d\n", m.ID, n) + if uint(n) != m.ID { + t.Errorf("id:%d but expected: %d", m.ID, n) + return + } + } + // delete ids=2,4,7. + err = db.Delete(&model.Setting{}, []uint{2, 4, 7}).Error + if err != nil { + panic(err) + } + + var count int64 + err = db.Model(&model.Setting{}).Where([]uint{2, 4, 7}).Count(&count).Error + if err != nil { + panic(err) + } + if count > 0 { + t.Errorf("DELETED ids: 2,4,7 found.") + return + } + // id=8 (next) created. + next := N + m := &model.Setting{Key: fmt.Sprintf("key-%d", next), Value: next} + err = db.Create(m).Error + if err != nil { + panic(err) + } + fmt.Printf("CREATED: %d/%d (next)\n", m.ID, next) + if uint(N) != m.ID { + t.Errorf("id:%d but expected: %d", m.ID, next) + return + } +} diff --git a/database/pk.go b/database/pk.go new file mode 100644 index 000000000..d9f19d49a --- /dev/null +++ b/database/pk.go @@ -0,0 +1,160 @@ +package database + +import ( + "errors" + "reflect" + "strings" + "sync" + + "github.com/konveyor/tackle2-hub/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" +) + +// PK singleton pk sequence. +var PK PkSequence + +// PkSequence provides a primary key sequence. +type PkSequence struct { + mutex sync.Mutex +} + +// Load highest key for all models. +func (r *PkSequence) Load(db *gorm.DB, models []any) (err error) { + r.mutex.Lock() + defer r.mutex.Unlock() + for _, m := range models { + mt := reflect.TypeOf(m) + if mt.Kind() == reflect.Ptr { + mt = mt.Elem() + } + kind := strings.ToUpper(mt.Name()) + db = r.session(db) + q := db.Table(kind) + q = q.Select("MAX(ID) id") + cursor, err := q.Rows() + if err != nil || !cursor.Next() { + // not a table with id. + // discarded. + continue + } + id := int64(0) + err = cursor.Scan(&id) + _ = cursor.Close() + if err != nil { + r.add(db, kind, uint(0)) + } else { + r.add(db, kind, uint(id)) + } + } + return +} + +// Next returns the next primary key. +func (r *PkSequence) Next(db *gorm.DB) (id uint) { + r.mutex.Lock() + defer r.mutex.Unlock() + kind := strings.ToUpper(db.Statement.Table) + m := &model.PK{} + db = r.session(db) + err := db.First(m, "Kind", kind).Error + if err != nil { + return + } + m.LastID++ + id = m.LastID + err = db.Save(m).Error + if err != nil { + panic(err) + } + return +} + +// session returns a new DB with a new session. +func (r *PkSequence) session(in *gorm.DB) (out *gorm.DB) { + out = &gorm.DB{ + Config: in.Config, + } + out.Config.Logger.LogMode(logger.Warn) + out.Statement = &gorm.Statement{ + DB: out, + ConnPool: in.Statement.ConnPool, + Context: in.Statement.Context, + Clauses: map[string]clause.Clause{}, + Vars: make([]interface{}, 0, 8), + } + return +} + +// add the last (higher) id for the kind. +func (r *PkSequence) add(db *gorm.DB, kind string, id uint) { + m := &model.PK{Kind: kind} + db = r.session(db) + err := db.First(m).Error + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + panic(err) + } + } + if m.LastID > id { + return + } + m.LastID = id + db = r.session(db) + err = db.Save(m).Error + if err != nil { + panic(err) + } +} + +// assignPk assigns PK as needed. +func assignPk(db *gorm.DB) { + statement := db.Statement + schema := statement.Schema + if schema == nil { + return + } + switch statement.ReflectValue.Kind() { + case reflect.Slice, + reflect.Array: + for i := 0; i < statement.ReflectValue.Len(); i++ { + for _, f := range schema.Fields { + if f.Name != "ID" { + continue + } + _, isZero := f.ValueOf( + statement.Context, + statement.ReflectValue.Index(i)) + if isZero { + id := PK.Next(db) + _ = f.Set( + statement.Context, + statement.ReflectValue.Index(i), + id) + + } + break + } + } + case reflect.Struct: + for _, f := range schema.Fields { + if f.Name != "ID" { + continue + } + _, isZero := f.ValueOf( + statement.Context, + statement.ReflectValue) + if isZero { + id := PK.Next(db) + _ = f.Set( + statement.Context, + statement.ReflectValue, + id) + } + break + } + default: + log.Info("[WARN] assignPk: unknown kind.") + } +} diff --git a/database/pkg.go b/database/pkg.go index b513c1c6c..6e9c5eb20 100644 --- a/database/pkg.go +++ b/database/pkg.go @@ -51,7 +51,16 @@ func Open(enforceFKs bool) (db *gorm.DB, err error) { err = liberr.Wrap(err) return } - err = db.AutoMigrate(model.Setting{}) + err = db.AutoMigrate(model.PK{}, model.Setting{}) + if err != nil { + err = liberr.Wrap(err) + return + } + err = PK.Load(db, []any{model.Setting{}}) + if err != nil { + return + } + err = db.Callback().Create().Before("gorm:before_create").Register("assign-pk", assignPk) if err != nil { err = liberr.Wrap(err) return diff --git a/migration/v14/model/core.go b/migration/v14/model/core.go index a5d6a5ab1..4f384bb6b 100644 --- a/migration/v14/model/core.go +++ b/migration/v14/model/core.go @@ -20,6 +20,13 @@ type Model struct { UpdateUser string } +// PK sequence. +type PK struct { + Kind string `gorm:"<-:create;primaryKey"` + LastID uint +} + +// Setting hub settings. type Setting struct { Model Key string `gorm:"<-:create;uniqueIndex"` diff --git a/migration/v14/model/pkg.go b/migration/v14/model/pkg.go index 6827e3c96..8f612b488 100644 --- a/migration/v14/model/pkg.go +++ b/migration/v14/model/pkg.go @@ -32,6 +32,7 @@ func All() []any { ImportTag{}, JobFunction{}, MigrationWave{}, + PK{}, Proxy{}, Review{}, Setting{}, diff --git a/model/pkg.go b/model/pkg.go index 5cf00cd5a..d6b546fb3 100644 --- a/model/pkg.go +++ b/model/pkg.go @@ -8,6 +8,8 @@ import ( // Field (data) types. type JSON = model.JSON +var ALL = model.All() + // Models type Model = model.Model type Application = model.Application @@ -29,6 +31,7 @@ type ImportSummary = model.ImportSummary type ImportTag = model.ImportTag type JobFunction = model.JobFunction type MigrationWave = model.MigrationWave +type PK = model.PK type Proxy = model.Proxy type Questionnaire = model.Questionnaire type Review = model.Review diff --git a/test/api/migrationwave/api_test.go b/test/api/migrationwave/api_test.go index afa2ac846..7c2622a84 100644 --- a/test/api/migrationwave/api_test.go +++ b/test/api/migrationwave/api_test.go @@ -18,6 +18,7 @@ func TestMigrationWaveCRUD(t *testing.T) { } assert.Must(t, Application.Create(&expectedApp)) createdApps = append(createdApps, expectedApp) + r.Applications[0].ID = expectedApp.ID } createdStakeholders := []api.Stakeholder{} @@ -28,6 +29,7 @@ func TestMigrationWaveCRUD(t *testing.T) { } assert.Must(t, Stakeholder.Create(&expectedStakeholder)) createdStakeholders = append(createdStakeholders, expectedStakeholder) + r.Stakeholders[0].ID = expectedStakeholder.ID } createdStakeholderGroups := []api.StakeholderGroup{} @@ -38,6 +40,7 @@ func TestMigrationWaveCRUD(t *testing.T) { } assert.Must(t, StakeholderGroup.Create(&expectedStakeholderGroup)) createdStakeholderGroups = append(createdStakeholderGroups, expectedStakeholderGroup) + r.StakeholderGroups[0].ID = expectedStakeholderGroup.ID } assert.Must(t, MigrationWave.Create(&r)) @@ -102,6 +105,7 @@ func TestMigrationWaveList(t *testing.T) { } assert.Must(t, Application.Create(&expectedApp)) createdApps = append(createdApps, expectedApp) + r.Applications[0].ID = expectedApp.ID } for _, stakeholder := range r.Stakeholders { @@ -111,6 +115,7 @@ func TestMigrationWaveList(t *testing.T) { } assert.Must(t, Stakeholder.Create(&expectedStakeholder)) createdStakeholders = append(createdStakeholders, expectedStakeholder) + r.Stakeholders[0].ID = expectedStakeholder.ID } for _, stakeholderGroup := range r.StakeholderGroups { @@ -120,6 +125,7 @@ func TestMigrationWaveList(t *testing.T) { } assert.Must(t, StakeholderGroup.Create(&expectedStakeholderGroup)) createdStakeholderGroups = append(createdStakeholderGroups, expectedStakeholderGroup) + r.StakeholderGroups[0].ID = expectedStakeholderGroup.ID } assert.Must(t, MigrationWave.Create(&r)) createdMigrationWaves = append(createdMigrationWaves, r) diff --git a/test/api/migrationwave/samples.go b/test/api/migrationwave/samples.go index 4c0ef9fa1..535d5dbae 100644 --- a/test/api/migrationwave/samples.go +++ b/test/api/migrationwave/samples.go @@ -13,19 +13,16 @@ var Samples = []api.MigrationWave{ EndDate: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local).Add(30 * time.Minute), Applications: []api.Ref{ { - ID: 1, Name: "Sample Application", }, }, Stakeholders: []api.Ref{ { - ID: 1, Name: "Sample Stakeholders", }, }, StakeholderGroups: []api.Ref{ { - ID: 1, Name: "Sample Stakeholders Groups", }, }, diff --git a/test/api/review/api_test.go b/test/api/review/api_test.go index 013f84842..d32555859 100644 --- a/test/api/review/api_test.go +++ b/test/api/review/api_test.go @@ -143,7 +143,7 @@ func TestReviewList(t *testing.T) { // Delete related reviews and applications. for _, review := range createdReviews { - assert.Must(t, Application.Delete(review.ID)) + assert.Must(t, Application.Delete(review.Application.ID)) assert.Must(t, Review.Delete(review.ID)) } } diff --git a/test/api/review/samples.go b/test/api/review/samples.go index 4fb33b981..64df44e99 100644 --- a/test/api/review/samples.go +++ b/test/api/review/samples.go @@ -12,7 +12,6 @@ var Samples = []api.Review{ WorkPriority: 1, Comments: "nil", Application: &api.Ref{ - ID: 1, Name: "Sample Review 1", }, }, @@ -23,7 +22,6 @@ var Samples = []api.Review{ WorkPriority: 2, Comments: "nil", Application: &api.Ref{ - ID: 2, Name: "Sample Review 2", }, }, diff --git a/test/api/ticket/api_test.go b/test/api/ticket/api_test.go index af6f277c1..71ea3d178 100644 --- a/test/api/ticket/api_test.go +++ b/test/api/ticket/api_test.go @@ -17,6 +17,7 @@ func TestTicketCRUD(t *testing.T) { Name: r.Application.Name, } assert.Must(t, Application.Create(&app)) + r.Application.ID = app.ID createdIdentities := []api.Identity{} createdTrackers := []api.Tracker{} @@ -27,8 +28,11 @@ func TestTicketCRUD(t *testing.T) { Kind: tracker.Kind, } assert.Must(t, Identity.Create(&identity)) + tracker.Identity.ID = identity.ID createdIdentities = append(createdIdentities, identity) assert.Must(t, Tracker.Create(&tracker)) + r.Tracker.ID = tracker.ID + r.Tracker.Name = tracker.Name createdTrackers = append(createdTrackers, tracker) } @@ -72,6 +76,7 @@ func TestTicketList(t *testing.T) { Name: r.Application.Name, } assert.Must(t, Application.Create(&app)) + r.Application.ID = app.ID createdIdentities := []api.Identity{} createdTrackers := []api.Tracker{} @@ -82,8 +87,11 @@ func TestTicketList(t *testing.T) { Kind: tracker.Kind, } assert.Must(t, Identity.Create(&identity)) + tracker.Identity.ID = identity.ID createdIdentities = append(createdIdentities, identity) assert.Must(t, Tracker.Create(&tracker)) + r.Tracker.ID = tracker.ID + r.Tracker.Name = tracker.Name createdTrackers = append(createdTrackers, tracker) } @@ -113,7 +121,7 @@ func TestTicketList(t *testing.T) { // Delete tickets and related resources. for _, ticket := range createdTickets { assert.Must(t, Ticket.Delete(ticket.ID)) - assert.Must(t, Application.Delete(ticket.ID)) + assert.Must(t, Application.Delete(ticket.Application.ID)) } for _, tracker := range createdTrackers { assert.Must(t, Tracker.Delete(tracker.ID)) diff --git a/test/api/ticket/samples.go b/test/api/ticket/samples.go index 5a74246ad..a4fe35eb4 100644 --- a/test/api/ticket/samples.go +++ b/test/api/ticket/samples.go @@ -2,7 +2,6 @@ package ticket import ( "github.com/konveyor/tackle2-hub/api" - TrackerSamples "github.com/konveyor/tackle2-hub/test/api/tracker" ) var Samples = []api.Ticket{ @@ -10,12 +9,10 @@ var Samples = []api.Ticket{ Kind: "10001", Parent: "10000", Application: api.Ref{ - ID: 1, Name: "Sample Application1", }, Tracker: api.Ref{ - ID: 1, - Name: TrackerSamples.Samples[0].Name, + Name: "Sample Ticket-Tracker", }, }, } diff --git a/test/api/tracker/samples.go b/test/api/tracker/samples.go index 1face0d01..e9701e729 100644 --- a/test/api/tracker/samples.go +++ b/test/api/tracker/samples.go @@ -14,7 +14,6 @@ var Samples = []api.Tracker{ Message: "Description of tracker", LastUpdated: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local), Identity: api.Ref{ - ID: 1, Name: "Sample Tracker Identity", }, Insecure: false, @@ -26,7 +25,6 @@ var Samples = []api.Tracker{ Message: "Description of tracker1", LastUpdated: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local), Identity: api.Ref{ - ID: 2, Name: "Sample Tracker Identity1", }, Insecure: false, From 7fe3d6a8adcbb93f28dd41fe6b8bf7b6f52411aa Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 13 Sep 2024 07:24:07 -0500 Subject: [PATCH 10/15] :sparkles: Broker KAI api through hub. (#750) Add `/services/` endpoint. Add `/services/kai/*` endpoint reverse-proxy to route defined in KAI_URL. Add auth scopes. Related: https://github.com/konveyor/operator/pull/376 --------- Signed-off-by: Jeff Ortel --- .github/workflows/main.yml | 10 +-- .github/workflows/test-nightly.yml | 4 +- Makefile | 2 +- api/error.go | 20 +++++- api/pkg.go | 1 + api/service.go | 99 ++++++++++++++++++++++++++++++ auth/roles.yaml | 16 +++++ 7 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 api/service.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45642a6fa..ab89020dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make fmt vet: @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make vet build: @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make cmd test-unit: @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make test test-api: @@ -60,7 +60,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: | rm -f $DB_PATH make run & diff --git a/.github/workflows/test-nightly.yml b/.github/workflows/test-nightly.yml index 60e81cf83..90350c819 100644 --- a/.github/workflows/test-nightly.yml +++ b/.github/workflows/test-nightly.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make test test-api: @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: | make vet DISCONNECTED=1 make run & diff --git a/Makefile b/Makefile index e8463bc85..b092ad5fd 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ $(CONTROLLERGEN): # Ensure goimports installed. $(GOIMPORTS): - go install golang.org/x/tools/cmd/goimports@latest + go install golang.org/x/tools/cmd/goimports@v0.24 # Build SAMPLE ADDON addon: fmt vet diff --git a/api/error.go b/api/error.go index 01dc4a4ba..fe79e47d9 100644 --- a/api/error.go +++ b/api/error.go @@ -2,6 +2,7 @@ package api import ( "errors" + "fmt" "net/http" "os" @@ -78,6 +79,22 @@ func (r *Forbidden) Is(err error) (matched bool) { return } +// NotFound reports resource not-found errors. +type NotFound struct { + Resource string + Reason string +} + +func (r *NotFound) Error() string { + return fmt.Sprintf("Resource '%s' not found. %s", r.Resource, r.Reason) +} + +func (r *NotFound) Is(err error) (matched bool) { + var forbidden *Forbidden + matched = errors.As(err, &forbidden) + return +} + // ErrorHandler handles error conditions from lower handlers. func ErrorHandler() gin.HandlerFunc { return func(ctx *gin.Context) { @@ -102,7 +119,8 @@ func ErrorHandler() gin.HandlerFunc { return } - if errors.Is(err, gorm.ErrRecordNotFound) { + if errors.Is(err, gorm.ErrRecordNotFound) || + errors.Is(err, &NotFound{}) { if ctx.Request.Method == http.MethodDelete { rtx.Status(http.StatusNoContent) return diff --git a/api/pkg.go b/api/pkg.go index 36f78c052..18efbcb3e 100644 --- a/api/pkg.go +++ b/api/pkg.go @@ -78,6 +78,7 @@ func All() []Handler { &RuleSetHandler{}, &SchemaHandler{}, &SettingHandler{}, + &ServiceHandler{}, &StakeholderHandler{}, &StakeholderGroupHandler{}, &TagHandler{}, diff --git a/api/service.go b/api/service.go new file mode 100644 index 000000000..d3e269727 --- /dev/null +++ b/api/service.go @@ -0,0 +1,99 @@ +package api + +import ( + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "os" + + "github.com/gin-gonic/gin" +) + +// Routes +const ( + ServicesRoot = "/services" + ServiceRoot = ServicesRoot + "/:name/*" + Wildcard +) + +// serviceRoutes name to route map. +var serviceRoutes = map[string]string{ + "kai": os.Getenv("KAI_URL"), +} + +// ServiceHandler handles service routes. +type ServiceHandler struct { + BaseHandler +} + +// AddRoutes adds routes. +func (h ServiceHandler) AddRoutes(e *gin.Engine) { + e.GET(ServicesRoot, h.List) + e.Any(ServiceRoot, h.Required, h.Forward) +} + +// List godoc +// @summary List named service routes. +// @description List named service routes. +// @tags services +// @produce json +// @success 200 {object} api.Service +// @router /services [get] +func (h ServiceHandler) List(ctx *gin.Context) { + var r []Service + for name, route := range serviceRoutes { + service := Service{Name: name, Route: route} + r = append(r, service) + } + + h.Respond(ctx, http.StatusOK, r) +} + +// Required enforces RBAC. +func (h ServiceHandler) Required(ctx *gin.Context) { + Required(ctx.Param(Name))(ctx) +} + +// Forward provides RBAC and forwards request to the service. +func (h ServiceHandler) Forward(ctx *gin.Context) { + path := ctx.Param(Wildcard) + name := ctx.Param(Name) + route, found := serviceRoutes[name] + if !found { + err := &NotFound{Resource: name} + _ = ctx.Error(err) + return + } + if route == "" { + err := fmt.Errorf("route for: '%s' not defined", name) + _ = ctx.Error(err) + return + } + u, err := url.Parse(route) + if err != nil { + err = &BadRequestError{Reason: err.Error()} + _ = ctx.Error(err) + return + } + proxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL.Scheme = u.Scheme + req.URL.Host = u.Host + req.URL.Path = path + Log.Info( + "Routing (service)", + "path", + ctx.Request.URL.Path, + "route", + req.URL.String()) + }, + } + + proxy.ServeHTTP(ctx.Writer, ctx.Request) +} + +// Service REST resource. +type Service struct { + Name string `json:"name"` + Route string `json:"route"` +} diff --git a/auth/roles.yaml b/auth/roles.yaml index aa65e7a77..361a2c1ae 100644 --- a/auth/roles.yaml +++ b/auth/roles.yaml @@ -82,6 +82,10 @@ - get - post - put + - name: kai + verbs: + - get + - post - name: proxies verbs: - delete @@ -286,6 +290,10 @@ - get - post - put + - name: kai + verbs: + - get + - post - name: proxies verbs: - get @@ -443,6 +451,10 @@ - name: jobfunctions verbs: - get + - name: kai + verbs: + - get + - post - name: proxies verbs: - get @@ -560,6 +572,10 @@ - name: jobfunctions verbs: - get + - name: kai + verbs: + - get + - post - name: proxies verbs: - get From cf20929f1a360881babf01a1adad5f63ed018079 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Thu, 26 Sep 2024 10:23:44 -0500 Subject: [PATCH 11/15] :sparkles: Upload files instead of multi-part form. (#743) Post a _manifest_ file instead of separate multi-part form files. Much simpler and more easily supports the addon staging the issues and deps files on disk rather than streaming. The more atomic approach will prevent transaction deadlock which can more easily occur when the addon-analyzer builder reported an error (which it should never do). The uploaded file contains markers used to delimited the documents. `^]` = `\x1D` = GS (group separator). ``` ^]BEGIN-MAIN^] --- commit: 1234 ^]END-MAIN^] ^]BEGIN-ISSUES^] --- ruleset: ruleset-1 rule: rule-1 incidents: ... ^]END-ISSUES^] ^]BEGIN-DEPS^] --- name: github.com/jboss version: 4.0 labels: - konveyor.io/language=java - konveyor.io/otherA=dog ^]END-DEPS^] ``` Flow: 1. post (upload) manifest.yaml file. 2. post `ref` to the manifest file. 3. delete manifest file. Orphaned files will be reaped. --- The binding client needed to be updated to handle different file encoding (MIME). --------- Signed-off-by: Jeff Ortel --- api/analysis.go | 198 ++++++++++++++---- api/file.go | 164 ++++++++++----- binding/application.go | 62 +++--- binding/client.go | 34 +++- binding/file.go | 16 +- hack/add/analysis.sh | 67 ++++--- migration/pkg.go | 2 + migration/v15/migrate.go | 20 ++ migration/v15/model/analysis.go | 170 ++++++++++++++++ migration/v15/model/application.go | 312 +++++++++++++++++++++++++++++ migration/v15/model/assessment.go | 102 ++++++++++ migration/v15/model/core.go | 309 ++++++++++++++++++++++++++++ migration/v15/model/mod.patch | 11 + migration/v15/model/pkg.go | 56 ++++++ model/pkg.go | 2 +- 15 files changed, 1374 insertions(+), 151 deletions(-) create mode 100644 migration/v15/migrate.go create mode 100644 migration/v15/model/analysis.go create mode 100644 migration/v15/model/application.go create mode 100644 migration/v15/model/assessment.go create mode 100644 migration/v15/model/core.go create mode 100644 migration/v15/model/mod.patch create mode 100644 migration/v15/model/pkg.go diff --git a/api/analysis.go b/api/analysis.go index 4080157cc..ed885c082 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -1,12 +1,15 @@ package api import ( + "bufio" "bytes" "encoding/json" "errors" + "fmt" "io" "net/http" "os" + "regexp" "sort" "strconv" "strings" @@ -53,9 +56,15 @@ const ( AppAnalysisIssuesRoot = AppAnalysisRoot + "/issues" ) +// Manifest markers. +// The GS=\x1D (group separator). const ( - IssueField = "issues" - DepField = "dependencies" + BeginMainMarker = "\x1DBEGIN-MAIN\x1D" + EndMainMarker = "\x1DEND-MAIN\x1D" + BeginIssuesMarker = "\x1DBEGIN-ISSUES\x1D" + EndIssuesMarker = "\x1DEND-ISSUES\x1D" + BeginDepsMarker = "\x1DBEGIN-DEPS\x1D" + EndDepsMarker = "\x1DEND-DEPS\x1D" ) // AnalysisHandler handles analysis resource routes. @@ -315,9 +324,20 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) { // @summary Create an analysis. // @description Create an analysis. // @description Form fields: -// @description - file: file that contains the api.Analysis resource. -// @description - issues: file that multiple api.Issue resources. -// @description - dependencies: file that multiple api.TechDependency resources. +// @description file: A manifest file that contains 3 sections +// @description containing documents delimited by markers. +// @description The manifest must contain ALL markers even when sections are empty. +// @description Note: `^]` = `\x1D` = GS (group separator). +// @description Section markers: +// @description ^]BEGIN-MAIN^] +// @description ^]END-MAIN^] +// @description ^]BEGIN-ISSUES^] +// @description ^]END-ISSUES^] +// @description ^]BEGIN-DEPS^] +// @description ^]END-DEPS^] +// @description The encoding must be: +// @description - application/json +// @description - application/x-yaml // @tags analyses // @produce json // @success 201 {object} api.Analysis @@ -337,32 +357,40 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { return } } + db := h.DB(ctx) // - // Analysis - input, err := ctx.FormFile(FileField) + // Manifest + fh := FileHandler{} + name := fmt.Sprintf("app.%d.manifest", id) + file, err := fh.create(ctx, name) if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - reader, err := input.Open() + defer func() { + err = fh.delete(ctx, file) + if err != nil { + _ = ctx.Error(err) + } + }() + reader := &ManifestReader{} + f, err := reader.open(file.Path, BeginMainMarker, EndMainMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding := input.Header.Get(ContentType) - d, err := h.Decoder(ctx, encoding, reader) + d, err := h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - r := Analysis{} - err = d.Decode(&r) + r := &Analysis{} + err = d.Decode(r) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -371,7 +399,6 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { analysis := r.Model() analysis.ApplicationID = id analysis.CreateUser = h.BaseHandler.CurrentUser(ctx) - db := h.DB(ctx) db.Logger = db.Logger.LogMode(logger.Error) err = db.Create(analysis).Error if err != nil { @@ -380,23 +407,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Issues - input, err = ctx.FormFile(IssueField) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = input.Open() + reader = &ManifestReader{} + f, err = reader.open(file.Path, BeginIssuesMarker, EndIssuesMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + d, err = h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -425,23 +446,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Dependencies - input, err = ctx.FormFile(DepField) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = input.Open() + reader = &ManifestReader{} + f, err = reader.open(file.Path, BeginDepsMarker, EndDepsMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + d, err = h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -2860,3 +2875,116 @@ func (r *yamlEncoder) embed(object any) encoder { r.write(s) return r } + +// ManifestReader analysis manifest reader. +// The manifest contains 3 sections containing documents delimited by markers. +// The manifest must contain ALL markers even when sections are empty. +// Note: `^]` = `\x1D` = GS (group separator). +// Section markers: +// +// ^]BEGIN-MAIN^] +// ^]END-MAIN^] +// ^]BEGIN-ISSUES^] +// ^]END-ISSUES^] +// ^]BEGIN-DEPS^] +// ^]END-DEPS^] +type ManifestReader struct { + file *os.File + marker map[string]int64 + begin int64 + end int64 + read int64 +} + +// scan manifest and catalog position of markers. +func (r *ManifestReader) scan(path string) (err error) { + if r.marker != nil { + return + } + r.file, err = os.Open(path) + if err != nil { + return + } + defer func() { + _ = r.file.Close() + }() + pattern, err := regexp.Compile(`^\x1D[A-Z-]+\x1D$`) + if err != nil { + return + } + p := int64(0) + r.marker = make(map[string]int64) + scanner := bufio.NewScanner(r.file) + for scanner.Scan() { + content := scanner.Text() + matched := strings.TrimSpace(content) + if pattern.Match([]byte(matched)) { + r.marker[matched] = p + } + p += int64(len(content)) + p++ + } + + return +} + +// open returns a read delimited by the specified markers. +func (r *ManifestReader) open(path, begin, end string) (reader io.ReadCloser, err error) { + found := false + err = r.scan(path) + if err != nil { + return + } + r.begin, found = r.marker[begin] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", begin), + } + return + } + r.end, found = r.marker[end] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", end), + } + return + } + if r.begin >= r.end { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s must preceed %s.", begin, end), + } + return + } + r.begin += int64(len(begin)) + r.begin++ + r.read = r.end - r.begin + r.file, err = os.Open(path) + if err != nil { + return + } + _, err = r.file.Seek(r.begin, io.SeekStart) + reader = r + return +} + +// Read bytes. +func (r *ManifestReader) Read(b []byte) (n int, err error) { + n, err = r.file.Read(b) + if n == 0 || err != nil { + return + } + if int64(n) > r.read { + n = int(r.read) + } + r.read -= int64(n) + if n < 1 { + err = io.EOF + } + return +} + +// Close the reader. +func (r *ManifestReader) Close() (err error) { + err = r.file.Close() + return +} diff --git a/api/file.go b/api/file.go index 5e280172f..74fc17aae 100644 --- a/api/file.go +++ b/api/file.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" "github.com/konveyor/tackle2-hub/model" ) @@ -70,52 +71,11 @@ func (h FileHandler) List(ctx *gin.Context) { // @router /files [post] // @param name path string true "File name" func (h FileHandler) Create(ctx *gin.Context) { - var err error - input, err := ctx.FormFile(FileField) + m, err := h.create(ctx, ctx.Param(ID)) if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - m := &model.File{} - m.Name = ctx.Param(ID) - m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(&m) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } - defer func() { - if err != nil { - h.Status(ctx, http.StatusInternalServerError) - _ = h.DB(ctx).Delete(&m) - return - } - }() - reader, err := input.Open() - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - defer func() { - _ = reader.Close() - }() - writer, err := os.Create(m.Path) - if err != nil { - return - } - defer func() { - _ = writer.Close() - }() - _, err = io.Copy(writer, reader) - if err != nil { - return - } - err = os.Chmod(m.Path, 0666) - if err != nil { - return - } r := File{} r.With(m) h.Respond(ctx, http.StatusCreated, r) @@ -224,20 +184,126 @@ func (h FileHandler) Delete(ctx *gin.Context) { _ = ctx.Error(err) return } - err = os.Remove(m.Path) + err = h.delete(ctx, m) if err != nil { - if !os.IsNotExist(err) { - _ = ctx.Error(err) + _ = ctx.Error(err) + return + } + + h.Status(ctx, http.StatusNoContent) +} + +// create a file. +func (h FileHandler) create(ctx *gin.Context, name string) (m *model.File, err error) { + mode := ctx.ContentType() + switch mode { + case binding.MIMEMultipartPOSTForm: + m, err = h.createMultipart(ctx, name) + case binding.MIMEYAML: + m, err = h.createBody(ctx, name, binding.MIMEYAML) + default: + m, err = h.createBody(ctx, name, binding.MIMEJSON) + } + return +} + +// create a file with multipart form. +func (h FileHandler) createMultipart(ctx *gin.Context, name string) (m *model.File, err error) { + input, err := ctx.FormFile(FileField) + if err != nil { + err = &BadRequestError{err.Error()} + return + } + m = &model.File{} + m.Name = name + m.Encoding = input.Header.Get(ContentType) + m.CreateUser = h.BaseHandler.CurrentUser(ctx) + db := h.DB(ctx) + err = db.Create(&m).Error + if err != nil { + return + } + defer func() { + if err != nil { + h.Status(ctx, http.StatusInternalServerError) + _ = db.Delete(&m) return } + }() + reader, err := input.Open() + if err != nil { + err = &BadRequestError{err.Error()} + return } - err = h.DB(ctx).Delete(m).Error + defer func() { + _ = reader.Close() + }() + writer, err := os.Create(m.Path) if err != nil { - _ = ctx.Error(err) return } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + return + } + err = os.Chmod(m.Path, 0666) + if err != nil { + return + } + return +} - h.Status(ctx, http.StatusNoContent) +// create a file with request body. +func (h FileHandler) createBody(ctx *gin.Context, name, encoding string) (m *model.File, err error) { + m = &model.File{} + m.Name = name + m.Encoding = encoding + m.CreateUser = h.BaseHandler.CurrentUser(ctx) + db := h.DB(ctx) + err = db.Create(&m).Error + if err != nil { + return + } + defer func() { + if err != nil { + h.Status(ctx, http.StatusInternalServerError) + _ = db.Delete(&m) + return + } + }() + reader := ctx.Request.Body + writer, err := os.Create(m.Path) + if err != nil { + return + } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + return + } + err = os.Chmod(m.Path, 0666) + if err != nil { + return + } + return +} + +// delete the specified file. +func (h FileHandler) delete(ctx *gin.Context, m *model.File) (err error) { + err = os.Remove(m.Path) + if err != nil { + if !os.IsNotExist(err) { + return + } + } + db := h.DB(ctx) + err = db.Delete(m).Error + return } // File REST resource. @@ -245,6 +311,7 @@ type File struct { Resource `yaml:",inline"` Name string `json:"name"` Path string `json:"path"` + Encoding string `yaml:"encoding,omitempty"` Expiration *time.Time `json:"expiration,omitempty"` } @@ -253,5 +320,6 @@ func (r *File) With(m *model.File) { r.Resource.With(&m.Model) r.Name = m.Name r.Path = m.Path + r.Encoding = m.Encoding r.Expiration = m.Expiration } diff --git a/binding/application.go b/binding/application.go index ec7406f43..7266987bc 100644 --- a/binding/application.go +++ b/binding/application.go @@ -1,16 +1,12 @@ package binding import ( - "bytes" "errors" - "io" - "net/http" "strconv" - mime "github.com/gin-gonic/gin/binding" + "github.com/gin-gonic/gin/binding" liberr "github.com/jortel/go-utils/error" "github.com/konveyor/tackle2-hub/api" - "gopkg.in/yaml.v2" ) // Application API. @@ -316,30 +312,38 @@ type Analysis struct { appId uint } -// Create an analysis report. -func (h *Analysis) Create(r *api.Analysis, encoding string, issues, deps io.Reader) (err error) { +// Create an analysis report using the manifest at the specified path. +// The manifest contains 3 sections containing documents delimited by markers. +// The manifest must contain ALL markers even when sections are empty. +// Note: `^]` = `\x1D` = GS (group separator). +// Section markers: +// +// ^]BEGIN-MAIN^] +// ^]END-MAIN^] +// ^]BEGIN-ISSUES^] +// ^]END-ISSUES^] +// ^]BEGIN-DEPS^] +// ^]END-DEPS^] +// +// The encoding must be: +// - application/json +// - application/x-yaml +func (h *Analysis) Create(manifest, encoding string) (r *api.Analysis, err error) { + switch encoding { + case "": + encoding = binding.MIMEJSON + case binding.MIMEJSON, + binding.MIMEYAML: + default: + err = liberr.New( + "Encoding: %s not supported", + encoding) + } + r = &api.Analysis{} path := Path(api.AppAnalysesRoot).Inject(Params{api.ID: h.appId}) - b, _ := yaml.Marshal(r) - err = h.client.FileSend( - path, - http.MethodPost, - []Field{ - { - Name: api.FileField, - Reader: bytes.NewReader(b), - Encoding: mime.MIMEYAML, - }, - { - Name: api.IssueField, - Encoding: encoding, - Reader: issues, - }, - { - Name: api.DepField, - Encoding: encoding, - Reader: deps, - }, - }, - r) + err = h.client.FilePostEncoded(path, manifest, r, encoding) + if err != nil { + return + } return } diff --git a/binding/client.go b/binding/client.go index 78ae07b85..a5976eb9e 100644 --- a/binding/client.go +++ b/binding/client.go @@ -457,11 +457,19 @@ func (r *Client) FileGet(path, destination string) (err error) { // FilePost uploads a file. // Returns the created File resource. func (r *Client) FilePost(path, source string, object any) (err error) { + err = r.FilePostEncoded(path, source, object, "") + return +} + +// FilePostEncoded uploads a file. +// Returns the created File resource. +func (r *Client) FilePostEncoded(path, source string, object any, encoding string) (err error) { if source == "" { fields := []Field{ { - Name: api.FileField, - Reader: bytes.NewReader([]byte{}), + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPost, fields, object) @@ -478,8 +486,9 @@ func (r *Client) FilePost(path, source string, object any) (err error) { } fields := []Field{ { - Name: api.FileField, - Path: source, + Name: api.FileField, + Path: source, + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPost, fields, object) @@ -489,11 +498,19 @@ func (r *Client) FilePost(path, source string, object any) (err error) { // FilePut uploads a file. // Returns the created File resource. func (r *Client) FilePut(path, source string, object any) (err error) { + err = r.FilePutEncoded(path, source, object, "") + return +} + +// FilePutEncoded uploads a file. +// Returns the created File resource. +func (r *Client) FilePutEncoded(path, source string, object any, encoding string) (err error) { if source == "" { fields := []Field{ { - Name: api.FileField, - Reader: bytes.NewReader([]byte{}), + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPut, fields, object) @@ -510,8 +527,9 @@ func (r *Client) FilePut(path, source string, object any) (err error) { } fields := []Field{ { - Name: api.FileField, - Path: source, + Name: api.FileField, + Path: source, + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPut, fields, object) diff --git a/binding/file.go b/binding/file.go index 22bdcb80b..86bd3d94b 100644 --- a/binding/file.go +++ b/binding/file.go @@ -42,17 +42,29 @@ func (h *File) Touch(name string) (r *api.File, err error) { // Post uploads a file. func (h *File) Post(source string) (r *api.File, err error) { + r, err = h.PostEncoded(source, "") + return +} + +// PostEncoded uploads a file. +func (h *File) PostEncoded(source string, encoding string) (r *api.File, err error) { r = &api.File{} path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) - err = h.client.FilePost(path, source, r) + err = h.client.FilePostEncoded(path, source, r, encoding) return } // Put uploads a file. func (h *File) Put(source string) (r *api.File, err error) { + r, err = h.PutEncoded(source, "") + return +} + +// PutEncoded uploads a file. +func (h *File) PutEncoded(source string, encoding string) (r *api.File, err error) { r = &api.File{} path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) - err = h.client.FilePut(path, source, r) + err = h.client.FilePutEncoded(path, source, r, encoding) return } diff --git a/hack/add/analysis.sh b/hack/add/analysis.sh index bb2cb79e9..568ff3341 100755 --- a/hack/add/analysis.sh +++ b/hack/add/analysis.sh @@ -3,20 +3,18 @@ set -e host="${HOST:-localhost:8080}" -app="${1:-1}" +appId="${1:-1}" nRuleSet="${2:-10}" nIssue="${3:-10}" nIncident="${4:-25}" -aPath="/tmp/analysis.yaml" -iPath="/tmp/issues.yaml" -dPath="/tmp/deps.yaml" +tmp=/tmp/${self}-${pid} +file="/tmp/manifest.yaml" -echo " Application: ${app}" +echo " Application: ${appId}" echo " RuleSets: ${nRuleSet}" echo " Issues: ${nIssue}" echo " Incidents: ${nIncident}" -echo " Issues path: ${iPath}" -echo " Deps path: ${dPath}" +echo " Manifest path: ${file}" sources=( konveyor.io/source=oraclejdk @@ -62,11 +60,18 @@ konveyor.io/target=hibernate konveyor.io/target=jbpm ) +# +# Analysis +# +printf "\x1DBEGIN-MAIN\x1D\n" > ${file} +echo -n "--- +commit: "1234" +" >> ${file} +printf "\x1DEND-MAIN\x1D\n" >> ${file} # # Issues # -file=${iPath} -echo "" > ${file} +printf "\x1DBEGIN-ISSUES\x1D\n" >> ${file} for r in $(seq 1 ${nRuleSet}) do for i in $(seq 1 ${nIssue}) @@ -150,17 +155,18 @@ fi done done done +printf "\x1DEND-ISSUES\x1D +\x1DBEGIN-DEPS\x1D\n" >> ${file} # # Deps # -file=${dPath} echo -n "--- name: github.com/jboss version: 4.0 labels: - konveyor.io/language=java - konveyor.io/otherA=dog -" > ${file} +" >> ${file} echo -n "--- name: github.com/jboss version: 5.0 @@ -192,23 +198,28 @@ echo -n "--- name: github.com/java version: 8 " >> ${file} -# -# Analysis -# -file=${aPath} -echo -n "--- -commit: "42b22a90" -issues: -dependencies: -" > ${file} +printf "\x1DEND-DEPS\x1D\n" >> ${file} -echo "Report CREATED" +echo "Manifest (file) GENERATED: ${file}" -mime="application/x-yaml" +# +# Post manifest. +code=$(curl -kSs -o ${tmp} -w "%{http_code}" \ + -F "file=@${file};type=application/x-yaml" \ + -H 'Accept:application/x-yaml' \ + http://${host}/applications/${appId}/analyses) +if [ ! $? -eq 0 ] +then + exit $? +fi +case ${code} in + 201) + echo "Analysis: created." + cat ${tmp} + ;; + *) + echo "Analysis create - FAILED: ${code}." + cat ${tmp} + exit 1 +esac -curl \ - -F "file=@${aPath};type=${mime}" \ - -F "issues=@${iPath};type=${mime}" \ - -F "dependencies=@${dPath};type=${mime}" \ - ${host}/applications/${app}/analyses \ - -H "Accept:${mime}" diff --git a/migration/pkg.go b/migration/pkg.go index f80ed8e48..5caaecbd8 100644 --- a/migration/pkg.go +++ b/migration/pkg.go @@ -7,6 +7,7 @@ import ( v12 "github.com/konveyor/tackle2-hub/migration/v12" v13 "github.com/konveyor/tackle2-hub/migration/v13" v14 "github.com/konveyor/tackle2-hub/migration/v14" + v15 "github.com/konveyor/tackle2-hub/migration/v15" v2 "github.com/konveyor/tackle2-hub/migration/v2" v3 "github.com/konveyor/tackle2-hub/migration/v3" v4 "github.com/konveyor/tackle2-hub/migration/v4" @@ -56,5 +57,6 @@ func All() []Migration { v12.Migration{}, v13.Migration{}, v14.Migration{}, + v15.Migration{}, } } diff --git a/migration/v15/migrate.go b/migration/v15/migrate.go new file mode 100644 index 000000000..1fe82e975 --- /dev/null +++ b/migration/v15/migrate.go @@ -0,0 +1,20 @@ +package v15 + +import ( + "github.com/jortel/go-utils/logr" + "github.com/konveyor/tackle2-hub/migration/v15/model" + "gorm.io/gorm" +) + +var log = logr.WithName("migration|v15") + +type Migration struct{} + +func (r Migration) Apply(db *gorm.DB) (err error) { + err = db.AutoMigrate(r.Models()...) + return +} + +func (r Migration) Models() []interface{} { + return model.All() +} diff --git a/migration/v15/model/analysis.go b/migration/v15/model/analysis.go new file mode 100644 index 000000000..bc33da893 --- /dev/null +++ b/migration/v15/model/analysis.go @@ -0,0 +1,170 @@ +package model + +import ( + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +// Analysis report. +type Analysis struct { + Model + Effort int + Commit string + Archived bool + Summary []ArchivedIssue `gorm:"type:json;serializer:json"` + Issues []Issue `gorm:"constraint:OnDelete:CASCADE"` + Dependencies []TechDependency `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID uint `gorm:"index;not null"` + Application *Application +} + +// TechDependency report dependency. +type TechDependency struct { + Model + Provider string `gorm:"uniqueIndex:depA"` + Name string `gorm:"uniqueIndex:depA"` + Version string `gorm:"uniqueIndex:depA"` + SHA string `gorm:"uniqueIndex:depA"` + Indirect bool + Labels []string `gorm:"type:json;serializer:json"` + AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` + Analysis *Analysis +} + +// Issue report issue (violation). +type Issue struct { + Model + RuleSet string `gorm:"uniqueIndex:issueA;not null"` + Rule string `gorm:"uniqueIndex:issueA;not null"` + Name string `gorm:"index"` + Description string + Category string `gorm:"index;not null"` + Incidents []Incident `gorm:"foreignKey:IssueID;constraint:OnDelete:CASCADE"` + Links []Link `gorm:"type:json;serializer:json"` + Facts json.Map `gorm:"type:json;serializer:json"` + Labels []string `gorm:"type:json;serializer:json"` + Effort int `gorm:"index;not null"` + AnalysisID uint `gorm:"index;uniqueIndex:issueA;not null"` + Analysis *Analysis +} + +// Incident report an issue incident. +type Incident struct { + Model + File string `gorm:"index;not null"` + Line int + Message string + CodeSnip string + Facts json.Map `gorm:"type:json;serializer:json"` + IssueID uint `gorm:"index;not null"` + Issue *Issue +} + +// RuleSet - Analysis ruleset. +type RuleSet struct { + Model + UUID *string `gorm:"uniqueIndex"` + Kind string + Name string `gorm:"uniqueIndex;not null"` + Description string + Repository Repository `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity + Rules []Rule `gorm:"constraint:OnDelete:CASCADE"` + DependsOn []RuleSet `gorm:"many2many:RuleSetDependencies;constraint:OnDelete:CASCADE"` +} + +func (r *RuleSet) Builtin() bool { + return r.UUID != nil +} + +// BeforeUpdate hook to avoid cyclic dependencies. +func (r *RuleSet) BeforeUpdate(db *gorm.DB) (err error) { + seen := make(map[uint]bool) + var nextDeps []RuleSet + var nextRuleSetIDs []uint + for _, dep := range r.DependsOn { + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + for len(nextRuleSetIDs) != 0 { + result := db.Preload("DependsOn").Where("ID IN ?", nextRuleSetIDs).Find(&nextDeps) + if result.Error != nil { + err = result.Error + return + } + nextRuleSetIDs = nextRuleSetIDs[:0] + for _, nextDep := range nextDeps { + for _, dep := range nextDep.DependsOn { + if seen[dep.ID] { + continue + } + if dep.ID == r.ID { + err = DependencyCyclicError{} + return + } + seen[dep.ID] = true + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + } + } + + return +} + +// Rule - Analysis rule. +type Rule struct { + Model + Name string + Description string + Labels []string `gorm:"type:json;serializer:json"` + RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` + RuleSet *RuleSet + FileID *uint `gorm:"uniqueIndex:RuleA" ref:"file"` + File *File +} + +// Target - analysis rule selector. +type Target struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex;not null"` + Description string + Provider string + Choice bool + Labels []TargetLabel `gorm:"type:json;serializer:json"` + ImageID uint `gorm:"index" ref:"file"` + Image *File + RuleSetID *uint `gorm:"index"` + RuleSet *RuleSet +} + +func (r *Target) Builtin() bool { + return r.UUID != nil +} + +// +// JSON Fields. +// + +// ArchivedIssue resource created when issues are archived. +type ArchivedIssue struct { + RuleSet string `json:"ruleSet"` + Rule string `json:"rule"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Description string `json:"description,omitempty" yaml:",omitempty"` + Category string `json:"category"` + Effort int `json:"effort"` + Incidents int `json:"incidents"` +} + +// Link URL link. +type Link struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` +} + +// TargetLabel - label format specific to Targets +type TargetLabel struct { + Name string `json:"name"` + Label string `json:"label"` +} diff --git a/migration/v15/model/application.go b/migration/v15/model/application.go new file mode 100644 index 000000000..cb363b78d --- /dev/null +++ b/migration/v15/model/application.go @@ -0,0 +1,312 @@ +package model + +import ( + "fmt" + "sync" + "time" + + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +type Application struct { + Model + BucketOwner + Name string `gorm:"index;unique;not null"` + Description string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Repository Repository `gorm:"type:json;serializer:json"` + Binary string + Facts []Fact `gorm:"constraint:OnDelete:CASCADE"` + Comments string + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ApplicationTags"` + Identities []Identity `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` + BusinessServiceID *uint `gorm:"index"` + BusinessService *BusinessService + OwnerID *uint `gorm:"index"` + Owner *Stakeholder `gorm:"foreignKey:OwnerID"` + Contributors []Stakeholder `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + Analyses []Analysis `gorm:"constraint:OnDelete:CASCADE"` + MigrationWaveID *uint `gorm:"index"` + MigrationWave *MigrationWave + Ticket *Ticket `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +type Fact struct { + ApplicationID uint `gorm:"<-:create;primaryKey"` + Key string `gorm:"<-:create;primaryKey"` + Source string `gorm:"<-:create;primaryKey;not null"` + Value any `gorm:"type:json;not null;serializer:json"` + Application *Application +} + +// ApplicationTag represents a row in the join table for the +// many-to-many relationship between Applications and Tags. +type ApplicationTag struct { + ApplicationID uint `gorm:"primaryKey"` + TagID uint `gorm:"primaryKey"` + Source string `gorm:"primaryKey;not null"` + Application Application `gorm:"constraint:OnDelete:CASCADE"` + Tag Tag `gorm:"constraint:OnDelete:CASCADE"` +} + +// TableName must return "ApplicationTags" to ensure compatibility +// with the autogenerated join table name. +func (ApplicationTag) TableName() string { + return "ApplicationTags" +} + +// depMutex ensures Dependency.Create() is not executed concurrently. +var depMutex sync.Mutex + +type Dependency struct { + Model + ToID uint `gorm:"index"` + To *Application `gorm:"foreignKey:ToID;constraint:OnDelete:CASCADE"` + FromID uint `gorm:"index"` + From *Application `gorm:"foreignKey:FromID;constraint:OnDelete:CASCADE"` +} + +// Create a dependency synchronized using a mutex. +func (r *Dependency) Create(db *gorm.DB) (err error) { + depMutex.Lock() + defer depMutex.Unlock() + err = db.Create(r).Error + return +} + +// BeforeCreate detects cyclic dependencies. +func (r *Dependency) BeforeCreate(db *gorm.DB) (err error) { + var nextDeps []*Dependency + var nextAppsIDs []uint + nextAppsIDs = append(nextAppsIDs, r.FromID) + for len(nextAppsIDs) != 0 { + db.Where("ToID IN ?", nextAppsIDs).Find(&nextDeps) + nextAppsIDs = nextAppsIDs[:0] // empty array, but keep capacity + for _, nextDep := range nextDeps { + if nextDep.FromID == r.ToID { + err = DependencyCyclicError{} + return + } + nextAppsIDs = append(nextAppsIDs, nextDep.FromID) + } + } + + return +} + +// DependencyCyclicError reports cyclic Dependency error. +type DependencyCyclicError struct{} + +func (e DependencyCyclicError) Error() string { + return "Cyclic dependencies are not permitted." +} + +type BusinessService struct { + Model + Name string `gorm:"index;unique;not null"` + Description string + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + StakeholderID *uint `gorm:"index"` + Stakeholder *Stakeholder +} + +type JobFunction struct { + Model + UUID *string `gorm:"uniqueIndex"` + Username string + Name string `gorm:"index;unique;not null"` + Stakeholders []Stakeholder `gorm:"constraint:OnDelete:SET NULL"` +} + +type Stakeholder struct { + Model + Name string `gorm:"not null;"` + Email string `gorm:"index;unique;not null"` + Groups []StakeholderGroup `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + BusinessServices []BusinessService `gorm:"constraint:OnDelete:SET NULL"` + JobFunctionID *uint `gorm:"index"` + JobFunction *JobFunction + Owns []Application `gorm:"foreignKey:OwnerID;constraint:OnDelete:SET NULL"` + Contributes []Application `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` +} + +type StakeholderGroup struct { + Model + Name string `gorm:"index;unique;not null"` + Username string + Description string + Stakeholders []Stakeholder `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type MigrationWave struct { + Model + Name string `gorm:"uniqueIndex:MigrationWaveA"` + StartDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + EndDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + Stakeholders []Stakeholder `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Archetype struct { + Model + Name string + Description string + Comments string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` + CriteriaTags []Tag `gorm:"many2many:ArchetypeCriteriaTags;constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ArchetypeTags;constraint:OnDelete:CASCADE"` + Stakeholders []Stakeholder `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Tag struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex:tagA;not null"` + Username string + CategoryID uint `gorm:"uniqueIndex:tagA;index;not null"` + Category TagCategory +} + +type TagCategory struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"index;unique;not null"` + Username string + Rank uint + Color string + Tags []Tag `gorm:"foreignKey:CategoryID;constraint:OnDelete:CASCADE"` +} + +type Ticket struct { + Model + // Kind of ticket in the external tracker. + Kind string `gorm:"not null"` + // Parent resource that this ticket should belong to in the tracker. (e.g. Jira project) + Parent string `gorm:"not null"` + // Custom fields to send to the tracker when creating the ticket + Fields json.Map `gorm:"type:json;serializer:json"` + // Whether the last attempt to do something with the ticket reported an error + Error bool + // Error message, if any + Message string + // Whether the ticket was created in the external tracker + Created bool + // Reference id in external tracker + Reference string + // URL to ticket in external tracker + Link string + // Status of ticket in external tracker + Status string + LastUpdated time.Time + Application *Application + ApplicationID uint `gorm:"uniqueIndex:ticketA;not null"` + Tracker *Tracker + TrackerID uint `gorm:"uniqueIndex:ticketA;not null"` +} + +type Tracker struct { + Model + Name string `gorm:"index;unique;not null"` + URL string + Kind string + Identity *Identity + IdentityID uint + Connected bool + LastUpdated time.Time + Message string + Insecure bool + Tickets []Ticket +} + +type Import struct { + Model + Filename string + ApplicationName string + BusinessService string + Comments string + Dependency string + DependencyDirection string + Description string + ErrorMessage string + IsValid bool + RecordType1 string + ImportSummary ImportSummary + ImportSummaryID uint `gorm:"index"` + Processed bool + ImportTags []ImportTag `gorm:"constraint:OnDelete:CASCADE"` + BinaryGroup string + BinaryArtifact string + BinaryVersion string + BinaryPackaging string + RepositoryKind string + RepositoryURL string + RepositoryBranch string + RepositoryPath string + Owner string + Contributors string +} + +func (r *Import) AsMap() (m map[string]any) { + m = make(map[string]any) + m["filename"] = r.Filename + m["applicationName"] = r.ApplicationName + // "Application Name" is necessary in order for + // the UI to display the error report correctly. + m["Application Name"] = r.ApplicationName + m["businessService"] = r.BusinessService + m["comments"] = r.Comments + m["dependency"] = r.Dependency + m["dependencyDirection"] = r.DependencyDirection + m["description"] = r.Description + m["errorMessage"] = r.ErrorMessage + m["isValid"] = r.IsValid + m["processed"] = r.Processed + m["recordType1"] = r.RecordType1 + for i, tag := range r.ImportTags { + m[fmt.Sprintf("category%v", i+1)] = tag.Category + m[fmt.Sprintf("tag%v", i+1)] = tag.Name + } + return +} + +type ImportSummary struct { + Model + Content []byte + Filename string + ImportStatus string + Imports []Import `gorm:"constraint:OnDelete:CASCADE"` + CreateEntities bool +} + +type ImportTag struct { + Model + Name string + Category string + ImportID uint `gorm:"index"` + Import *Import +} + +// +// JSON Fields. +// + +// Repository represents an SCM repository. +type Repository struct { + Kind string `json:"kind"` + URL string `json:"url"` + Branch string `json:"branch"` + Tag string `json:"tag"` + Path string `json:"path"` +} diff --git a/migration/v15/model/assessment.go b/migration/v15/model/assessment.go new file mode 100644 index 000000000..0b51e714d --- /dev/null +++ b/migration/v15/model/assessment.go @@ -0,0 +1,102 @@ +package model + +type Questionnaire struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"unique"` + Description string + Required bool + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +// Builtin returns true if this is a Konveyor-provided questionnaire. +func (r *Questionnaire) Builtin() bool { + return r.UUID != nil +} + +type Assessment struct { + Model + ApplicationID *uint `gorm:"uniqueIndex:AssessmentA"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex:AssessmentB"` + Archetype *Archetype + QuestionnaireID uint `gorm:"uniqueIndex:AssessmentA;uniqueIndex:AssessmentB"` + Questionnaire Questionnaire + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` + Stakeholders []Stakeholder `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Review struct { + Model + BusinessCriticality uint `gorm:"not null"` + EffortEstimate string `gorm:"not null"` + ProposedAction string `gorm:"not null"` + WorkPriority uint `gorm:"not null"` + Comments string + ApplicationID *uint `gorm:"uniqueIndex"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex"` + Archetype *Archetype +} + +// +// JSON Fields. +// + +// Section represents a group of questions in a questionnaire. +type Section struct { + Order uint `json:"order" yaml:"order"` + Name string `json:"name" yaml:"name"` + Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` + Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +} + +// Question represents a question in a questionnaire. +type Question struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Explanation string `json:"explanation" yaml:"explanation"` + IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` + ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` + Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` +} + +// Answer represents an answer to a question in a questionnaire. +type Answer struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` + Rationale string `json:"rationale" yaml:"rationale"` + Mitigation string `json:"mitigation" yaml:"mitigation"` + ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` + AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` + Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` + AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` +} + +// CategorizedTag represents a human-readable pair of category and tag. +type CategorizedTag struct { + Category string `json:"category" yaml:"category"` + Tag string `json:"tag" yaml:"tag"` +} + +// RiskMessages contains messages to display for each risk level. +type RiskMessages struct { + Red string `json:"red" yaml:"red"` + Yellow string `json:"yellow" yaml:"yellow"` + Green string `json:"green" yaml:"green"` + Unknown string `json:"unknown" yaml:"unknown"` +} + +// Thresholds contains the threshold values for determining risk for the questionnaire. +type Thresholds struct { + Red uint `json:"red" yaml:"red"` + Yellow uint `json:"yellow" yaml:"yellow"` + Unknown uint `json:"unknown" yaml:"unknown"` +} diff --git a/migration/v15/model/core.go b/migration/v15/model/core.go new file mode 100644 index 000000000..c3b2a85b4 --- /dev/null +++ b/migration/v15/model/core.go @@ -0,0 +1,309 @@ +package model + +import ( + "os" + "path" + "time" + + "github.com/google/uuid" + liberr "github.com/jortel/go-utils/error" + "github.com/konveyor/tackle2-hub/encryption" + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +// Model Base model. +type Model struct { + ID uint `gorm:"<-:create;primaryKey"` + CreateTime time.Time `gorm:"<-:create;autoCreateTime"` + CreateUser string `gorm:"<-:create"` + UpdateUser string +} + +// PK sequence. +type PK struct { + Kind string `gorm:"<-:create;primaryKey"` + LastID uint +} + +// Setting hub settings. +type Setting struct { + Model + Key string `gorm:"<-:create;uniqueIndex"` + Value any `gorm:"type:json;serializer:json"` +} + +// As unmarshalls the value of the Setting into the `ptr` parameter. +func (r *Setting) As(ptr any) (err error) { + bytes, err := json.Marshal(r.Value) + if err != nil { + err = liberr.Wrap(err) + } + err = json.Unmarshal(bytes, ptr) + if err != nil { + err = liberr.Wrap(err) + } + return +} + +type Bucket struct { + Model + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *Bucket) BeforeCreate(db *gorm.DB) (err error) { + if m.Path == "" { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + uid.String()) + err = os.MkdirAll(m.Path, 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + } + return +} + +type BucketOwner struct { + BucketID *uint `gorm:"index" ref:"bucket"` + Bucket *Bucket +} + +func (m *BucketOwner) BeforeCreate(db *gorm.DB) (err error) { + if !m.HasBucket() { + b := &Bucket{} + err = db.Create(b).Error + m.SetBucket(&b.ID) + } + return +} + +func (m *BucketOwner) SetBucket(id *uint) { + m.BucketID = id + m.Bucket = nil +} + +func (m *BucketOwner) HasBucket() (b bool) { + return m.BucketID != nil +} + +type File struct { + Model + Name string + Encoding string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *File) BeforeCreate(db *gorm.DB) (err error) { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + ".file", + uid.String()) + err = os.MkdirAll(path.Dir(m.Path), 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + return +} + +type Task struct { + Model + BucketOwner + Name string `gorm:"index"` + Kind string + Addon string `gorm:"index"` + Extensions []string `gorm:"type:json;serializer:json"` + State string `gorm:"index"` + Locator string `gorm:"index"` + Priority int + Policy TaskPolicy `gorm:"type:json;serializer:json"` + TTL TTL `gorm:"type:json;serializer:json"` + Data json.Data `gorm:"type:json;serializer:json"` + Started *time.Time + Terminated *time.Time + Errors []TaskError `gorm:"type:json;serializer:json"` + Events []TaskEvent `gorm:"type:json;serializer:json"` + Pod string `gorm:"index"` + Retries int + Attached []Attachment `gorm:"type:json;serializer:json" ref:"[]file"` + Report *TaskReport `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID *uint `gorm:"index"` + Application *Application + TaskGroupID *uint `gorm:"<-:create"` + TaskGroup *TaskGroup +} + +func (m *Task) BeforeCreate(db *gorm.DB) (err error) { + err = m.BucketOwner.BeforeCreate(db) + return +} + +type TaskReport struct { + Model + Status string + Total int + Completed int + Activity []string `gorm:"type:json;serializer:json"` + Errors []TaskError `gorm:"type:json;serializer:json"` + Attached []Attachment `gorm:"type:json;serializer:json" ref:"[]file"` + Result json.Data `gorm:"type:json;serializer:json"` + TaskID uint `gorm:"<-:create;uniqueIndex"` + Task *Task +} + +type TaskGroup struct { + Model + BucketOwner + Name string + Kind string + Addon string + Extensions []string `gorm:"type:json;serializer:json"` + State string + Priority int + Policy TaskPolicy `gorm:"type:json;serializer:json"` + Data json.Data `gorm:"type:json;serializer:json"` + List []Task `gorm:"type:json;serializer:json"` + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` +} + +// Proxy configuration. +// kind = (http|https) +type Proxy struct { + Model + Enabled bool + Kind string `gorm:"uniqueIndex"` + Host string `gorm:"not null"` + Port int + Excluded []string `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity +} + +// Identity represents and identity with a set of credentials. +type Identity struct { + Model + Kind string `gorm:"not null"` + Name string `gorm:"index;unique;not null"` + Description string + User string + Password string + Key string + Settings string + Proxies []Proxy `gorm:"constraint:OnDelete:SET NULL"` + Applications []Application `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` +} + +// Encrypt sensitive fields. +// The ref identity is used to determine when sensitive fields +// have changed and need to be (re)encrypted. +func (r *Identity) Encrypt(ref *Identity) (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != ref.Password { + if r.Password != "" { + r.Password, err = aes.Encrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Key != ref.Key { + if r.Key != "" { + r.Key, err = aes.Encrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Settings != ref.Settings { + if r.Settings != "" { + r.Settings, err = aes.Encrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + return +} + +// Decrypt sensitive fields. +func (r *Identity) Decrypt() (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != "" { + r.Password, err = aes.Decrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Key != "" { + r.Key, err = aes.Decrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Settings != "" { + r.Settings, err = aes.Decrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + return +} + +// +// JSON Fields. +// + +// Attachment file attachment. +type Attachment struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Activity int `json:"activity,omitempty" yaml:",omitempty"` +} + +// TaskError used in Task.Errors. +type TaskError struct { + Severity string `json:"severity"` + Description string `json:"description"` +} + +// TaskEvent task event. +type TaskEvent struct { + Kind string `json:"kind"` + Count int `json:"count"` + Reason string `json:"reason,omitempty" yaml:",omitempty"` + Last time.Time `json:"last"` +} + +// TaskPolicy scheduling policy. +type TaskPolicy struct { + Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` + PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` + PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` +} + +// TTL time-to-live. +type TTL struct { + Created int `json:"created,omitempty" yaml:",omitempty"` + Pending int `json:"pending,omitempty" yaml:",omitempty"` + Running int `json:"running,omitempty" yaml:",omitempty"` + Succeeded int `json:"succeeded,omitempty" yaml:",omitempty"` + Failed int `json:"failed,omitempty" yaml:",omitempty"` +} diff --git a/migration/v15/model/mod.patch b/migration/v15/model/mod.patch new file mode 100644 index 000000000..e2a2ae8ae --- /dev/null +++ b/migration/v15/model/mod.patch @@ -0,0 +1,11 @@ +diff -ruN migration/v14/model/core.go migration/v15/model/core.go +--- migration/v14/model/core.go 2024-09-20 04:44:49.750736163 -0700 ++++ migration/v15/model/core.go 2024-09-20 04:47:13.750375198 -0700 +@@ -95,6 +95,7 @@ + type File struct { + Model + Name string ++ Encoding string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time + } diff --git a/migration/v15/model/pkg.go b/migration/v15/model/pkg.go new file mode 100644 index 000000000..8f612b488 --- /dev/null +++ b/migration/v15/model/pkg.go @@ -0,0 +1,56 @@ +package model + +import ( + "github.com/konveyor/tackle2-hub/settings" +) + +var ( + Settings = &settings.Settings +) + +// JSON field (data) type. +type JSON = []byte + +// All builds all models. +// Models are enumerated such that each are listed after +// all the other models on which they may depend. +func All() []any { + return []any{ + Application{}, + TechDependency{}, + Incident{}, + Analysis{}, + Issue{}, + Bucket{}, + BusinessService{}, + Dependency{}, + File{}, + Fact{}, + Identity{}, + Import{}, + ImportSummary{}, + ImportTag{}, + JobFunction{}, + MigrationWave{}, + PK{}, + Proxy{}, + Review{}, + Setting{}, + RuleSet{}, + Rule{}, + Stakeholder{}, + StakeholderGroup{}, + Tag{}, + TagCategory{}, + Target{}, + Task{}, + TaskGroup{}, + TaskReport{}, + Ticket{}, + Tracker{}, + ApplicationTag{}, + Questionnaire{}, + Assessment{}, + Archetype{}, + } +} diff --git a/model/pkg.go b/model/pkg.go index d6b546fb3..f1c5d0cf6 100644 --- a/model/pkg.go +++ b/model/pkg.go @@ -2,7 +2,7 @@ package model import ( "github.com/konveyor/tackle2-hub/migration/json" - "github.com/konveyor/tackle2-hub/migration/v14/model" + "github.com/konveyor/tackle2-hub/migration/v15/model" ) // Field (data) types. From 3972ae386c4a1fffa42e6afd67deb70c460d6923 Mon Sep 17 00:00:00 2001 From: David Zager Date: Thu, 26 Sep 2024 14:07:28 -0400 Subject: [PATCH 12/15] :seedling: .github correct path to workflow for image build (#751) Signed-off-by: David Zager --- .github/workflows/march-image-build-push.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/march-image-build-push.yml b/.github/workflows/march-image-build-push.yml index 25bbcf065..453a1f23f 100644 --- a/.github/workflows/march-image-build-push.yml +++ b/.github/workflows/march-image-build-push.yml @@ -19,9 +19,12 @@ jobs: runs-on: ubuntu-20.04 strategy: fail-fast: false - uses: konveyor/release-tools/build-push-images.yaml@main + uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main with: registry: "quay.io/konveyor" image_name: "tackle2-hub" containerfile: "./Dockerfile" architectures: '[ "amd64", "arm64" ]' + secrets: + registry_username: ${{ secrets.QUAY_PUBLISH_ROBOT }} + registry_password: ${{ secrets.QUAY_PUBLISH_TOKEN }} From 2bbcab058d3b3f00fbd652a94ed86c9348ea64dc Mon Sep 17 00:00:00 2001 From: David Zager Date: Thu, 26 Sep 2024 16:50:15 -0400 Subject: [PATCH 13/15] :seedling: properly define reusable workflow (#752) Signed-off-by: David Zager --- .github/workflows/march-image-build-push.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/march-image-build-push.yml b/.github/workflows/march-image-build-push.yml index 453a1f23f..e70724b8f 100644 --- a/.github/workflows/march-image-build-push.yml +++ b/.github/workflows/march-image-build-push.yml @@ -15,10 +15,6 @@ concurrency: jobs: push-quay: - name: Build and Push Manifest - runs-on: ubuntu-20.04 - strategy: - fail-fast: false uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main with: registry: "quay.io/konveyor" From d8211f6f24b12df8bda043792c30905f20bc9f76 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Thu, 10 Oct 2024 14:06:00 -0500 Subject: [PATCH 14/15] :sparkles: Run task pod as AnyUser. (#755) To support running the task pods as _AnyUser_ instead of root: - The task manager needs to no longer RunAs user root. - The /addon directory needs to be an _EmptyDir_. This is because the addon-analyzer Dockerfile cannot create the /addon directory as owned by the _AnyUser_. Signed-off-by: Jeff Ortel --- settings/addon.go | 13 ++++++++++--- task/manager.go | 20 ++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/settings/addon.go b/settings/addon.go index 94c30c57a..e022a7975 100644 --- a/settings/addon.go +++ b/settings/addon.go @@ -7,13 +7,16 @@ import ( ) const ( - EnvHubBaseURL = "HUB_BASE_URL" - EnvHubToken = "TOKEN" - EnvTask = "TASK" + EnvHubBaseURL = "HUB_BASE_URL" + EnvHubToken = "TOKEN" + EnvTask = "TASK" + EnvAddonHomeDir = "ADDON_HOME" ) // Addon settings. type Addon struct { + // HomeDir working directory. + HomeDir string // Hub settings. Hub struct { // URL for the hub API. @@ -27,6 +30,10 @@ type Addon struct { func (r *Addon) Load() (err error) { var found bool + r.HomeDir, found = os.LookupEnv(EnvAddonHomeDir) + if !found { + r.HomeDir = "/addon" + } r.Hub.URL, found = os.LookupEnv(EnvHubBaseURL) if !found { r.Hub.URL = "http://localhost:8080" diff --git a/task/manager.go b/task/manager.go index 5bd34fcf4..c939c3d03 100644 --- a/task/manager.go +++ b/task/manager.go @@ -83,6 +83,7 @@ const ( ) const ( + Addon = "addon" Shared = "shared" Cache = "cache" ) @@ -1643,6 +1644,12 @@ func (r *Task) specification( addon *crd.Addon, extensions []crd.Extension, secret *core.Secret) (specification core.PodSpec) { + addonDir := core.Volume{ + Name: Addon, + VolumeSource: core.VolumeSource{ + EmptyDir: &core.EmptyDirVolumeSource{}, + }, + } shared := core.Volume{ Name: Shared, VolumeSource: core.VolumeSource{ @@ -1670,6 +1677,7 @@ func (r *Task) specification( InitContainers: init, Containers: plain, Volumes: []core.Volume{ + addonDir, shared, cache, }, @@ -1683,7 +1691,6 @@ func (r *Task) containers( addon *crd.Addon, extensions []crd.Extension, secret *core.Secret) (init []core.Container, plain []core.Container) { - userid := int64(0) token := &core.EnvVarSource{ SecretKeyRef: &core.SecretKeySelector{ Key: settings.EnvHubToken, @@ -1707,11 +1714,12 @@ func (r *Task) containers( container := &plain[i] injector.Inject(container) r.propagateEnv(&plain[0], container) - container.SecurityContext = &core.SecurityContext{ - RunAsUser: &userid, - } container.VolumeMounts = append( container.VolumeMounts, + core.VolumeMount{ + Name: Addon, + MountPath: Settings.Addon.HomeDir, + }, core.VolumeMount{ Name: Shared, MountPath: Settings.Shared.Path, @@ -1722,6 +1730,10 @@ func (r *Task) containers( }) container.Env = append( container.Env, + core.EnvVar{ + Name: settings.EnvAddonHomeDir, + Value: Settings.Addon.HomeDir, + }, core.EnvVar{ Name: settings.EnvSharedPath, Value: Settings.Shared.Path, From 29deafefb37af3d34931e96a38f6a15c7e27023c Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Thu, 10 Oct 2024 15:31:35 -0500 Subject: [PATCH 15/15] :bug: Support reused connection for better performance. (#753) Fixes: https://issues.redhat.com/browse/MTA-4007 --------- Signed-off-by: Jeff Ortel --- Makefile | 3 + database/db_test.go | 76 +++++++++++++++- database/driver.go | 211 +++++++++++++++++++++++++++++++++++++++----- database/pkg.go | 2 +- 4 files changed, 269 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index b092ad5fd..77de6b71d 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,9 @@ endif test: go test -count=1 -v $(shell go list ./... | grep -v "hub/test") +test-db: + go test -count=1 -timeout=6h -v ./database... + # Run Hub REST API tests. test-api: HUB_BASE_URL=$(HUB_BASE_URL) go test -count=1 -p=1 -v -failfast ./test/api/... diff --git a/database/db_test.go b/database/db_test.go index dde8c6fae..3a4590f09 100644 --- a/database/db_test.go +++ b/database/db_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/konveyor/tackle2-hub/api" "github.com/konveyor/tackle2-hub/model" "gorm.io/gorm" "k8s.io/utils/env" @@ -13,6 +14,40 @@ import ( var N, _ = env.GetInt("TEST_CONCURRENT", 10) +func TestDriver(t *testing.T) { + pid := os.Getpid() + Settings.DB.Path = fmt.Sprintf("/tmp/driver-%d.db", pid) + defer func() { + _ = os.Remove(Settings.DB.Path) + }() + db, err := Open(true) + if err != nil { + panic(err) + } + key := "driver" + m := &model.Setting{Key: key, Value: "Test"} + // insert. + err = db.Create(m).Error + if err != nil { + panic(err) + } + // update + err = db.Save(m).Error + if err != nil { + panic(err) + } + // select + err = db.First(m, m.ID).Error + if err != nil { + panic(err) + } + // delete + err = db.Delete(m).Error + if err != nil { + panic(err) + } +} + func TestConcurrent(t *testing.T) { pid := os.Getpid() Settings.DB.Path = fmt.Sprintf("/tmp/concurrent-%d.db", pid) @@ -23,12 +58,35 @@ func TestConcurrent(t *testing.T) { if err != nil { panic(err) } + + type A struct { + model.Model + } + + type B struct { + N int + model.Model + A A + AID uint + } + err = db.Migrator().AutoMigrate(&A{}, &B{}) + if err != nil { + panic(err) + } + + a := A{} + err = db.Create(&a).Error + if err != nil { + panic(err) + } + dq := make(chan int, N) for w := 0; w < N; w++ { go func(id int) { fmt.Printf("Started %d\n", id) - for n := 0; n < N*10; n++ { - m := &model.Setting{Key: fmt.Sprintf("key-%d-%d", id, n), Value: n} + for n := 0; n < N*100; n++ { + m := &B{N: n, A: a} + m.CreateUser = "Test" fmt.Printf("(%.4d) CREATE: %.4d\n", id, n) uErr := db.Create(m).Error if uErr != nil { @@ -45,6 +103,20 @@ func TestConcurrent(t *testing.T) { panic(uErr) } } + for i := 0; i < 10; i++ { + fmt.Printf("(%.4d) LIST: %.4d/%.4d\n", id, n, i) + page := api.Page{} + cursor := api.Cursor{} + mx := B{} + dbx := db.Model(mx) + dbx = dbx.Joins("A") + dbx = dbx.Limit(10) + cursor.With(dbx, page) + for cursor.Next(&mx) { + time.Sleep(time.Millisecond + 10) + fmt.Printf("(%.4d) NEXT: %.4d/%.4d ID=%d\n", id, n, i, mx.ID) + } + } for i := 0; i < 4; i++ { uErr = db.Transaction(func(tx *gorm.DB) (err error) { time.Sleep(time.Millisecond * 10) diff --git a/database/driver.go b/database/driver.go index 14bfbb1fa..0c50360b1 100644 --- a/database/driver.go +++ b/database/driver.go @@ -9,12 +9,16 @@ import ( "github.com/mattn/go-sqlite3" ) +// Driver is a wrapper around the SQLite driver. +// The purpose is to prevent database locked errors using +// a mutex around write operations. type Driver struct { mutex sync.Mutex wrapped driver.Driver dsn string } +// Open a connection. func (d *Driver) Open(dsn string) (conn driver.Conn, err error) { d.wrapped = &sqlite3.SQLiteDriver{} conn, err = d.wrapped.Open(dsn) @@ -28,27 +32,33 @@ func (d *Driver) Open(dsn string) (conn driver.Conn, err error) { return } +// OpenConnector opens a connection. func (d *Driver) OpenConnector(dsn string) (dc driver.Connector, err error) { d.dsn = dsn dc = d return } +// Connect opens a connection. func (d *Driver) Connect(context.Context) (conn driver.Conn, err error) { conn, err = d.Open(d.dsn) return } +// Driver returns the underlying driver. func (d *Driver) Driver() driver.Driver { return d } +// Conn is a DB connection. type Conn struct { mutex *sync.Mutex wrapped driver.Conn hasMutex bool + hasTx bool } +// Ping the DB. func (c *Conn) Ping(ctx context.Context) (err error) { if p, cast := c.wrapped.(driver.Pinger); cast { err = p.Ping(ctx) @@ -56,22 +66,35 @@ func (c *Conn) Ping(ctx context.Context) (err error) { return } +// ResetSession reset the connection. +// - Reset the Tx. +// - Release the mutex. func (c *Conn) ResetSession(ctx context.Context) (err error) { + defer func() { + c.hasTx = false + c.release() + }() if p, cast := c.wrapped.(driver.SessionResetter); cast { err = p.ResetSession(ctx) } return } + +// IsValid returns true when the connection is valid. +// When true, the connection may be reused by the sql package. func (c *Conn) IsValid() (b bool) { + b = true if p, cast := c.wrapped.(driver.Validator); cast { b = p.IsValid() } return } +// QueryContext execute a query with context. func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Rows, err error) { if c.needsMutex(query) { c.acquire() + defer c.release() } if p, cast := c.wrapped.(driver.QueryerContext); cast { r, err = p.QueryContext(ctx, query, args) @@ -79,24 +102,34 @@ func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.Nam return } -func (c *Conn) PrepareContext(ctx context.Context, query string) (s driver.Stmt, err error) { +// ExecContext executes an SQL/DDL statement with context. +func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Result, err error) { if c.needsMutex(query) { c.acquire() + defer c.release() } - if p, cast := c.wrapped.(driver.ConnPrepareContext); cast { - s, err = p.PrepareContext(ctx, query) + if p, cast := c.wrapped.(driver.ExecerContext); cast { + r, err = p.ExecContext(ctx, query, args) } return } -func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Result, err error) { +// Begin a transaction. +func (c *Conn) Begin() (tx driver.Tx, err error) { c.acquire() - if p, cast := c.wrapped.(driver.ExecerContext); cast { - r, err = p.ExecContext(ctx, query, args) + tx, err = c.wrapped.Begin() + if err != nil { + return + } + tx = &Tx{ + conn: c, + wrapped: tx, } + c.hasTx = true return } +// BeginTx begins a transaction. func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { c.acquire() if p, cast := c.wrapped.(driver.ConnBeginTx); cast { @@ -104,32 +137,49 @@ func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx } else { tx, err = c.wrapped.Begin() } + tx = &Tx{ + conn: c, + wrapped: tx, + } + c.hasTx = true return } -func (c *Conn) Prepare(query string) (s driver.Stmt, err error) { - if c.needsMutex(query) { - c.acquire() +// Prepare a statement. +func (c *Conn) Prepare(query string) (stmt driver.Stmt, err error) { + stmt, err = c.wrapped.Prepare(query) + stmt = &Stmt{ + conn: c, + wrapped: stmt, + query: query, } - s, err = c.wrapped.Prepare(query) return } -func (c *Conn) Close() (err error) { - err = c.wrapped.Close() - c.release() +// PrepareContext prepares a statement with context. +func (c *Conn) PrepareContext(ctx context.Context, query string) (stmt driver.Stmt, err error) { + if p, cast := c.wrapped.(driver.ConnPrepareContext); cast { + stmt, err = p.PrepareContext(ctx, query) + } else { + stmt, err = c.Prepare(query) + } + stmt = &Stmt{ + conn: c, + wrapped: stmt, + query: query, + } return } -func (c *Conn) Begin() (tx driver.Tx, err error) { - c.acquire() - tx, err = c.wrapped.Begin() - if err != nil { - return - } +// Close the connection. +func (c *Conn) Close() (err error) { + err = c.wrapped.Close() + c.hasMutex = false + c.release() return } +// needsMutex returns true when the query should is a write operation. func (c *Conn) needsMutex(query string) (matched bool) { if query == "" { return @@ -144,6 +194,9 @@ func (c *Conn) needsMutex(query string) (matched bool) { return } +// acquire the mutex. +// Since Locks are not reentrant, the mutex is acquired +// only if this connection has not already acquired it. func (c *Conn) acquire() { if !c.hasMutex { c.mutex.Lock() @@ -151,9 +204,127 @@ func (c *Conn) acquire() { } } +// release the mutex. +// Released only when: +// - This connection has acquired it +// - Not in a transaction. func (c *Conn) release() { - if c.hasMutex { + if c.hasMutex && !c.hasTx { c.mutex.Unlock() c.hasMutex = false } } + +// endTx report transaction has ended. +func (c *Conn) endTx() { + c.hasTx = false +} + +// Stmt is a SQL/DDL statement. +type Stmt struct { + wrapped driver.Stmt + conn *Conn + query string +} + +// Close the statement. +func (s *Stmt) Close() (err error) { + err = s.wrapped.Close() + return +} + +// NumInput returns the number of (query) input parameters. +func (s *Stmt) NumInput() (n int) { + n = s.wrapped.NumInput() + return +} + +// Exec executes the statement. +func (s *Stmt) Exec(args []driver.Value) (r driver.Result, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + r, err = s.wrapped.Exec(args) + return +} + +// ExecContext executes the statement with context. +func (s *Stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (r driver.Result, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + if p, cast := s.wrapped.(driver.StmtExecContext); cast { + r, err = p.ExecContext(ctx, args) + } else { + r, err = s.Exec(s.values(args)) + } + return +} + +// Query executes a query. +func (s *Stmt) Query(args []driver.Value) (r driver.Rows, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + r, err = s.wrapped.Query(args) + return +} + +// QueryContext executes a query. +func (s *Stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (r driver.Rows, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + if p, cast := s.wrapped.(driver.StmtQueryContext); cast { + r, err = p.QueryContext(ctx, args) + } else { + r, err = s.Query(s.values(args)) + } + return +} + +// values converts named-values to values. +func (s *Stmt) values(named []driver.NamedValue) (out []driver.Value) { + for i := range named { + out = append(out, named[i].Value) + } + return +} + +// needsMutex returns true when the query should is a write operation. +func (s *Stmt) needsMutex() (matched bool) { + matched = s.conn.needsMutex(s.query) + return +} + +// Tx is a transaction. +type Tx struct { + wrapped driver.Tx + conn *Conn +} + +// Commit the transaction. +// Releases the mutex. +func (t *Tx) Commit() (err error) { + defer func() { + t.conn.endTx() + t.conn.release() + }() + err = t.wrapped.Commit() + return +} + +// Rollback the transaction. +// Releases the mutex. +func (t *Tx) Rollback() (err error) { + defer func() { + t.conn.endTx() + t.conn.release() + }() + err = t.wrapped.Rollback() + return +} diff --git a/database/pkg.go b/database/pkg.go index 6e9c5eb20..83cfc4f34 100644 --- a/database/pkg.go +++ b/database/pkg.go @@ -18,7 +18,7 @@ var log = logr.WithName("db") var Settings = &settings.Settings const ( - ConnectionString = "file:%s?_journal=WAL&_timeout=100" + ConnectionString = "file:%s?_journal=WAL" FKsOn = "&_foreign_keys=yes" FKsOff = "&_foreign_keys=no" )