From e15a8a4fbf23d17a7c2504a250f8c76e80228fcd Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Thu, 26 Sep 2024 10:23:44 -0500 Subject: [PATCH] :sparkles: Upload files instead of multi-part form. (#743) Post a _manifest_ file instead of separate multi-part form files. Much simpler and more easily supports the addon staging the issues and deps files on disk rather than streaming. The more atomic approach will prevent transaction deadlock which can more easily occur when the addon-analyzer builder reported an error (which it should never do). The uploaded file contains markers used to delimited the documents. `^]` = `\x1D` = GS (group separator). ``` ^]BEGIN-MAIN^] --- commit: 1234 ^]END-MAIN^] ^]BEGIN-ISSUES^] --- ruleset: ruleset-1 rule: rule-1 incidents: ... ^]END-ISSUES^] ^]BEGIN-DEPS^] --- name: github.com/jboss version: 4.0 labels: - konveyor.io/language=java - konveyor.io/otherA=dog ^]END-DEPS^] ``` Flow: 1. post (upload) manifest.yaml file. 2. post `ref` to the manifest file. 3. delete manifest file. Orphaned files will be reaped. --- The binding client needed to be updated to handle different file encoding (MIME). --------- Signed-off-by: Jeff Ortel --- api/analysis.go | 198 ++++++++++++++---- api/file.go | 164 ++++++++++----- binding/application.go | 62 +++--- binding/client.go | 34 +++- binding/file.go | 16 +- hack/add/analysis.sh | 67 ++++--- migration/pkg.go | 2 + migration/v15/migrate.go | 20 ++ migration/v15/model/analysis.go | 170 ++++++++++++++++ migration/v15/model/application.go | 312 +++++++++++++++++++++++++++++ migration/v15/model/assessment.go | 102 ++++++++++ migration/v15/model/core.go | 309 ++++++++++++++++++++++++++++ migration/v15/model/mod.patch | 11 + migration/v15/model/pkg.go | 56 ++++++ model/pkg.go | 2 +- 15 files changed, 1374 insertions(+), 151 deletions(-) create mode 100644 migration/v15/migrate.go create mode 100644 migration/v15/model/analysis.go create mode 100644 migration/v15/model/application.go create mode 100644 migration/v15/model/assessment.go create mode 100644 migration/v15/model/core.go create mode 100644 migration/v15/model/mod.patch create mode 100644 migration/v15/model/pkg.go diff --git a/api/analysis.go b/api/analysis.go index 4080157cc..ed885c082 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -1,12 +1,15 @@ package api import ( + "bufio" "bytes" "encoding/json" "errors" + "fmt" "io" "net/http" "os" + "regexp" "sort" "strconv" "strings" @@ -53,9 +56,15 @@ const ( AppAnalysisIssuesRoot = AppAnalysisRoot + "/issues" ) +// Manifest markers. +// The GS=\x1D (group separator). const ( - IssueField = "issues" - DepField = "dependencies" + BeginMainMarker = "\x1DBEGIN-MAIN\x1D" + EndMainMarker = "\x1DEND-MAIN\x1D" + BeginIssuesMarker = "\x1DBEGIN-ISSUES\x1D" + EndIssuesMarker = "\x1DEND-ISSUES\x1D" + BeginDepsMarker = "\x1DBEGIN-DEPS\x1D" + EndDepsMarker = "\x1DEND-DEPS\x1D" ) // AnalysisHandler handles analysis resource routes. @@ -315,9 +324,20 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) { // @summary Create an analysis. // @description Create an analysis. // @description Form fields: -// @description - file: file that contains the api.Analysis resource. -// @description - issues: file that multiple api.Issue resources. -// @description - dependencies: file that multiple api.TechDependency resources. +// @description file: A manifest file that contains 3 sections +// @description containing documents delimited by markers. +// @description The manifest must contain ALL markers even when sections are empty. +// @description Note: `^]` = `\x1D` = GS (group separator). +// @description Section markers: +// @description ^]BEGIN-MAIN^] +// @description ^]END-MAIN^] +// @description ^]BEGIN-ISSUES^] +// @description ^]END-ISSUES^] +// @description ^]BEGIN-DEPS^] +// @description ^]END-DEPS^] +// @description The encoding must be: +// @description - application/json +// @description - application/x-yaml // @tags analyses // @produce json // @success 201 {object} api.Analysis @@ -337,32 +357,40 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { return } } + db := h.DB(ctx) // - // Analysis - input, err := ctx.FormFile(FileField) + // Manifest + fh := FileHandler{} + name := fmt.Sprintf("app.%d.manifest", id) + file, err := fh.create(ctx, name) if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - reader, err := input.Open() + defer func() { + err = fh.delete(ctx, file) + if err != nil { + _ = ctx.Error(err) + } + }() + reader := &ManifestReader{} + f, err := reader.open(file.Path, BeginMainMarker, EndMainMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding := input.Header.Get(ContentType) - d, err := h.Decoder(ctx, encoding, reader) + d, err := h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - r := Analysis{} - err = d.Decode(&r) + r := &Analysis{} + err = d.Decode(r) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -371,7 +399,6 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { analysis := r.Model() analysis.ApplicationID = id analysis.CreateUser = h.BaseHandler.CurrentUser(ctx) - db := h.DB(ctx) db.Logger = db.Logger.LogMode(logger.Error) err = db.Create(analysis).Error if err != nil { @@ -380,23 +407,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Issues - input, err = ctx.FormFile(IssueField) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = input.Open() + reader = &ManifestReader{} + f, err = reader.open(file.Path, BeginIssuesMarker, EndIssuesMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + d, err = h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -425,23 +446,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Dependencies - input, err = ctx.FormFile(DepField) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = input.Open() + reader = &ManifestReader{} + f, err = reader.open(file.Path, BeginDepsMarker, EndDepsMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + d, err = h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -2860,3 +2875,116 @@ func (r *yamlEncoder) embed(object any) encoder { r.write(s) return r } + +// ManifestReader analysis manifest reader. +// The manifest contains 3 sections containing documents delimited by markers. +// The manifest must contain ALL markers even when sections are empty. +// Note: `^]` = `\x1D` = GS (group separator). +// Section markers: +// +// ^]BEGIN-MAIN^] +// ^]END-MAIN^] +// ^]BEGIN-ISSUES^] +// ^]END-ISSUES^] +// ^]BEGIN-DEPS^] +// ^]END-DEPS^] +type ManifestReader struct { + file *os.File + marker map[string]int64 + begin int64 + end int64 + read int64 +} + +// scan manifest and catalog position of markers. +func (r *ManifestReader) scan(path string) (err error) { + if r.marker != nil { + return + } + r.file, err = os.Open(path) + if err != nil { + return + } + defer func() { + _ = r.file.Close() + }() + pattern, err := regexp.Compile(`^\x1D[A-Z-]+\x1D$`) + if err != nil { + return + } + p := int64(0) + r.marker = make(map[string]int64) + scanner := bufio.NewScanner(r.file) + for scanner.Scan() { + content := scanner.Text() + matched := strings.TrimSpace(content) + if pattern.Match([]byte(matched)) { + r.marker[matched] = p + } + p += int64(len(content)) + p++ + } + + return +} + +// open returns a read delimited by the specified markers. +func (r *ManifestReader) open(path, begin, end string) (reader io.ReadCloser, err error) { + found := false + err = r.scan(path) + if err != nil { + return + } + r.begin, found = r.marker[begin] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", begin), + } + return + } + r.end, found = r.marker[end] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", end), + } + return + } + if r.begin >= r.end { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s must preceed %s.", begin, end), + } + return + } + r.begin += int64(len(begin)) + r.begin++ + r.read = r.end - r.begin + r.file, err = os.Open(path) + if err != nil { + return + } + _, err = r.file.Seek(r.begin, io.SeekStart) + reader = r + return +} + +// Read bytes. +func (r *ManifestReader) Read(b []byte) (n int, err error) { + n, err = r.file.Read(b) + if n == 0 || err != nil { + return + } + if int64(n) > r.read { + n = int(r.read) + } + r.read -= int64(n) + if n < 1 { + err = io.EOF + } + return +} + +// Close the reader. +func (r *ManifestReader) Close() (err error) { + err = r.file.Close() + return +} diff --git a/api/file.go b/api/file.go index 5e280172f..74fc17aae 100644 --- a/api/file.go +++ b/api/file.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" "github.com/konveyor/tackle2-hub/model" ) @@ -70,52 +71,11 @@ func (h FileHandler) List(ctx *gin.Context) { // @router /files [post] // @param name path string true "File name" func (h FileHandler) Create(ctx *gin.Context) { - var err error - input, err := ctx.FormFile(FileField) + m, err := h.create(ctx, ctx.Param(ID)) if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - m := &model.File{} - m.Name = ctx.Param(ID) - m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(&m) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } - defer func() { - if err != nil { - h.Status(ctx, http.StatusInternalServerError) - _ = h.DB(ctx).Delete(&m) - return - } - }() - reader, err := input.Open() - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - defer func() { - _ = reader.Close() - }() - writer, err := os.Create(m.Path) - if err != nil { - return - } - defer func() { - _ = writer.Close() - }() - _, err = io.Copy(writer, reader) - if err != nil { - return - } - err = os.Chmod(m.Path, 0666) - if err != nil { - return - } r := File{} r.With(m) h.Respond(ctx, http.StatusCreated, r) @@ -224,20 +184,126 @@ func (h FileHandler) Delete(ctx *gin.Context) { _ = ctx.Error(err) return } - err = os.Remove(m.Path) + err = h.delete(ctx, m) if err != nil { - if !os.IsNotExist(err) { - _ = ctx.Error(err) + _ = ctx.Error(err) + return + } + + h.Status(ctx, http.StatusNoContent) +} + +// create a file. +func (h FileHandler) create(ctx *gin.Context, name string) (m *model.File, err error) { + mode := ctx.ContentType() + switch mode { + case binding.MIMEMultipartPOSTForm: + m, err = h.createMultipart(ctx, name) + case binding.MIMEYAML: + m, err = h.createBody(ctx, name, binding.MIMEYAML) + default: + m, err = h.createBody(ctx, name, binding.MIMEJSON) + } + return +} + +// create a file with multipart form. +func (h FileHandler) createMultipart(ctx *gin.Context, name string) (m *model.File, err error) { + input, err := ctx.FormFile(FileField) + if err != nil { + err = &BadRequestError{err.Error()} + return + } + m = &model.File{} + m.Name = name + m.Encoding = input.Header.Get(ContentType) + m.CreateUser = h.BaseHandler.CurrentUser(ctx) + db := h.DB(ctx) + err = db.Create(&m).Error + if err != nil { + return + } + defer func() { + if err != nil { + h.Status(ctx, http.StatusInternalServerError) + _ = db.Delete(&m) return } + }() + reader, err := input.Open() + if err != nil { + err = &BadRequestError{err.Error()} + return } - err = h.DB(ctx).Delete(m).Error + defer func() { + _ = reader.Close() + }() + writer, err := os.Create(m.Path) if err != nil { - _ = ctx.Error(err) return } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + return + } + err = os.Chmod(m.Path, 0666) + if err != nil { + return + } + return +} - h.Status(ctx, http.StatusNoContent) +// create a file with request body. +func (h FileHandler) createBody(ctx *gin.Context, name, encoding string) (m *model.File, err error) { + m = &model.File{} + m.Name = name + m.Encoding = encoding + m.CreateUser = h.BaseHandler.CurrentUser(ctx) + db := h.DB(ctx) + err = db.Create(&m).Error + if err != nil { + return + } + defer func() { + if err != nil { + h.Status(ctx, http.StatusInternalServerError) + _ = db.Delete(&m) + return + } + }() + reader := ctx.Request.Body + writer, err := os.Create(m.Path) + if err != nil { + return + } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + return + } + err = os.Chmod(m.Path, 0666) + if err != nil { + return + } + return +} + +// delete the specified file. +func (h FileHandler) delete(ctx *gin.Context, m *model.File) (err error) { + err = os.Remove(m.Path) + if err != nil { + if !os.IsNotExist(err) { + return + } + } + db := h.DB(ctx) + err = db.Delete(m).Error + return } // File REST resource. @@ -245,6 +311,7 @@ type File struct { Resource `yaml:",inline"` Name string `json:"name"` Path string `json:"path"` + Encoding string `yaml:"encoding,omitempty"` Expiration *time.Time `json:"expiration,omitempty"` } @@ -253,5 +320,6 @@ func (r *File) With(m *model.File) { r.Resource.With(&m.Model) r.Name = m.Name r.Path = m.Path + r.Encoding = m.Encoding r.Expiration = m.Expiration } diff --git a/binding/application.go b/binding/application.go index ec7406f43..7266987bc 100644 --- a/binding/application.go +++ b/binding/application.go @@ -1,16 +1,12 @@ package binding import ( - "bytes" "errors" - "io" - "net/http" "strconv" - mime "github.com/gin-gonic/gin/binding" + "github.com/gin-gonic/gin/binding" liberr "github.com/jortel/go-utils/error" "github.com/konveyor/tackle2-hub/api" - "gopkg.in/yaml.v2" ) // Application API. @@ -316,30 +312,38 @@ type Analysis struct { appId uint } -// Create an analysis report. -func (h *Analysis) Create(r *api.Analysis, encoding string, issues, deps io.Reader) (err error) { +// Create an analysis report using the manifest at the specified path. +// The manifest contains 3 sections containing documents delimited by markers. +// The manifest must contain ALL markers even when sections are empty. +// Note: `^]` = `\x1D` = GS (group separator). +// Section markers: +// +// ^]BEGIN-MAIN^] +// ^]END-MAIN^] +// ^]BEGIN-ISSUES^] +// ^]END-ISSUES^] +// ^]BEGIN-DEPS^] +// ^]END-DEPS^] +// +// The encoding must be: +// - application/json +// - application/x-yaml +func (h *Analysis) Create(manifest, encoding string) (r *api.Analysis, err error) { + switch encoding { + case "": + encoding = binding.MIMEJSON + case binding.MIMEJSON, + binding.MIMEYAML: + default: + err = liberr.New( + "Encoding: %s not supported", + encoding) + } + r = &api.Analysis{} path := Path(api.AppAnalysesRoot).Inject(Params{api.ID: h.appId}) - b, _ := yaml.Marshal(r) - err = h.client.FileSend( - path, - http.MethodPost, - []Field{ - { - Name: api.FileField, - Reader: bytes.NewReader(b), - Encoding: mime.MIMEYAML, - }, - { - Name: api.IssueField, - Encoding: encoding, - Reader: issues, - }, - { - Name: api.DepField, - Encoding: encoding, - Reader: deps, - }, - }, - r) + err = h.client.FilePostEncoded(path, manifest, r, encoding) + if err != nil { + return + } return } diff --git a/binding/client.go b/binding/client.go index 78ae07b85..a5976eb9e 100644 --- a/binding/client.go +++ b/binding/client.go @@ -457,11 +457,19 @@ func (r *Client) FileGet(path, destination string) (err error) { // FilePost uploads a file. // Returns the created File resource. func (r *Client) FilePost(path, source string, object any) (err error) { + err = r.FilePostEncoded(path, source, object, "") + return +} + +// FilePostEncoded uploads a file. +// Returns the created File resource. +func (r *Client) FilePostEncoded(path, source string, object any, encoding string) (err error) { if source == "" { fields := []Field{ { - Name: api.FileField, - Reader: bytes.NewReader([]byte{}), + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPost, fields, object) @@ -478,8 +486,9 @@ func (r *Client) FilePost(path, source string, object any) (err error) { } fields := []Field{ { - Name: api.FileField, - Path: source, + Name: api.FileField, + Path: source, + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPost, fields, object) @@ -489,11 +498,19 @@ func (r *Client) FilePost(path, source string, object any) (err error) { // FilePut uploads a file. // Returns the created File resource. func (r *Client) FilePut(path, source string, object any) (err error) { + err = r.FilePutEncoded(path, source, object, "") + return +} + +// FilePutEncoded uploads a file. +// Returns the created File resource. +func (r *Client) FilePutEncoded(path, source string, object any, encoding string) (err error) { if source == "" { fields := []Field{ { - Name: api.FileField, - Reader: bytes.NewReader([]byte{}), + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPut, fields, object) @@ -510,8 +527,9 @@ func (r *Client) FilePut(path, source string, object any) (err error) { } fields := []Field{ { - Name: api.FileField, - Path: source, + Name: api.FileField, + Path: source, + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPut, fields, object) diff --git a/binding/file.go b/binding/file.go index 22bdcb80b..86bd3d94b 100644 --- a/binding/file.go +++ b/binding/file.go @@ -42,17 +42,29 @@ func (h *File) Touch(name string) (r *api.File, err error) { // Post uploads a file. func (h *File) Post(source string) (r *api.File, err error) { + r, err = h.PostEncoded(source, "") + return +} + +// PostEncoded uploads a file. +func (h *File) PostEncoded(source string, encoding string) (r *api.File, err error) { r = &api.File{} path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) - err = h.client.FilePost(path, source, r) + err = h.client.FilePostEncoded(path, source, r, encoding) return } // Put uploads a file. func (h *File) Put(source string) (r *api.File, err error) { + r, err = h.PutEncoded(source, "") + return +} + +// PutEncoded uploads a file. +func (h *File) PutEncoded(source string, encoding string) (r *api.File, err error) { r = &api.File{} path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) - err = h.client.FilePut(path, source, r) + err = h.client.FilePutEncoded(path, source, r, encoding) return } diff --git a/hack/add/analysis.sh b/hack/add/analysis.sh index bb2cb79e9..568ff3341 100755 --- a/hack/add/analysis.sh +++ b/hack/add/analysis.sh @@ -3,20 +3,18 @@ set -e host="${HOST:-localhost:8080}" -app="${1:-1}" +appId="${1:-1}" nRuleSet="${2:-10}" nIssue="${3:-10}" nIncident="${4:-25}" -aPath="/tmp/analysis.yaml" -iPath="/tmp/issues.yaml" -dPath="/tmp/deps.yaml" +tmp=/tmp/${self}-${pid} +file="/tmp/manifest.yaml" -echo " Application: ${app}" +echo " Application: ${appId}" echo " RuleSets: ${nRuleSet}" echo " Issues: ${nIssue}" echo " Incidents: ${nIncident}" -echo " Issues path: ${iPath}" -echo " Deps path: ${dPath}" +echo " Manifest path: ${file}" sources=( konveyor.io/source=oraclejdk @@ -62,11 +60,18 @@ konveyor.io/target=hibernate konveyor.io/target=jbpm ) +# +# Analysis +# +printf "\x1DBEGIN-MAIN\x1D\n" > ${file} +echo -n "--- +commit: "1234" +" >> ${file} +printf "\x1DEND-MAIN\x1D\n" >> ${file} # # Issues # -file=${iPath} -echo "" > ${file} +printf "\x1DBEGIN-ISSUES\x1D\n" >> ${file} for r in $(seq 1 ${nRuleSet}) do for i in $(seq 1 ${nIssue}) @@ -150,17 +155,18 @@ fi done done done +printf "\x1DEND-ISSUES\x1D +\x1DBEGIN-DEPS\x1D\n" >> ${file} # # Deps # -file=${dPath} echo -n "--- name: github.com/jboss version: 4.0 labels: - konveyor.io/language=java - konveyor.io/otherA=dog -" > ${file} +" >> ${file} echo -n "--- name: github.com/jboss version: 5.0 @@ -192,23 +198,28 @@ echo -n "--- name: github.com/java version: 8 " >> ${file} -# -# Analysis -# -file=${aPath} -echo -n "--- -commit: "42b22a90" -issues: -dependencies: -" > ${file} +printf "\x1DEND-DEPS\x1D\n" >> ${file} -echo "Report CREATED" +echo "Manifest (file) GENERATED: ${file}" -mime="application/x-yaml" +# +# Post manifest. +code=$(curl -kSs -o ${tmp} -w "%{http_code}" \ + -F "file=@${file};type=application/x-yaml" \ + -H 'Accept:application/x-yaml' \ + http://${host}/applications/${appId}/analyses) +if [ ! $? -eq 0 ] +then + exit $? +fi +case ${code} in + 201) + echo "Analysis: created." + cat ${tmp} + ;; + *) + echo "Analysis create - FAILED: ${code}." + cat ${tmp} + exit 1 +esac -curl \ - -F "file=@${aPath};type=${mime}" \ - -F "issues=@${iPath};type=${mime}" \ - -F "dependencies=@${dPath};type=${mime}" \ - ${host}/applications/${app}/analyses \ - -H "Accept:${mime}" diff --git a/migration/pkg.go b/migration/pkg.go index f80ed8e48..5caaecbd8 100644 --- a/migration/pkg.go +++ b/migration/pkg.go @@ -7,6 +7,7 @@ import ( v12 "github.com/konveyor/tackle2-hub/migration/v12" v13 "github.com/konveyor/tackle2-hub/migration/v13" v14 "github.com/konveyor/tackle2-hub/migration/v14" + v15 "github.com/konveyor/tackle2-hub/migration/v15" v2 "github.com/konveyor/tackle2-hub/migration/v2" v3 "github.com/konveyor/tackle2-hub/migration/v3" v4 "github.com/konveyor/tackle2-hub/migration/v4" @@ -56,5 +57,6 @@ func All() []Migration { v12.Migration{}, v13.Migration{}, v14.Migration{}, + v15.Migration{}, } } diff --git a/migration/v15/migrate.go b/migration/v15/migrate.go new file mode 100644 index 000000000..1fe82e975 --- /dev/null +++ b/migration/v15/migrate.go @@ -0,0 +1,20 @@ +package v15 + +import ( + "github.com/jortel/go-utils/logr" + "github.com/konveyor/tackle2-hub/migration/v15/model" + "gorm.io/gorm" +) + +var log = logr.WithName("migration|v15") + +type Migration struct{} + +func (r Migration) Apply(db *gorm.DB) (err error) { + err = db.AutoMigrate(r.Models()...) + return +} + +func (r Migration) Models() []interface{} { + return model.All() +} diff --git a/migration/v15/model/analysis.go b/migration/v15/model/analysis.go new file mode 100644 index 000000000..bc33da893 --- /dev/null +++ b/migration/v15/model/analysis.go @@ -0,0 +1,170 @@ +package model + +import ( + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +// Analysis report. +type Analysis struct { + Model + Effort int + Commit string + Archived bool + Summary []ArchivedIssue `gorm:"type:json;serializer:json"` + Issues []Issue `gorm:"constraint:OnDelete:CASCADE"` + Dependencies []TechDependency `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID uint `gorm:"index;not null"` + Application *Application +} + +// TechDependency report dependency. +type TechDependency struct { + Model + Provider string `gorm:"uniqueIndex:depA"` + Name string `gorm:"uniqueIndex:depA"` + Version string `gorm:"uniqueIndex:depA"` + SHA string `gorm:"uniqueIndex:depA"` + Indirect bool + Labels []string `gorm:"type:json;serializer:json"` + AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` + Analysis *Analysis +} + +// Issue report issue (violation). +type Issue struct { + Model + RuleSet string `gorm:"uniqueIndex:issueA;not null"` + Rule string `gorm:"uniqueIndex:issueA;not null"` + Name string `gorm:"index"` + Description string + Category string `gorm:"index;not null"` + Incidents []Incident `gorm:"foreignKey:IssueID;constraint:OnDelete:CASCADE"` + Links []Link `gorm:"type:json;serializer:json"` + Facts json.Map `gorm:"type:json;serializer:json"` + Labels []string `gorm:"type:json;serializer:json"` + Effort int `gorm:"index;not null"` + AnalysisID uint `gorm:"index;uniqueIndex:issueA;not null"` + Analysis *Analysis +} + +// Incident report an issue incident. +type Incident struct { + Model + File string `gorm:"index;not null"` + Line int + Message string + CodeSnip string + Facts json.Map `gorm:"type:json;serializer:json"` + IssueID uint `gorm:"index;not null"` + Issue *Issue +} + +// RuleSet - Analysis ruleset. +type RuleSet struct { + Model + UUID *string `gorm:"uniqueIndex"` + Kind string + Name string `gorm:"uniqueIndex;not null"` + Description string + Repository Repository `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity + Rules []Rule `gorm:"constraint:OnDelete:CASCADE"` + DependsOn []RuleSet `gorm:"many2many:RuleSetDependencies;constraint:OnDelete:CASCADE"` +} + +func (r *RuleSet) Builtin() bool { + return r.UUID != nil +} + +// BeforeUpdate hook to avoid cyclic dependencies. +func (r *RuleSet) BeforeUpdate(db *gorm.DB) (err error) { + seen := make(map[uint]bool) + var nextDeps []RuleSet + var nextRuleSetIDs []uint + for _, dep := range r.DependsOn { + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + for len(nextRuleSetIDs) != 0 { + result := db.Preload("DependsOn").Where("ID IN ?", nextRuleSetIDs).Find(&nextDeps) + if result.Error != nil { + err = result.Error + return + } + nextRuleSetIDs = nextRuleSetIDs[:0] + for _, nextDep := range nextDeps { + for _, dep := range nextDep.DependsOn { + if seen[dep.ID] { + continue + } + if dep.ID == r.ID { + err = DependencyCyclicError{} + return + } + seen[dep.ID] = true + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + } + } + + return +} + +// Rule - Analysis rule. +type Rule struct { + Model + Name string + Description string + Labels []string `gorm:"type:json;serializer:json"` + RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` + RuleSet *RuleSet + FileID *uint `gorm:"uniqueIndex:RuleA" ref:"file"` + File *File +} + +// Target - analysis rule selector. +type Target struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex;not null"` + Description string + Provider string + Choice bool + Labels []TargetLabel `gorm:"type:json;serializer:json"` + ImageID uint `gorm:"index" ref:"file"` + Image *File + RuleSetID *uint `gorm:"index"` + RuleSet *RuleSet +} + +func (r *Target) Builtin() bool { + return r.UUID != nil +} + +// +// JSON Fields. +// + +// ArchivedIssue resource created when issues are archived. +type ArchivedIssue struct { + RuleSet string `json:"ruleSet"` + Rule string `json:"rule"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Description string `json:"description,omitempty" yaml:",omitempty"` + Category string `json:"category"` + Effort int `json:"effort"` + Incidents int `json:"incidents"` +} + +// Link URL link. +type Link struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` +} + +// TargetLabel - label format specific to Targets +type TargetLabel struct { + Name string `json:"name"` + Label string `json:"label"` +} diff --git a/migration/v15/model/application.go b/migration/v15/model/application.go new file mode 100644 index 000000000..cb363b78d --- /dev/null +++ b/migration/v15/model/application.go @@ -0,0 +1,312 @@ +package model + +import ( + "fmt" + "sync" + "time" + + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +type Application struct { + Model + BucketOwner + Name string `gorm:"index;unique;not null"` + Description string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Repository Repository `gorm:"type:json;serializer:json"` + Binary string + Facts []Fact `gorm:"constraint:OnDelete:CASCADE"` + Comments string + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ApplicationTags"` + Identities []Identity `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` + BusinessServiceID *uint `gorm:"index"` + BusinessService *BusinessService + OwnerID *uint `gorm:"index"` + Owner *Stakeholder `gorm:"foreignKey:OwnerID"` + Contributors []Stakeholder `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + Analyses []Analysis `gorm:"constraint:OnDelete:CASCADE"` + MigrationWaveID *uint `gorm:"index"` + MigrationWave *MigrationWave + Ticket *Ticket `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +type Fact struct { + ApplicationID uint `gorm:"<-:create;primaryKey"` + Key string `gorm:"<-:create;primaryKey"` + Source string `gorm:"<-:create;primaryKey;not null"` + Value any `gorm:"type:json;not null;serializer:json"` + Application *Application +} + +// ApplicationTag represents a row in the join table for the +// many-to-many relationship between Applications and Tags. +type ApplicationTag struct { + ApplicationID uint `gorm:"primaryKey"` + TagID uint `gorm:"primaryKey"` + Source string `gorm:"primaryKey;not null"` + Application Application `gorm:"constraint:OnDelete:CASCADE"` + Tag Tag `gorm:"constraint:OnDelete:CASCADE"` +} + +// TableName must return "ApplicationTags" to ensure compatibility +// with the autogenerated join table name. +func (ApplicationTag) TableName() string { + return "ApplicationTags" +} + +// depMutex ensures Dependency.Create() is not executed concurrently. +var depMutex sync.Mutex + +type Dependency struct { + Model + ToID uint `gorm:"index"` + To *Application `gorm:"foreignKey:ToID;constraint:OnDelete:CASCADE"` + FromID uint `gorm:"index"` + From *Application `gorm:"foreignKey:FromID;constraint:OnDelete:CASCADE"` +} + +// Create a dependency synchronized using a mutex. +func (r *Dependency) Create(db *gorm.DB) (err error) { + depMutex.Lock() + defer depMutex.Unlock() + err = db.Create(r).Error + return +} + +// BeforeCreate detects cyclic dependencies. +func (r *Dependency) BeforeCreate(db *gorm.DB) (err error) { + var nextDeps []*Dependency + var nextAppsIDs []uint + nextAppsIDs = append(nextAppsIDs, r.FromID) + for len(nextAppsIDs) != 0 { + db.Where("ToID IN ?", nextAppsIDs).Find(&nextDeps) + nextAppsIDs = nextAppsIDs[:0] // empty array, but keep capacity + for _, nextDep := range nextDeps { + if nextDep.FromID == r.ToID { + err = DependencyCyclicError{} + return + } + nextAppsIDs = append(nextAppsIDs, nextDep.FromID) + } + } + + return +} + +// DependencyCyclicError reports cyclic Dependency error. +type DependencyCyclicError struct{} + +func (e DependencyCyclicError) Error() string { + return "Cyclic dependencies are not permitted." +} + +type BusinessService struct { + Model + Name string `gorm:"index;unique;not null"` + Description string + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + StakeholderID *uint `gorm:"index"` + Stakeholder *Stakeholder +} + +type JobFunction struct { + Model + UUID *string `gorm:"uniqueIndex"` + Username string + Name string `gorm:"index;unique;not null"` + Stakeholders []Stakeholder `gorm:"constraint:OnDelete:SET NULL"` +} + +type Stakeholder struct { + Model + Name string `gorm:"not null;"` + Email string `gorm:"index;unique;not null"` + Groups []StakeholderGroup `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + BusinessServices []BusinessService `gorm:"constraint:OnDelete:SET NULL"` + JobFunctionID *uint `gorm:"index"` + JobFunction *JobFunction + Owns []Application `gorm:"foreignKey:OwnerID;constraint:OnDelete:SET NULL"` + Contributes []Application `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` +} + +type StakeholderGroup struct { + Model + Name string `gorm:"index;unique;not null"` + Username string + Description string + Stakeholders []Stakeholder `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type MigrationWave struct { + Model + Name string `gorm:"uniqueIndex:MigrationWaveA"` + StartDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + EndDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + Stakeholders []Stakeholder `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Archetype struct { + Model + Name string + Description string + Comments string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` + CriteriaTags []Tag `gorm:"many2many:ArchetypeCriteriaTags;constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ArchetypeTags;constraint:OnDelete:CASCADE"` + Stakeholders []Stakeholder `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Tag struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex:tagA;not null"` + Username string + CategoryID uint `gorm:"uniqueIndex:tagA;index;not null"` + Category TagCategory +} + +type TagCategory struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"index;unique;not null"` + Username string + Rank uint + Color string + Tags []Tag `gorm:"foreignKey:CategoryID;constraint:OnDelete:CASCADE"` +} + +type Ticket struct { + Model + // Kind of ticket in the external tracker. + Kind string `gorm:"not null"` + // Parent resource that this ticket should belong to in the tracker. (e.g. Jira project) + Parent string `gorm:"not null"` + // Custom fields to send to the tracker when creating the ticket + Fields json.Map `gorm:"type:json;serializer:json"` + // Whether the last attempt to do something with the ticket reported an error + Error bool + // Error message, if any + Message string + // Whether the ticket was created in the external tracker + Created bool + // Reference id in external tracker + Reference string + // URL to ticket in external tracker + Link string + // Status of ticket in external tracker + Status string + LastUpdated time.Time + Application *Application + ApplicationID uint `gorm:"uniqueIndex:ticketA;not null"` + Tracker *Tracker + TrackerID uint `gorm:"uniqueIndex:ticketA;not null"` +} + +type Tracker struct { + Model + Name string `gorm:"index;unique;not null"` + URL string + Kind string + Identity *Identity + IdentityID uint + Connected bool + LastUpdated time.Time + Message string + Insecure bool + Tickets []Ticket +} + +type Import struct { + Model + Filename string + ApplicationName string + BusinessService string + Comments string + Dependency string + DependencyDirection string + Description string + ErrorMessage string + IsValid bool + RecordType1 string + ImportSummary ImportSummary + ImportSummaryID uint `gorm:"index"` + Processed bool + ImportTags []ImportTag `gorm:"constraint:OnDelete:CASCADE"` + BinaryGroup string + BinaryArtifact string + BinaryVersion string + BinaryPackaging string + RepositoryKind string + RepositoryURL string + RepositoryBranch string + RepositoryPath string + Owner string + Contributors string +} + +func (r *Import) AsMap() (m map[string]any) { + m = make(map[string]any) + m["filename"] = r.Filename + m["applicationName"] = r.ApplicationName + // "Application Name" is necessary in order for + // the UI to display the error report correctly. + m["Application Name"] = r.ApplicationName + m["businessService"] = r.BusinessService + m["comments"] = r.Comments + m["dependency"] = r.Dependency + m["dependencyDirection"] = r.DependencyDirection + m["description"] = r.Description + m["errorMessage"] = r.ErrorMessage + m["isValid"] = r.IsValid + m["processed"] = r.Processed + m["recordType1"] = r.RecordType1 + for i, tag := range r.ImportTags { + m[fmt.Sprintf("category%v", i+1)] = tag.Category + m[fmt.Sprintf("tag%v", i+1)] = tag.Name + } + return +} + +type ImportSummary struct { + Model + Content []byte + Filename string + ImportStatus string + Imports []Import `gorm:"constraint:OnDelete:CASCADE"` + CreateEntities bool +} + +type ImportTag struct { + Model + Name string + Category string + ImportID uint `gorm:"index"` + Import *Import +} + +// +// JSON Fields. +// + +// Repository represents an SCM repository. +type Repository struct { + Kind string `json:"kind"` + URL string `json:"url"` + Branch string `json:"branch"` + Tag string `json:"tag"` + Path string `json:"path"` +} diff --git a/migration/v15/model/assessment.go b/migration/v15/model/assessment.go new file mode 100644 index 000000000..0b51e714d --- /dev/null +++ b/migration/v15/model/assessment.go @@ -0,0 +1,102 @@ +package model + +type Questionnaire struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"unique"` + Description string + Required bool + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +// Builtin returns true if this is a Konveyor-provided questionnaire. +func (r *Questionnaire) Builtin() bool { + return r.UUID != nil +} + +type Assessment struct { + Model + ApplicationID *uint `gorm:"uniqueIndex:AssessmentA"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex:AssessmentB"` + Archetype *Archetype + QuestionnaireID uint `gorm:"uniqueIndex:AssessmentA;uniqueIndex:AssessmentB"` + Questionnaire Questionnaire + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` + Stakeholders []Stakeholder `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Review struct { + Model + BusinessCriticality uint `gorm:"not null"` + EffortEstimate string `gorm:"not null"` + ProposedAction string `gorm:"not null"` + WorkPriority uint `gorm:"not null"` + Comments string + ApplicationID *uint `gorm:"uniqueIndex"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex"` + Archetype *Archetype +} + +// +// JSON Fields. +// + +// Section represents a group of questions in a questionnaire. +type Section struct { + Order uint `json:"order" yaml:"order"` + Name string `json:"name" yaml:"name"` + Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` + Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +} + +// Question represents a question in a questionnaire. +type Question struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Explanation string `json:"explanation" yaml:"explanation"` + IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` + ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` + Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` +} + +// Answer represents an answer to a question in a questionnaire. +type Answer struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` + Rationale string `json:"rationale" yaml:"rationale"` + Mitigation string `json:"mitigation" yaml:"mitigation"` + ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` + AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` + Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` + AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` +} + +// CategorizedTag represents a human-readable pair of category and tag. +type CategorizedTag struct { + Category string `json:"category" yaml:"category"` + Tag string `json:"tag" yaml:"tag"` +} + +// RiskMessages contains messages to display for each risk level. +type RiskMessages struct { + Red string `json:"red" yaml:"red"` + Yellow string `json:"yellow" yaml:"yellow"` + Green string `json:"green" yaml:"green"` + Unknown string `json:"unknown" yaml:"unknown"` +} + +// Thresholds contains the threshold values for determining risk for the questionnaire. +type Thresholds struct { + Red uint `json:"red" yaml:"red"` + Yellow uint `json:"yellow" yaml:"yellow"` + Unknown uint `json:"unknown" yaml:"unknown"` +} diff --git a/migration/v15/model/core.go b/migration/v15/model/core.go new file mode 100644 index 000000000..c3b2a85b4 --- /dev/null +++ b/migration/v15/model/core.go @@ -0,0 +1,309 @@ +package model + +import ( + "os" + "path" + "time" + + "github.com/google/uuid" + liberr "github.com/jortel/go-utils/error" + "github.com/konveyor/tackle2-hub/encryption" + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +// Model Base model. +type Model struct { + ID uint `gorm:"<-:create;primaryKey"` + CreateTime time.Time `gorm:"<-:create;autoCreateTime"` + CreateUser string `gorm:"<-:create"` + UpdateUser string +} + +// PK sequence. +type PK struct { + Kind string `gorm:"<-:create;primaryKey"` + LastID uint +} + +// Setting hub settings. +type Setting struct { + Model + Key string `gorm:"<-:create;uniqueIndex"` + Value any `gorm:"type:json;serializer:json"` +} + +// As unmarshalls the value of the Setting into the `ptr` parameter. +func (r *Setting) As(ptr any) (err error) { + bytes, err := json.Marshal(r.Value) + if err != nil { + err = liberr.Wrap(err) + } + err = json.Unmarshal(bytes, ptr) + if err != nil { + err = liberr.Wrap(err) + } + return +} + +type Bucket struct { + Model + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *Bucket) BeforeCreate(db *gorm.DB) (err error) { + if m.Path == "" { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + uid.String()) + err = os.MkdirAll(m.Path, 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + } + return +} + +type BucketOwner struct { + BucketID *uint `gorm:"index" ref:"bucket"` + Bucket *Bucket +} + +func (m *BucketOwner) BeforeCreate(db *gorm.DB) (err error) { + if !m.HasBucket() { + b := &Bucket{} + err = db.Create(b).Error + m.SetBucket(&b.ID) + } + return +} + +func (m *BucketOwner) SetBucket(id *uint) { + m.BucketID = id + m.Bucket = nil +} + +func (m *BucketOwner) HasBucket() (b bool) { + return m.BucketID != nil +} + +type File struct { + Model + Name string + Encoding string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *File) BeforeCreate(db *gorm.DB) (err error) { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + ".file", + uid.String()) + err = os.MkdirAll(path.Dir(m.Path), 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + return +} + +type Task struct { + Model + BucketOwner + Name string `gorm:"index"` + Kind string + Addon string `gorm:"index"` + Extensions []string `gorm:"type:json;serializer:json"` + State string `gorm:"index"` + Locator string `gorm:"index"` + Priority int + Policy TaskPolicy `gorm:"type:json;serializer:json"` + TTL TTL `gorm:"type:json;serializer:json"` + Data json.Data `gorm:"type:json;serializer:json"` + Started *time.Time + Terminated *time.Time + Errors []TaskError `gorm:"type:json;serializer:json"` + Events []TaskEvent `gorm:"type:json;serializer:json"` + Pod string `gorm:"index"` + Retries int + Attached []Attachment `gorm:"type:json;serializer:json" ref:"[]file"` + Report *TaskReport `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID *uint `gorm:"index"` + Application *Application + TaskGroupID *uint `gorm:"<-:create"` + TaskGroup *TaskGroup +} + +func (m *Task) BeforeCreate(db *gorm.DB) (err error) { + err = m.BucketOwner.BeforeCreate(db) + return +} + +type TaskReport struct { + Model + Status string + Total int + Completed int + Activity []string `gorm:"type:json;serializer:json"` + Errors []TaskError `gorm:"type:json;serializer:json"` + Attached []Attachment `gorm:"type:json;serializer:json" ref:"[]file"` + Result json.Data `gorm:"type:json;serializer:json"` + TaskID uint `gorm:"<-:create;uniqueIndex"` + Task *Task +} + +type TaskGroup struct { + Model + BucketOwner + Name string + Kind string + Addon string + Extensions []string `gorm:"type:json;serializer:json"` + State string + Priority int + Policy TaskPolicy `gorm:"type:json;serializer:json"` + Data json.Data `gorm:"type:json;serializer:json"` + List []Task `gorm:"type:json;serializer:json"` + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` +} + +// Proxy configuration. +// kind = (http|https) +type Proxy struct { + Model + Enabled bool + Kind string `gorm:"uniqueIndex"` + Host string `gorm:"not null"` + Port int + Excluded []string `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity +} + +// Identity represents and identity with a set of credentials. +type Identity struct { + Model + Kind string `gorm:"not null"` + Name string `gorm:"index;unique;not null"` + Description string + User string + Password string + Key string + Settings string + Proxies []Proxy `gorm:"constraint:OnDelete:SET NULL"` + Applications []Application `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` +} + +// Encrypt sensitive fields. +// The ref identity is used to determine when sensitive fields +// have changed and need to be (re)encrypted. +func (r *Identity) Encrypt(ref *Identity) (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != ref.Password { + if r.Password != "" { + r.Password, err = aes.Encrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Key != ref.Key { + if r.Key != "" { + r.Key, err = aes.Encrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Settings != ref.Settings { + if r.Settings != "" { + r.Settings, err = aes.Encrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + return +} + +// Decrypt sensitive fields. +func (r *Identity) Decrypt() (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != "" { + r.Password, err = aes.Decrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Key != "" { + r.Key, err = aes.Decrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Settings != "" { + r.Settings, err = aes.Decrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + return +} + +// +// JSON Fields. +// + +// Attachment file attachment. +type Attachment struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Activity int `json:"activity,omitempty" yaml:",omitempty"` +} + +// TaskError used in Task.Errors. +type TaskError struct { + Severity string `json:"severity"` + Description string `json:"description"` +} + +// TaskEvent task event. +type TaskEvent struct { + Kind string `json:"kind"` + Count int `json:"count"` + Reason string `json:"reason,omitempty" yaml:",omitempty"` + Last time.Time `json:"last"` +} + +// TaskPolicy scheduling policy. +type TaskPolicy struct { + Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` + PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` + PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` +} + +// TTL time-to-live. +type TTL struct { + Created int `json:"created,omitempty" yaml:",omitempty"` + Pending int `json:"pending,omitempty" yaml:",omitempty"` + Running int `json:"running,omitempty" yaml:",omitempty"` + Succeeded int `json:"succeeded,omitempty" yaml:",omitempty"` + Failed int `json:"failed,omitempty" yaml:",omitempty"` +} diff --git a/migration/v15/model/mod.patch b/migration/v15/model/mod.patch new file mode 100644 index 000000000..e2a2ae8ae --- /dev/null +++ b/migration/v15/model/mod.patch @@ -0,0 +1,11 @@ +diff -ruN migration/v14/model/core.go migration/v15/model/core.go +--- migration/v14/model/core.go 2024-09-20 04:44:49.750736163 -0700 ++++ migration/v15/model/core.go 2024-09-20 04:47:13.750375198 -0700 +@@ -95,6 +95,7 @@ + type File struct { + Model + Name string ++ Encoding string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time + } diff --git a/migration/v15/model/pkg.go b/migration/v15/model/pkg.go new file mode 100644 index 000000000..8f612b488 --- /dev/null +++ b/migration/v15/model/pkg.go @@ -0,0 +1,56 @@ +package model + +import ( + "github.com/konveyor/tackle2-hub/settings" +) + +var ( + Settings = &settings.Settings +) + +// JSON field (data) type. +type JSON = []byte + +// All builds all models. +// Models are enumerated such that each are listed after +// all the other models on which they may depend. +func All() []any { + return []any{ + Application{}, + TechDependency{}, + Incident{}, + Analysis{}, + Issue{}, + Bucket{}, + BusinessService{}, + Dependency{}, + File{}, + Fact{}, + Identity{}, + Import{}, + ImportSummary{}, + ImportTag{}, + JobFunction{}, + MigrationWave{}, + PK{}, + Proxy{}, + Review{}, + Setting{}, + RuleSet{}, + Rule{}, + Stakeholder{}, + StakeholderGroup{}, + Tag{}, + TagCategory{}, + Target{}, + Task{}, + TaskGroup{}, + TaskReport{}, + Ticket{}, + Tracker{}, + ApplicationTag{}, + Questionnaire{}, + Assessment{}, + Archetype{}, + } +} diff --git a/model/pkg.go b/model/pkg.go index d6b546fb3..f1c5d0cf6 100644 --- a/model/pkg.go +++ b/model/pkg.go @@ -2,7 +2,7 @@ package model import ( "github.com/konveyor/tackle2-hub/migration/json" - "github.com/konveyor/tackle2-hub/migration/v14/model" + "github.com/konveyor/tackle2-hub/migration/v15/model" ) // Field (data) types.