Skip to content

Commit

Permalink
✨ Upload files instead of multi-part form. (#743)
Browse files Browse the repository at this point in the history 
Post a _manifest_ file instead of separate multi-part form files.
Much simpler and more easily supports the addon staging the issues and
deps files on disk rather than streaming. The more atomic approach will
prevent transaction deadlock which can more easily occur when the
addon-analyzer builder reported an error (which it should never do).

The uploaded file contains markers used to delimited the documents.
`^]` = `\x1D` = GS (group separator).

```
^]BEGIN-MAIN^]
---
commit: 1234
^]END-MAIN^]
^]BEGIN-ISSUES^]
---
ruleset: ruleset-1
rule: rule-1
incidents:
...
^]END-ISSUES^]
^]BEGIN-DEPS^]
---
name: github.com/jboss
version: 4.0
labels:
- konveyor.io/language=java
- konveyor.io/otherA=dog
^]END-DEPS^]
```

Flow:
1. post (upload) manifest.yaml file.
2. post `ref` to the manifest file.
3. delete manifest file.

Orphaned files will be reaped.

---

The binding client needed to be updated to handle different file
encoding (MIME).

---------

Signed-off-by: Jeff Ortel <[email protected]>
  • Loading branch information
@jortel
jortel authored Sep 26, 2024
1 parent 7477257 commit e15a8a4
Show file tree
Hide file tree
Showing 15 changed files with 1,374 additions and 151 deletions.
198 changes: 163 additions & 35 deletions api/analysis.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
package api

import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -53,9 +56,15 @@ const (
AppAnalysisIssuesRoot = AppAnalysisRoot + "/issues"
)

// Manifest markers.
// The GS=\x1D (group separator).
const (
IssueField = "issues"
DepField = "dependencies"
BeginMainMarker = "\x1DBEGIN-MAIN\x1D"
EndMainMarker = "\x1DEND-MAIN\x1D"
BeginIssuesMarker = "\x1DBEGIN-ISSUES\x1D"
EndIssuesMarker = "\x1DEND-ISSUES\x1D"
BeginDepsMarker = "\x1DBEGIN-DEPS\x1D"
EndDepsMarker = "\x1DEND-DEPS\x1D"
)

// AnalysisHandler handles analysis resource routes.
Expand Down Expand Up @@ -315,9 +324,20 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) {
// @summary Create an analysis.
// @description Create an analysis.
// @description Form fields:
// @description - file: file that contains the api.Analysis resource.
// @description - issues: file that multiple api.Issue resources.
// @description - dependencies: file that multiple api.TechDependency resources.
// @description file: A manifest file that contains 3 sections
// @description containing documents delimited by markers.
// @description The manifest must contain ALL markers even when sections are empty.
// @description Note: `^]` = `\x1D` = GS (group separator).
// @description Section markers:
// @description ^]BEGIN-MAIN^]
// @description ^]END-MAIN^]
// @description ^]BEGIN-ISSUES^]
// @description ^]END-ISSUES^]
// @description ^]BEGIN-DEPS^]
// @description ^]END-DEPS^]
// @description The encoding must be:
// @description - application/json
// @description - application/x-yaml
// @tags analyses
// @produce json
// @success 201 {object} api.Analysis
Expand All @@ -337,32 +357,40 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
return
}
}
db := h.DB(ctx)
//
// Analysis
input, err := ctx.FormFile(FileField)
// Manifest
fh := FileHandler{}
name := fmt.Sprintf("app.%d.manifest", id)
file, err := fh.create(ctx, name)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
reader, err := input.Open()
defer func() {
err = fh.delete(ctx, file)
if err != nil {
_ = ctx.Error(err)
}
}()
reader := &ManifestReader{}
f, err := reader.open(file.Path, BeginMainMarker, EndMainMarker)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
defer func() {
_ = reader.Close()
_ = f.Close()
}()
encoding := input.Header.Get(ContentType)
d, err := h.Decoder(ctx, encoding, reader)
d, err := h.Decoder(ctx, file.Encoding, reader)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
r := Analysis{}
err = d.Decode(&r)
r := &Analysis{}
err = d.Decode(r)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
Expand All @@ -371,7 +399,6 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
analysis := r.Model()
analysis.ApplicationID = id
analysis.CreateUser = h.BaseHandler.CurrentUser(ctx)
db := h.DB(ctx)
db.Logger = db.Logger.LogMode(logger.Error)
err = db.Create(analysis).Error
if err != nil {
Expand All @@ -380,23 +407,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
}
//
// Issues
input, err = ctx.FormFile(IssueField)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
reader, err = input.Open()
reader = &ManifestReader{}
f, err = reader.open(file.Path, BeginIssuesMarker, EndIssuesMarker)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
defer func() {
_ = reader.Close()
_ = f.Close()
}()
encoding = input.Header.Get(ContentType)
d, err = h.Decoder(ctx, encoding, reader)
d, err = h.Decoder(ctx, file.Encoding, reader)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
Expand Down Expand Up @@ -425,23 +446,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
}
//
// Dependencies
input, err = ctx.FormFile(DepField)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
reader, err = input.Open()
reader = &ManifestReader{}
f, err = reader.open(file.Path, BeginDepsMarker, EndDepsMarker)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
defer func() {
_ = reader.Close()
_ = f.Close()
}()
encoding = input.Header.Get(ContentType)
d, err = h.Decoder(ctx, encoding, reader)
d, err = h.Decoder(ctx, file.Encoding, reader)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
Expand Down Expand Up @@ -2860,3 +2875,116 @@ func (r *yamlEncoder) embed(object any) encoder {
r.write(s)
return r
}

// ManifestReader analysis manifest reader.
// The manifest contains 3 sections containing documents delimited by markers.
// The manifest must contain ALL markers even when sections are empty.
// Note: `^]` = `\x1D` = GS (group separator).
// Section markers:
//
// ^]BEGIN-MAIN^]
// ^]END-MAIN^]
// ^]BEGIN-ISSUES^]
// ^]END-ISSUES^]
// ^]BEGIN-DEPS^]
// ^]END-DEPS^]
type ManifestReader struct {
file *os.File
marker map[string]int64
begin int64
end int64
read int64
}

// scan manifest and catalog position of markers.
func (r *ManifestReader) scan(path string) (err error) {
if r.marker != nil {
return
}
r.file, err = os.Open(path)
if err != nil {
return
}
defer func() {
_ = r.file.Close()
}()
pattern, err := regexp.Compile(`^\x1D[A-Z-]+\x1D$`)
if err != nil {
return
}
p := int64(0)
r.marker = make(map[string]int64)
scanner := bufio.NewScanner(r.file)
for scanner.Scan() {
content := scanner.Text()
matched := strings.TrimSpace(content)
if pattern.Match([]byte(matched)) {
r.marker[matched] = p
}
p += int64(len(content))
p++
}

return
}

// open returns a read delimited by the specified markers.
func (r *ManifestReader) open(path, begin, end string) (reader io.ReadCloser, err error) {
found := false
err = r.scan(path)
if err != nil {
return
}
r.begin, found = r.marker[begin]
if !found {
err = &BadRequestError{
Reason: fmt.Sprintf("marker: %s not found.", begin),
}
return
}
r.end, found = r.marker[end]
if !found {
err = &BadRequestError{
Reason: fmt.Sprintf("marker: %s not found.", end),
}
return
}
if r.begin >= r.end {
err = &BadRequestError{
Reason: fmt.Sprintf("marker: %s must preceed %s.", begin, end),
}
return
}
r.begin += int64(len(begin))
r.begin++
r.read = r.end - r.begin
r.file, err = os.Open(path)
if err != nil {
return
}
_, err = r.file.Seek(r.begin, io.SeekStart)
reader = r
return
}

// Read bytes.
func (r *ManifestReader) Read(b []byte) (n int, err error) {
n, err = r.file.Read(b)
if n == 0 || err != nil {
return
}
if int64(n) > r.read {
n = int(r.read)
}
r.read -= int64(n)
if n < 1 {
err = io.EOF
}
return
}

// Close the reader.
func (r *ManifestReader) Close() (err error) {
err = r.file.Close()
return
}
Loading

0 comments on commit e15a8a4

Please sign in to comment.