Merge branch 'main' into chore/database/housekeeping

.github/workflows/integration-test.yml

@@ -0,0 +1,49 @@
+# name of the action
+name: integration-test
+
+# trigger on pull_request events that modify this file or any database files
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/integration-test.yml'
+      - 'database/**'
+
+# pipeline to execute
+jobs:
+  database:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15-alpine
+        env:
+          POSTGRES_DB: vela
+          POSTGRES_PASSWORD: notARealPassword12345
+          POSTGRES_USER: vela
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    env:
+      POSTGRES_ADDR: postgres://vela:notARealPassword12345@localhost:5432/vela
+      SQLITE_ADDR: vela.db
+
+    steps:
+      - name: clone
+        uses: actions/checkout@v3
+
+      - name: install go
+        uses: actions/setup-go@v4
+        with:
+          # use version from go.mod file
+          go-version-file: 'go.mod'
+          cache: true
+          check-latest: true
+
+      - name: test
+        run: |
+          make integration-test

.github/workflows/test.yml

@@ -25,10 +25,10 @@ jobs:
 
     - name: test
       run: |
-        go test -race -covermode=atomic -coverprofile=coverage.out ./...
+        make test
 
     - name: coverage
       uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.out
+        file: coverage.out

Dockerfile

@@ -2,7 +2,7 @@
 #
 # Use of this source code is governed by the LICENSE file in this repository.
 
-FROM alpine as certs
+FROM alpine:3.18.2@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70 as certs
 
 RUN apk add --update --no-cache ca-certificates
 

Dockerfile-alpine

@@ -2,7 +2,7 @@
 #
 # Use of this source code is governed by the LICENSE file in this repository.
 
-FROM alpine
+FROM alpine:3.18.2@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70
 
 RUN apk add --update --no-cache ca-certificates
 

Makefile

@@ -91,6 +91,13 @@ fix:
 	@echo "### Fixing Go Code"
 	@go fix ./...
 
+# The `integration-test` target is intended to run all integration tests for the Go source code.
+.PHONY: integration-test
+integration-test:
+	@echo
+	@echo "### Integration Testing"
+	INTEGRATION=1 go test -run TestDatabase_Integration ./...
+
 # The `test` target is intended to run
 # the tests for the Go source code.
 #
@@ -99,18 +106,15 @@ fix:
 test:
 	@echo
 	@echo "### Testing Go Code"
-	@go test -race ./...
+	@go test -race -covermode=atomic -coverprofile=coverage.out ./...
 
 # The `test-cover` target is intended to run
 # the tests for the Go source code and then
 # open the test coverage report.
 #
 # Usage: `make test-cover`
 .PHONY: test-cover
-test-cover:
-	@echo
-	@echo "### Creating test coverage report"
-	@go test -race -covermode=atomic -coverprofile=coverage.out ./...
+test-cover: test
 	@echo
 	@echo "### Opening test coverage report"
 	@go tool cover -html=coverage.out

api/admin/repo.go

@@ -66,7 +66,7 @@ func UpdateRepo(c *gin.Context) {
 	}
 
 	// send API call to update the repo
-	err = database.FromContext(c).UpdateRepo(input)
+	r, err := database.FromContext(c).UpdateRepo(input)
 	if err != nil {
 		retErr := fmt.Errorf("unable to update repo %d: %w", input.GetID(), err)
 
@@ -75,5 +75,5 @@ func UpdateRepo(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, input)
+	c.JSON(http.StatusOK, r)
 }

api/build/create.go

@@ -329,7 +329,7 @@ func CreateBuild(c *gin.Context) {
 	}
 
 	// send API call to update repo for ensuring counter is incremented
-	err = database.FromContext(c).UpdateRepo(r)
+	r, err = database.FromContext(c).UpdateRepo(r)
 	if err != nil {
 		retErr := fmt.Errorf("unable to create new build: failed to update repo %s: %w", r.GetFullName(), err)
 

api/build/restart.go

@@ -320,7 +320,7 @@ func RestartBuild(c *gin.Context) {
 	}
 
 	// send API call to update repo for ensuring counter is incremented
-	err = database.FromContext(c).UpdateRepo(r)
+	r, err = database.FromContext(c).UpdateRepo(r)
 	if err != nil {
 		retErr := fmt.Errorf("unable to restart build: failed to update repo %s: %w", r.GetFullName(), err)
 		util.HandleError(c, http.StatusBadRequest, retErr)

api/pipeline/compile.go

@@ -97,7 +97,7 @@ func CompilePipeline(c *gin.Context) {
 	compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u)
 
 	// compile the pipeline
-	pipeline, _, err := compiler.CompileLite(p.GetData(), true, true, nil)
+	pipeline, _, err := compiler.CompileLite(p.GetData(), true, true)
 	if err != nil {
 		retErr := fmt.Errorf("unable to compile pipeline %s: %w", entry, err)
 

api/pipeline/expand.go

@@ -98,7 +98,7 @@ func ExpandPipeline(c *gin.Context) {
 	compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u)
 
 	// expand the templates in the pipeline
-	pipeline, _, err := compiler.CompileLite(p.GetData(), true, false, nil)
+	pipeline, _, err := compiler.CompileLite(p.GetData(), true, false)
 	if err != nil {
 		retErr := fmt.Errorf("unable to expand pipeline %s: %w", entry, err)
 

api/pipeline/validate.go

@@ -105,7 +105,7 @@ func ValidatePipeline(c *gin.Context) {
 	}
 
 	// validate the pipeline
-	pipeline, _, err := compiler.CompileLite(p.GetData(), template, false, nil)
+	pipeline, _, err := compiler.CompileLite(p.GetData(), template, false)
 	if err != nil {
 		retErr := fmt.Errorf("unable to validate pipeline %s: %w", entry, err)
 

api/repo/chown.go

@@ -68,7 +68,7 @@ func ChownRepo(c *gin.Context) {
 	r.SetUserID(u.GetID())
 
 	// send API call to update the repo
-	err := database.FromContext(c).UpdateRepo(r)
+	_, err := database.FromContext(c).UpdateRepo(r)
 	if err != nil {
 		retErr := fmt.Errorf("unable to change owner of repo %s to %s: %w", r.GetFullName(), u.GetName(), err)
 

api/repo/create.go

@@ -285,30 +285,24 @@ func CreateRepo(c *gin.Context) {
 		dbRepo.SetActive(true)
 
 		// send API call to update the repo
-		err = database.FromContext(c).UpdateRepo(dbRepo)
+		r, err = database.FromContext(c).UpdateRepo(dbRepo)
 		if err != nil {
 			retErr := fmt.Errorf("unable to set repo %s to active: %w", dbRepo.GetFullName(), err)
 
 			util.HandleError(c, http.StatusInternalServerError, retErr)
 
 			return
 		}
-
-		// send API call to capture the updated repo
-		r, _ = database.FromContext(c).GetRepoForOrg(dbRepo.GetOrg(), dbRepo.GetName())
 	} else {
 		// send API call to create the repo
-		err = database.FromContext(c).CreateRepo(r)
+		r, err = database.FromContext(c).CreateRepo(r)
 		if err != nil {
 			retErr := fmt.Errorf("unable to create new repo %s: %w", r.GetFullName(), err)
 
 			util.HandleError(c, http.StatusInternalServerError, retErr)
 
 			return
 		}
-
-		// send API call to capture the created repo
-		r, _ = database.FromContext(c).GetRepoForOrg(r.GetOrg(), r.GetName())
 	}
 
 	// create init hook in the DB after repo has been added in order to capture its ID

api/repo/delete.go

@@ -88,7 +88,7 @@ func DeleteRepo(c *gin.Context) {
 	// Mark the repo as inactive
 	r.SetActive(false)
 
-	err = database.FromContext(c).UpdateRepo(r)
+	_, err = database.FromContext(c).UpdateRepo(r)
 	if err != nil {
 		retErr := fmt.Errorf("unable to set repo %s to inactive: %w", r.GetFullName(), err)
 

api/repo/repair.go

@@ -122,7 +122,7 @@ func RepairRepo(c *gin.Context) {
 		r.SetActive(true)
 
 		// send API call to update the repo
-		err := database.FromContext(c).UpdateRepo(r)
+		_, err := database.FromContext(c).UpdateRepo(r)
 		if err != nil {
 			retErr := fmt.Errorf("unable to set repo %s to active: %w", r.GetFullName(), err)
 

api/repo/update.go

@@ -295,7 +295,7 @@ func UpdateRepo(c *gin.Context) {
 	}
 
 	// send API call to update the repo
-	err = database.FromContext(c).UpdateRepo(r)
+	r, err = database.FromContext(c).UpdateRepo(r)
 	if err != nil {
 		retErr := fmt.Errorf("unable to update repo %s: %w", r.GetFullName(), err)
 
@@ -304,8 +304,5 @@ func UpdateRepo(c *gin.Context) {
 		return
 	}
 
-	// send API call to capture the updated repo
-	r, _ = database.FromContext(c).GetRepoForOrg(r.GetOrg(), r.GetName())
-
 	c.JSON(http.StatusOK, r)
 }

api/scm/sync.go

@@ -80,7 +80,7 @@ func SyncRepo(c *gin.Context) {
 		r.SetActive(false)
 
 		// update repo in database
-		err := database.FromContext(c).UpdateRepo(r)
+		_, err := database.FromContext(c).UpdateRepo(r)
 		if err != nil {
 			retErr := fmt.Errorf("unable to update repo for org %s: %w", o, err)
 

api/scm/sync_org.go

@@ -113,7 +113,7 @@ func SyncReposForOrg(c *gin.Context) {
 		if err != nil {
 			repo.SetActive(false)
 
-			err := database.FromContext(c).UpdateRepo(repo)
+			_, err := database.FromContext(c).UpdateRepo(repo)
 			if err != nil {
 				retErr := fmt.Errorf("unable to update repo for org %s: %w", o, err)
 

api/webhook/post.go

@@ -621,7 +621,7 @@ func PostWebhook(c *gin.Context) {
 	} // end of retry loop
 
 	// send API call to update repo for ensuring counter is incremented
-	err = database.FromContext(c).UpdateRepo(repo)
+	repo, err = database.FromContext(c).UpdateRepo(repo)
 	if err != nil {
 		retErr := fmt.Errorf("%s: failed to update repo %s: %w", baseErr, repo.GetFullName(), err)
 		util.HandleError(c, http.StatusBadRequest, retErr)
@@ -759,7 +759,7 @@ func handleRepositoryEvent(c *gin.Context, m *types.Metadata, h *library.Hook, r
 		}
 
 		// update repo object in the database after applying edits
-		err = database.FromContext(c).UpdateRepo(dbRepo)
+		dbRepo, err = database.FromContext(c).UpdateRepo(dbRepo)
 		if err != nil {
 			retErr := fmt.Errorf("%s: failed to update repo %s: %w", baseErr, r.GetFullName(), err)
 
@@ -891,7 +891,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types
 	dbR.SetPreviousName(r.GetPreviousName())
 
 	// update the repo in the database
-	err = database.FromContext(c).UpdateRepo(dbR)
+	dbR, err = database.FromContext(c).UpdateRepo(dbR)
 	if err != nil {
 		retErr := fmt.Errorf("%s: failed to update repo %s/%s in database", baseErr, prevOrg, prevRepo)
 		util.HandleError(c, http.StatusBadRequest, retErr)

cmd/vela-server/schedule.go

@@ -373,7 +373,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat
 	} // end of retry loop
 
 	// send API call to update repo for ensuring counter is incremented
-	err = database.UpdateRepo(r)
+	r, err = database.UpdateRepo(r)
 	if err != nil {
 		return fmt.Errorf("unable to update repo %s: %w", r.GetFullName(), err)
 	}

compiler/engine.go

@@ -25,7 +25,7 @@ type Engine interface {
 	// CompileLite defines a function that produces an light executable
 	// representation of a pipeline from an object. This calls
 	// Parse internally to convert the object to a yaml configuration.
-	CompileLite(interface{}, bool, bool, []string) (*yaml.Build, *library.Pipeline, error)
+	CompileLite(interface{}, bool, bool) (*yaml.Build, *library.Pipeline, error)
 
 	// Duplicate defines a function that
 	// creates a clone of the Engine.
@@ -130,6 +130,9 @@ type Engine interface {
 	// WithLocal defines a function that sets
 	// the compiler local field in the Engine.
 	WithLocal(bool) Engine
+	// WithLocalTemplates defines a function that sets
+	// the compiler local templates field in the Engine.
+	WithLocalTemplates([]string) Engine
 	// WithMetadata defines a function that sets
 	// the compiler Metadata type in the Engine.
 	WithMetadata(*types.Metadata) Engine