ci: divide CI workflows (#9243)

This PR divides ci-dgraph-tests and ci-dgraph-upgrade-tests workflows into three smaller workflows: core, systest, and vector tests. - Vector tests include only test packages related to vectors. - Systest includes system test packages such as backup, restore, export, import, etc. - All remaining packages fall under the core suite.
hypermodeinc · Dec 19, 2024 · 6207892 · 6207892
1 parent d59c337
commit 6207892
Show file tree

Hide file tree

Showing 6 changed files with 215 additions and 11 deletions.
diff --git a/.github/workflows/ci-dgraph-tests.yml → .github/workflows/ci-dgraph-core-tests.yml b/.github/workflows/ci-dgraph-tests.yml → .github/workflows/ci-dgraph-core-tests.yml
@@ -1,4 +1,4 @@
-name: ci-dgraph-tests
+name: ci-dgraph-core-tests
 on:
   pull_request:
     paths:
@@ -13,7 +13,7 @@ on:
       - main
       - 'release/**'
 jobs:
-  dgraph-tests:
+  dgraph-core-tests:
     if: github.event.pull_request.draft == false
     runs-on: warp-ubuntu-latest-x64-4x
     steps:
@@ -44,15 +44,15 @@ jobs:
           go clean -testcache
           # clean up docker containers before test execution
           cd t; ./t -r
-      - name: Run Unit Tests
+      - name: Run Core Tests
         run: |
           #!/bin/bash
           # go env settings
           export GOPATH=~/go
           # move the binary
           cp dgraph/dgraph ~/go/bin/dgraph
-          # run the unit and integration tests
-          cd t; ./t
+          # run the unit and core tests
+          cd t; ./t --suite=core
           # clean up docker containers after test execution
           ./t -r
           # sleep

diff --git a/...hub/workflows/ci-dgraph-upgrade-tests.yml → ...orkflows/ci-dgraph-core-upgrade-tests.yml b/...hub/workflows/ci-dgraph-upgrade-tests.yml → ...orkflows/ci-dgraph-core-upgrade-tests.yml
@@ -1,4 +1,4 @@
-name: ci-dgraph-upgrade-tests
+name: ci-dgraph-core-upgrade-tests
 on:
   pull_request:
     paths:
@@ -31,16 +31,20 @@ jobs:
           #!/bin/bash
           # clean cache
           go clean -testcache
-      - name: Run Upgrade Tests
+      - name: Run Core Upgrade Tests
         run: |
           #!/bin/bash
           # go env settings
           export GOPATH=~/go
           export DGRAPH_UPGRADE_MAIN_ONLY=true
           # move the binary
           cp dgraph/dgraph ~/go/bin/dgraph
-          # run the tests
-          go test -v -timeout=120m -failfast -tags=upgrade ./...
+          # run the core upgrade tests
+          go test -tags=upgrade \
+          github.com/dgraph-io/dgraph/v24/ee/acl \
+          github.com/dgraph-io/dgraph/v24/worker \
+          github.com/dgraph-io/dgraph/v24/query \
+          -v -timeout=120m -failfast
           # clean up docker containers after test execution
           go clean -testcache
           # sleep

diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests.yml b/.github/workflows/ci-dgraph-system-upgrade-tests.yml
@@ -0,0 +1,51 @@
+name: ci-dgraph-system-upgrade-tests
+on:
+  pull_request:
+    paths:
+      - '**/*.go'
+      - '**/go.mod'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    branches:
+      - main
+      - 'release/**'
+jobs:
+  dgraph-upgrade-tests:
+    if: github.event.pull_request.draft == false
+    runs-on: warp-ubuntu-latest-x64-4x
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Make Linux Build and Docker Image
+        run: make docker-image
+      - name: Clean Up Environment
+        run: |
+          #!/bin/bash
+          # clean cache
+          go clean -testcache
+      - name: Run System Upgrade Tests
+        run: |
+          #!/bin/bash
+          # go env settings
+          export GOPATH=~/go
+          export DGRAPH_UPGRADE_MAIN_ONLY=true
+          # move the binary
+          cp dgraph/dgraph ~/go/bin/dgraph
+          # run the sytem upgrade tests
+          go test -tags=upgrade  github.com/dgraph-io/dgraph/v24/systest/mutations-and-queries \
+          github.com/dgraph-io/dgraph/v24/systest/plugin \
+          github.com/dgraph-io/dgraph/v24/systest/license \
+          github.com/dgraph-io/dgraph/v24/systest/multi-tenancy \
+          -v -timeout=120m -failfast
+          # clean up docker containers after test execution
+          go clean -testcache
+          # sleep
+          sleep 5
diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml
@@ -0,0 +1,59 @@
+name: ci-dgraph-system-tests
+on:
+  pull_request:
+    paths:
+      - '**/*.go'
+      - '**/go.mod'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    branches:
+      - main
+      - 'release/**'
+jobs:
+  dgraph-systest-tests:
+    if: github.event.pull_request.draft == false
+    runs-on: warp-ubuntu-latest-x64-4x
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Install protobuf-compiler
+        run: sudo apt update && sudo apt install -y protobuf-compiler
+      - name: Check protobuf
+        run: |
+          cd ./protos
+          go mod tidy
+          make regenerate
+          git diff --exit-code -- .
+      - name: Make Linux Build and Docker Image
+        run: make docker-image
+      - name: Build Test Binary
+        run: |
+          #!/bin/bash
+          # build the test binary
+          cd t; go build .
+      - name: Clean Up Environment
+        run: |
+          #!/bin/bash
+          # clean cache
+          go clean -testcache
+          # clean up docker containers before test execution
+          cd t; ./t -r
+      - name: Run Systests
+        run: |
+          #!/bin/bash
+          # go env settings
+          export GOPATH=~/go
+          # move the binary
+          cp dgraph/dgraph ~/go/bin/dgraph
+          # run the unit and systests
+          cd t; ./t --suite=systest
+          # clean up docker containers after test execution
+          ./t -r
+          # sleep
+          sleep 5
diff --git a/.github/workflows/ci-dgraph-vector-tests.yml b/.github/workflows/ci-dgraph-vector-tests.yml
@@ -0,0 +1,59 @@
+name: ci-dgraph-vector-tests
+on:
+  pull_request:
+    paths:
+      - '**/*.go'
+      - '**/go.mod'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    branches:
+      - main
+      - 'release/**'
+jobs:
+  dgraph-vector-tests:
+    if: github.event.pull_request.draft == false
+    runs-on: warp-ubuntu-latest-x64-4x
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Install protobuf-compiler
+        run: sudo apt update && sudo apt install -y protobuf-compiler
+      - name: Check protobuf
+        run: |
+          cd ./protos
+          go mod tidy
+          make regenerate
+          git diff --exit-code -- .
+      - name: Make Linux Build and Docker Image
+        run: make docker-image
+      - name: Build Test Binary
+        run: |
+          #!/bin/bash
+          # build the test binary
+          cd t; go build .
+      - name: Clean Up Environment
+        run: |
+          #!/bin/bash
+          # clean cache
+          go clean -testcache
+          # clean up docker containers before test execution
+          cd t; ./t -r
+      - name: Run Vector Tests
+        run: |
+          #!/bin/bash
+          # go env settings
+          export GOPATH=~/go
+          # move the binary
+          cp dgraph/dgraph ~/go/bin/dgraph
+          # run the unit and vector tests
+          cd t; ./t --suite=vector
+          # clean up docker containers after test execution
+          ./t -r
+          # sleep
+          sleep 5
diff --git a/t/t.go b/t/t.go
@@ -720,7 +720,7 @@ func testSuiteContains(suite string) bool {
 func isValidPackageForSuite(pkg string) bool {
 	valid := false
 	if testSuiteContains("all") {
-		valid = true
+		return true
 	}
 	if testSuiteContains("ldbc") {
 		valid = valid || isLDBCPackage(pkg)
@@ -731,6 +731,15 @@ func isValidPackageForSuite(pkg string) bool {
 	if testSuiteContains("unit") {
 		valid = valid || (!isLoadPackage(pkg) && !isLDBCPackage(pkg))
 	}
+	if testSuiteContains("vector") {
+		valid = valid || isVectorPackage(pkg)
+	}
+	if testSuiteContains("systest") {
+		valid = valid || isSystestPackage(pkg)
+	}
+	if testSuiteContains("core") {
+		valid = valid || isCorePackage(pkg)
+	}
 	if valid {
 		return valid
 	}
@@ -750,6 +759,28 @@ func isLDBCPackage(pkg string) bool {
 	return strings.HasSuffix(pkg, "/systest/ldbc")
 }
 
+func isSystestPackage(pkg string) bool {
+	if !strings.Contains(pkg, "/systest") {
+		return false
+	}
+	return !isExcludedFromSystest(pkg)
+}
+
+func isExcludedFromSystest(pkg string) bool {
+	return isLDBCPackage(pkg) || isLoadPackage(pkg) || isVectorPackage(pkg)
+}
+
+func isCorePackage(pkg string) bool {
+	if isSystestPackage(pkg) || isLDBCPackage(pkg) || isVectorPackage(pkg) || isLoadPackage(pkg) {
+		return false
+	}
+	return true
+}
+
+func isVectorPackage(pkg string) bool {
+	return strings.HasSuffix(pkg, "/vector")
+}
+
 var datafiles = map[string]string{
 	"1million-noindex.schema": "https://github.com/dgraph-io/benchmarks/blob/master/data/1million-noindex.schema?raw=true",
 	"1million.schema":         "https://github.com/dgraph-io/benchmarks/blob/master/data/1million.schema?raw=true",
@@ -1041,7 +1072,7 @@ func run() error {
 
 func validateAllowed(testSuite []string) {
 
-	allowed := []string{"all", "ldbc", "load", "unit"}
+	allowed := []string{"all", "ldbc", "load", "unit", "systest", "vector", "core"}
 	for _, str := range testSuite {
 		onlyAllowed := false
 		for _, allowedStr := range allowed {