Update go to v1.20

.github/workflows/go.yml

@@ -47,7 +47,7 @@ jobs:
       uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        file: .coverage/coverage-unit.txt
+        file: .coverage/unit/coverage-unit.txt
         flags: unit-tests
         name: codecov-unit-test
 

.github/workflows/kind.yml

@@ -154,10 +154,18 @@ jobs:
       - name: Run e2e tests
         run: |
           mkdir log
-          ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-e2e-kind.sh
+          ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-e2e-kind.sh --coverage
       - name: Tar log files
         if: ${{ failure() }}
         run: tar -czf log.tar.gz log
+      - name: Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: .coverage/complete-kind-e2e-coverage.txt
+          fail_ci_if_error: ${{ github.event_name == 'push' }}
+          flags: kind-e2e-tests
+          name: codecov-kind-e2e-test
       - name: Upload test log
         uses: actions/upload-artifact@v3
         if: ${{ failure() }}

Makefile

@@ -44,7 +44,7 @@ trivy-scan: .trivy-bin check-TRIVY_TARGET_IMAGE
 
 .PHONY: .coverage
 .coverage:
-	mkdir -p $(CURDIR)/.coverage
+	mkdir -p $(CURDIR)/.coverage/unit
 
 .PHONY: test-unit
 ifeq ($(UNAME_S),Linux)
@@ -72,7 +72,7 @@ DOCKER_ENV := \
 		-v $(DOCKER_CACHE)/gopath:/tmp/gopath \
 		-v $(DOCKER_CACHE)/gocache:/tmp/gocache \
 		-v $(CURDIR):/usr/src/antrea.io/theia \
-		golang:1.19
+		golang:1.20
 
 .PHONY: docker-test-unit
 docker-test-unit: $(DOCKER_CACHE)
@@ -96,9 +96,9 @@ add-copyright:
 .linux-test-unit: .coverage
 	@echo
 	@echo "==> Running unit tests <=="
-	$(GO) test -race -coverpkg=antrea.io/theia/plugins/...,antrea.io/theia/pkg/...  \
-	  -coverprofile=.coverage/coverage-unit.txt -covermode=atomic \
-	  antrea.io/theia/plugins/... antrea.io/theia/pkg/... 
+	$(GO) test -race -coverpkg=antrea.io/theia/plugins/...,antrea.io/theia/pkg/...,antrea.io/theia/cmd/...  \
+	  -coverprofile=.coverage/unit/coverage-unit.txt -covermode=atomic \
+	  antrea.io/theia/plugins/... antrea.io/theia/pkg/... antrea.io/theia/cmd/... \
 
 .PHONY: tidy
 tidy:
@@ -118,7 +118,7 @@ fmt:
 
 .golangci-bin:
 	@echo "===> Installing Golangci-lint <==="
-	@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $@ v1.50.0
+	@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $@ v1.52.2
 
 .PHONY: golangci
 golangci: .golangci-bin
@@ -187,7 +187,7 @@ clickhouse-monitor:
 .PHONY: clickhouse-monitor-plugin
 clickhouse-monitor-plugin:
 	@mkdir -p $(BINDIR)
-	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-monitor
+	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -cover -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-monitor
 
 .PHONY: theia-manager
 theia-manager:
@@ -200,7 +200,7 @@ theia-manager:
 .PHONY: theia-manager-bin
 theia-manager-bin:
 	@mkdir -p $(BINDIR)
-	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/cmd/theia-manager
+	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -cover -ldflags '$(LDFLAGS)' antrea.io/theia/cmd/theia-manager
 
 .PHONY: clickhouse-server
 clickhouse-server:
@@ -218,7 +218,7 @@ clickhouse-server-multi-arch:
 .PHONY: clickhouse-schema-management-plugin
 clickhouse-schema-management-plugin:
 	@mkdir -p $(BINDIR)
-	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-schema-management
+	GOOS=linux CGO_ENABLED=0 $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-schema-management
 
 # Theia currently supports two spark jobs, Throughput Anomaly Detection and Policy Recommendation.
 # This Dockerfile helps create unified dockerfile for both the spark jobs.

build/charts/theia/templates/_helpers.tpl

@@ -4,6 +4,9 @@
 - name: clickhouse-monitor
   image: {{ include "clickHouseMonitorImage" . | quote }}
   imagePullPolicy: {{ $clickhouse.monitor.image.pullPolicy }}
+  volumeMounts:
+    - name: clickhouse-monitor-coverage
+      mountPath: /clickhouse-monitor-coverage
   env:
     - name: CLICKHOUSE_USERNAME
       valueFrom:
@@ -31,6 +34,8 @@
       value: {{ $clickhouse.monitor.execInterval }}
     - name: SKIP_ROUNDS_NUM
       value: {{ $clickhouse.monitor.skipRoundsNum | quote }}
+    - name: GOCOVERDIR
+      value: "/clickhouse-monitor-coverage"
 {{- end }}
 
 {{- define "clickhouse.server.container" }}
@@ -88,6 +93,10 @@
     medium: Memory
     sizeLimit: {{ $clickhouse.storage.size }}
 {{- end }}
+- hostPath:
+    path: /var/log/cm-coverage
+    type: DirectoryOrCreate
+  name: clickhouse-monitor-coverage
 {{- end }}
 
 

build/charts/theia/templates/theia-manager/deployment.yaml

@@ -52,6 +52,8 @@ spec:
                   key: password
             - name: CLICKHOUSE_URL
               value: "tcp://clickhouse-clickhouse.{{ .Release.Namespace }}.svc:{{ .Values.clickhouse.service.tcpPort }}"
+            - name: GOCOVERDIR
+              value: "/theia-manager-coverage"
           ports:
             - name: "theia-api-http"
               containerPort: {{ .Values.theiaManager.apiServer.apiPort }}
@@ -63,6 +65,8 @@ spec:
               name: theia-manager-tls
             - mountPath: /var/log/antrea/theia-manager
               name: host-var-log-antrea-theia-manager
+            - mountPath: /theia-manager-coverage
+              name: theia-manager-coverage
       nodeSelector:
         kubernetes.io/os: linux
         kubernetes.io/arch: amd64
@@ -81,4 +85,8 @@ spec:
           hostPath:
             path: /var/log/antrea/theia-manager
             type: DirectoryOrCreate
+        - name: theia-manager-coverage
+          hostPath:
+            path: /var/log/tm-coverage
+            type: DirectoryOrCreate
 {{- end }}

build/charts/theia/values.yaml

@@ -270,3 +270,4 @@ theiaManager:
     tlsMinVersion: ""
   # -- Log verbosity switch for Theia Manager.
   logVerbosity: 0
+

build/images/Dockerfile.clickhouse-monitor.ubuntu

@@ -7,7 +7,8 @@ WORKDIR /theia
 # Statically links clickhouse-monitor-plugin binary.
 RUN CGO_ENABLED=0 make clickhouse-monitor-plugin
 
-FROM scratch
+FROM ubuntu:22.04
+RUN mkdir -p clickhouse-monitor-coverage
 
 LABEL maintainer="Antrea <[email protected]>"
 LABEL description="A docker image to deploy the ClickHouse monitor plugin."

build/images/Dockerfile.theia-manager.ubuntu

@@ -5,6 +5,7 @@ COPY . /theia
 WORKDIR /theia
 
 RUN make theia-manager-bin
+RUN mkdir theia-manager-coverage
 
 # Chose this base image so that a shell is available for users to exec into the container
 FROM ubuntu:23.04

build/images/deps/go-version

@@ -1 +1 @@
-1.19
+1.20

build/yamls/flow-visibility.yml

@@ -6977,6 +6977,8 @@ spec:
               name: clickhouse-secret
         - name: CLICKHOUSE_URL
           value: tcp://clickhouse-clickhouse.flow-visibility.svc:9000
+        - name: GOCOVERDIR
+          value: /theia-manager-coverage
         image: projects.registry.vmware.com/antrea/theia-manager:latest
         imagePullPolicy: IfNotPresent
         name: theia-manager
@@ -6991,6 +6993,8 @@ spec:
           name: theia-manager-tls
         - mountPath: /var/log/antrea/theia-manager
           name: host-var-log-antrea-theia-manager
+        - mountPath: /theia-manager-coverage
+          name: theia-manager-coverage
       nodeSelector:
         kubernetes.io/arch: amd64
         kubernetes.io/os: linux
@@ -7008,6 +7012,10 @@ spec:
           path: /var/log/antrea/theia-manager
           type: DirectoryOrCreate
         name: host-var-log-antrea-theia-manager
+      - hostPath:
+          path: /var/log/tm-coverage
+          type: DirectoryOrCreate
+        name: theia-manager-coverage
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -7282,9 +7290,14 @@ spec:
             value: 1m
           - name: SKIP_ROUNDS_NUM
             value: "3"
+          - name: GOCOVERDIR
+            value: /clickhouse-monitor-coverage
           image: projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
           imagePullPolicy: IfNotPresent
           name: clickhouse-monitor
+          volumeMounts:
+          - mountPath: /clickhouse-monitor-coverage
+            name: clickhouse-monitor-coverage
         volumes:
         - configMap:
             items:
@@ -7318,6 +7331,10 @@ spec:
             medium: Memory
             sizeLimit: 8Gi
           name: clickhouse-storage-volume
+        - hostPath:
+            path: /var/log/cm-coverage
+            type: DirectoryOrCreate
+          name: clickhouse-monitor-coverage
     serviceTemplates:
     - name: service-template
       spec:

ci/kind/test-e2e-kind.sh

@@ -28,6 +28,7 @@ _usage="Usage: $0 [--ip-family <v4|v6>] [--help|-h]
         --setup-only                  Only perform setting up the cluster and run test.
         --cleanup-only                Only perform cleaning up the cluster.
         --test-only                   Only run test on current cluster. Not set up/clean up the cluster.
+        --coverage                    Collect coverage data while running test.
         --help, -h                    Print this message and exit.
 "
 
@@ -60,6 +61,7 @@ skiplist=""
 setup_only=false
 cleanup_only=false
 test_only=false
+coverage=false
 while [[ $# -gt 0 ]]
 do
 key="$1"
@@ -85,6 +87,10 @@ case $key in
     test_only=true
     shift
     ;;
+    --coverage)
+    coverage=true
+    shift
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -100,6 +106,11 @@ if [[ $cleanup_only == "true" ]];then
   $TESTBED_CMD destroy kind
   exit 0
 fi
+if [[ $coverage == "true" ]];then
+  mkdir -p .coverage/clickhouse-monitor-coverage/
+  mkdir -p .coverage/theia-manager-coverage/
+  mkdir -p .coverage/merged/
+fi
 
 trap "quit" INT EXIT
 
@@ -173,14 +184,32 @@ function run_test {
   rm -rf $TMP_DIR
   sleep 1
 
-  go test -v -timeout=45m antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist
+  if $coverage; then
+    go test -v -timeout=30m antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR -cover -covermode=atomic --skip=$skiplist
+  else
+    go test -v -timeout=30m antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist
+  fi
+
+}
+
+function coverage_and_cleanup_test {
+  if $coverage; then
+    COVERAGE=true go test -v -timeout=10m -run=TestCoverageAndCleanup antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist
+    touch .coverage/complete-kind-e2e-coverage.txt
+    go tool covdata merge -i=.coverage/clickhouse-monitor-coverage,.coverage/theia-manager-coverage -o .coverage/merged
+    go tool covdata textfmt -i=.coverage/merged -o .coverage/complete-kind-e2e-coverage.txt
+    rm -rf .coverage/clickhouse-monitor-coverage
+    rm -rf .coverage/theia-manager-coverage
+    rm -rf .coverage/merged
+  fi
 }
 
 echo "======== Test encap mode =========="
 if [[ $test_only == "false" ]];then
   setup_cluster "--images \"$COMMON_IMAGES\""
 fi
 run_test
+coverage_and_cleanup_test
 
 rm -rf $PWD/bin
 

codecov.yml

@@ -33,6 +33,16 @@ coverage:
         threshold: 1%
         flags:
           - unit-tests
+      theia-kind-e2e-tests:
+        target: 60%
+        threshold: 1%
+        flags:
+          - kind-e2e-tests
+      python-unit-tests:
+        target: 50%
+        threshold: 1%
+        flags:
+          - python-coverage
 
 flag_management:
   default_rules:
@@ -45,3 +55,4 @@ ignore:
   - "**/pkg/client"
   - "third_party"
   - "pkg/api"
+  - "snowflake"

go.mod

@@ -1,6 +1,6 @@
 module antrea.io/theia
 
-go 1.19
+go 1.20
 
 require (
 	antrea.io/antrea v1.11.1

pkg/theia/commands/anomaly_detection_list_test.go

@@ -114,11 +114,11 @@ func TestAnomalyDetectionList(t *testing.T) {
 			orig := os.Stdout
 			r, w, _ := os.Pipe()
 			os.Stdout = w
+			defer func() { os.Stdout = orig }()
 			err := anomalyDetectionList(cmd, []string{})
 			if tt.expectedErrorMsg == "" {
 				assert.NoError(t, err)
 				outcome := readStdout(t, r, w)
-				os.Stdout = orig
 				assert.Contains(t, outcome, "test1")
 				for _, msg := range tt.expectedMsg {
 					assert.Contains(t, outcome, msg)

pkg/theia/commands/anomaly_detection_retrieve_test.go

@@ -302,6 +302,7 @@ func TestAnomalyDetectorRetrieve(t *testing.T) {
 			orig := os.Stdout
 			r, w, _ := os.Pipe()
 			os.Stdout = w
+			defer func() { os.Stdout = orig }()
 			err := throughputAnomalyDetectionRetrieve(cmd, []string{})
 			if tt.expectedErrorMsg == "" {
 				if tt.filePath != "" {
@@ -314,7 +315,6 @@ func TestAnomalyDetectorRetrieve(t *testing.T) {
 				} else {
 					assert.NoError(t, err)
 					outcome := readStdouttad(t, r, w)
-					os.Stdout = orig
 					for _, msg := range tt.expectedMsg {
 						assert.Contains(t, outcome, msg)
 					}

pkg/theia/commands/anomaly_detection_status_test.go

@@ -181,6 +181,7 @@ func TestAnomalyDetectorStatus(t *testing.T) {
 			orig := os.Stdout
 			r, w, _ := os.Pipe()
 			os.Stdout = w
+			defer func() { os.Stdout = orig }()
 			if tt.name == "Valid case with args" {
 				err = anomalyDetectionStatus(cmd, []string{tadName})
 			} else {
@@ -189,7 +190,6 @@ func TestAnomalyDetectorStatus(t *testing.T) {
 			if tt.expectedErrorMsg == "" {
 				assert.NoError(t, err)
 				outcome := readStdout(t, r, w)
-				os.Stdout = orig
 				for _, msg := range tt.expectedMsg {
 					assert.Contains(t, outcome, msg)
 				}

pkg/theia/commands/clickhouse_status_test.go

@@ -219,11 +219,11 @@ func TestGetStatus(t *testing.T) {
 			orig := os.Stdout
 			r, w, _ := os.Pipe()
 			os.Stdout = w
+			defer func() { os.Stdout = orig }()
 			err := getStatus(cmd, []string{})
 			if tt.expectedErrorMsg == "" {
 				assert.NoError(t, err)
 				outcome := readStdout(t, r, w)
-				os.Stdout = orig
 				for _, msg := range tt.expectedMsg {
 					assert.Contains(t, outcome, msg)
 				}