From 61605a7127ee170e875cb7d5ca2cb48e598254fe Mon Sep 17 00:00:00 2001 From: Tushar Tathgur Date: Tue, 18 Jul 2023 16:40:55 -0700 Subject: [PATCH] Separating Upgrade test and ClickHouse Migration Tests Latest changes in Antrea CRDs require us to delete the downgrade and check test, to keep it on par with Antrea, we have separated the upgrade test and the ClickHouse migration tests. In order to test for the data Schema management, we have added new ClickHouse migration test. Signed-off-by: Tushar Tathgur --- .github/workflows/kind.yml | 58 ++++ build/charts/theia/README.md | 2 +- build/charts/theia/values.yaml | 2 +- build/yamls/flow-visibility.yml | 319 +------------------- ci/kind/test-clickhouse-migrate.sh | 263 ++++++++++++++++ ci/kind/test-upgrade-theia.sh | 7 +- test/e2e/clickhouse_migrate_test.go | 177 +++++++++++ test/e2e/fixture.go | 2 +- test/e2e/flowvisibility_test.go | 4 +- test/e2e/framework.go | 16 +- test/e2e/policyrecommendation_test.go | 2 +- test/e2e/theia_clickhouse_test.go | 2 +- test/e2e/throughputanomalydetection_test.go | 2 +- test/e2e/upgrade_test.go | 149 +-------- test/e2e/util.go | 25 +- 15 files changed, 549 insertions(+), 481 deletions(-) create mode 100755 ci/kind/test-clickhouse-migrate.sh create mode 100644 test/e2e/clickhouse_migrate_test.go diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index d7c2b66e..e0ceb244 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -222,6 +222,63 @@ jobs: name: upgrade-from-theia-version-n-1.tar.gz path: log.tar.gz retention-days: 30 + + test-ClickHouse-migrate-from-N-1: + name: Migrate ClickHouse from Theia version N-1 + needs: + - build-clickhouse-monitor-image + - build-clickhouse-server-image + runs-on: [ubuntu-latest] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "/usr/local/lib/android" + df -h + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + - name: Download ClickHouse monitor images from previous jobs + uses: actions/download-artifact@v3 + with: + name: clickhouse-monitor + - name: Load Theia image + run: | + docker load -i clickhouse-monitor.tar + docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest + - name: Download ClickHouse server images from previous jobs + uses: actions/download-artifact@v3 + with: + name: clickhouse-server + - name: Load Theia image + run: | + docker load -i clickhouse-server.tar + docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest + - name: Install Kind + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run migrate test + run: | + mkdir log + ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-clickhouse-migrate.sh --from-version-n-minus 1 + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: migrate-clickhouse-from-theia-version-n-1.tar.gz + path: log.tar.gz + retention-days: 30 # Runs after all other jobs in the workflow succeed and deletes Theia Docker images uploaded as temporary # artifacts. It uses a third-party, MIT-licensed action (geekyeggo/delete-artifact). While Github @@ -236,6 +293,7 @@ jobs: - build-theia-manager-image - test-e2e-encap - test-upgrade-from-N-1 + - test-ClickHouse-migrate-from-N-1 runs-on: [ubuntu-latest] steps: - name: Delete spark-jobs diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md index f198cefc..f2c3f1a7 100644 --- a/build/charts/theia/README.md +++ b/build/charts/theia/README.md @@ -80,7 +80,7 @@ Kubernetes: `>= 1.16.0-0` | theiaManager.apiServer.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "theia-manager-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. | | theiaManager.apiServer.tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Theia Manager APIservers. If empty, the default Go Cipher Suites will be used. | | theiaManager.apiServer.tlsMinVersion | string | `""` | TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. | -| theiaManager.enable | bool | `true` | Determine whether to install Theia Manager. | +| theiaManager.enable | bool | `false` | Determine whether to install Theia Manager. | | theiaManager.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-manager","tag":""}` | Container image used by Theia Manager. | | theiaManager.logVerbosity | int | `0` | Log verbosity switch for Theia Manager. | diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml index 279f1f83..36bdaedf 100644 --- a/build/charts/theia/values.yaml +++ b/build/charts/theia/values.yaml @@ -249,7 +249,7 @@ sparkOperator: tag: "v1beta2-1.3.3-3.1.1" theiaManager: # -- Determine whether to install Theia Manager. - enable: true + enable: false # -- Container image used by Theia Manager. image: repository: "projects.registry.vmware.com/antrea/theia-manager" diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 61b0fcea..c436c9d1 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -18,22 +18,6 @@ metadata: name: grafana namespace: flow-visibility --- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: theia-cli - name: theia-cli - namespace: flow-visibility ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: theia-manager - name: theia-manager - namespace: flow-visibility ---- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -52,144 +36,6 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app: theia-cli - name: theia-cli -rules: -- apiGroups: - - intelligence.theia.antrea.io - resources: - - networkpolicyrecommendations - - throughputanomalydetectors - verbs: - - get - - list - - create - - delete -- apiGroups: - - stats.theia.antrea.io - resources: - - clickhouse - verbs: - - get -- apiGroups: - - system.theia.antrea.io - resources: - - supportbundles - verbs: - - get - - create - - delete -- apiGroups: - - system.theia.antrea.io - resources: - - supportbundles/download - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app: theia-manager - name: theia-manager-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - "" - resourceNames: - - theia-ca - resources: - - configmaps - verbs: - - get - - update -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create -- apiGroups: - - "" - resourceNames: - - extension-apiserver-authentication - resources: - - configmaps - verbs: - - get - - list - - watch -- apiGroups: - - crd.theia.antrea.io - resources: - - networkpolicyrecommendations - - recommendednetworkpolicies - - throughputanomalydetectors - verbs: - - get - - list - - watch - - create - - delete -- apiGroups: - - crd.theia.antrea.io - resources: - - networkpolicyrecommendations/status - - throughputanomalydetectors/status - verbs: - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - get - - create -- apiGroups: - - "" - resources: - - pods/log - verbs: - - get -- apiGroups: - - "" - resources: - - services - - secrets - verbs: - - get -- apiGroups: - - sparkoperator.k8s.io - resources: - - sparkapplications - verbs: - - create - - delete - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -205,36 +51,6 @@ subjects: name: grafana namespace: flow-visibility --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app: theia-cli - name: theia-cli -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: theia-cli -subjects: -- kind: ServiceAccount - name: theia-cli - namespace: flow-visibility ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app: theia-manager - name: theia-manager-cluster-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: theia-manager-role -subjects: -- kind: ServiceAccount - name: theia-manager - namespace: flow-visibility ---- apiVersion: v1 data: 000001_0-1-0.down.sql: "" @@ -6649,36 +6465,6 @@ metadata: namespace: flow-visibility --- apiVersion: v1 -data: - theia-manager.conf: | - # apiServer contains APIServer related configuration options. - apiServer: - # The port for the theia-manager APIServer to serve on. - apiPort: 11347 - - # Indicates whether to use auto-generated self-signed TLS certificate. - # If false, a Secret named "theia-manager-tls" must be provided with the following keys: - # ca.crt: - # tls.crt: - # tls.key: - selfSignedCert: true - - # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. - # https://golang.org/pkg/crypto/tls/#pkg-constants - # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always - # prefer TLS1.3 Cipher Suites whenever possible. - tlsCipherSuites: "" - - # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. - tlsMinVersion: "" -kind: ConfigMap -metadata: - labels: - app: theia-manager - name: theia-manager-configmap - namespace: flow-visibility ---- -apiVersion: v1 kind: Secret metadata: name: clickhouse-secret @@ -6701,15 +6487,6 @@ stringData: type: Opaque --- apiVersion: v1 -kind: Secret -metadata: - annotations: - kubernetes.io/service-account.name: theia-cli - name: theia-cli-account-token - namespace: flow-visibility -type: kubernetes.io/service-account-token ---- -apiVersion: v1 kind: Service metadata: name: grafana @@ -6726,21 +6503,6 @@ spec: --- apiVersion: v1 kind: Service -metadata: - labels: - app: theia-manager - name: theia-manager - namespace: flow-visibility -spec: - ports: - - port: 11347 - protocol: TCP - targetPort: theia-api-http - selector: - app: theia-manager ---- -apiVersion: v1 -kind: Service metadata: labels: app: zookeeper @@ -6931,85 +6693,6 @@ spec: name: grafana-dashboard-config --- apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: theia-manager - name: theia-manager - namespace: flow-visibility -spec: - replicas: 1 - selector: - matchLabels: - app: theia-manager - template: - metadata: - labels: - app: theia-manager - spec: - containers: - - args: - - --config - - /etc/theia-manager/theia-manager.conf - - --logtostderr=false - - --log_dir=/var/log/antrea/theia-manager - - --alsologtostderr - - --log_file_max_size=100 - - --log_file_max_num=4 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CLICKHOUSE_USERNAME - valueFrom: - secretKeyRef: - key: username - name: clickhouse-secret - - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: clickhouse-secret - - name: CLICKHOUSE_URL - value: tcp://clickhouse-clickhouse.flow-visibility.svc:9000 - image: projects.registry.vmware.com/antrea/theia-manager:latest - imagePullPolicy: IfNotPresent - name: theia-manager - ports: - - containerPort: 11347 - name: theia-api-http - volumeMounts: - - mountPath: /etc/theia-manager - name: theia-manager-config - readOnly: true - - mountPath: /var/run/theia/theia-manager-tls - name: theia-manager-tls - - mountPath: /var/log/antrea/theia-manager - name: host-var-log-antrea-theia-manager - nodeSelector: - kubernetes.io/arch: amd64 - kubernetes.io/os: linux - serviceAccountName: theia-manager - volumes: - - configMap: - name: theia-manager-configmap - name: theia-manager-config - - name: theia-manager-tls - secret: - defaultMode: 256 - optional: true - secretName: theia-manager-tls - - hostPath: - path: /var/log/antrea/theia-manager - type: DirectoryOrCreate - name: host-var-log-antrea-theia-manager ---- -apiVersion: apps/v1 kind: StatefulSet metadata: labels: @@ -7282,7 +6965,7 @@ spec: value: 1m - name: SKIP_ROUNDS_NUM value: "3" - image: projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest + image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent name: clickhouse-monitor volumes: diff --git a/ci/kind/test-clickhouse-migrate.sh b/ci/kind/test-clickhouse-migrate.sh new file mode 100755 index 00000000..202cc13f --- /dev/null +++ b/ci/kind/test-clickhouse-migrate.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash + +# Copyright 2022 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function echoerr { + >&2 echo "$@" +} + +THEIA_FROM_TAG= +FROM_VERSION_N_MINUS= + +_usage="Usage: $0 [--from-tag ] [--from-version-n-minus ] +Perform some basic tests to make sure that Theia can be migrated from the provided version to the +current checked-out version. One of [--from-tag ] or [--from-version-n-minus ] must be +provided. + --from-tag Migrate from this version of Theia (pulled from upstream + Antrea) to the current version. + --from-version-n-minus Get all the released versions of Theia and run the migrate + test from the latest bug fix release for *minor* version + N-{COUNT}. N-1 designates the latest minor release. If this + script is run from a release branch, it will only consider + releases which predate that release branch. + --help, -h Print this message and exit +" + +function print_usage { + echoerr "$_usage" +} + +function print_help { + echoerr "Try '$0 --help' for more information." +} + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR=$THIS_DIR/../.. + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --from-tag) + THEIA_FROM_TAG="$2" + shift 2 + ;; + --from-version-n-minus) + FROM_VERSION_N_MINUS="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) # unknown option + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +if [ -z "$THEIA_FROM_TAG" ] && [ -z "$FROM_VERSION_N_MINUS" ]; then + echoerr "One of --from-tag or --from-version-n-minus must be provided" + print_help + exit 1 +fi + +case $FROM_VERSION_N_MINUS in + ''|*[!0-9]*) + echoerr "--from-version-n-minus must be a number greater than 0" + print_help + exit 1 + ;; + *) + ;; +esac + +if [ ! "$FROM_VERSION_N_MINUS" -gt "0" ]; then + echoerr "--from-version-n-minus must be a number greater than 0" + print_help + exit 1 +fi + +function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; } + +# We want to ignore all minor versions greater than the current version, as an migrate test implies +# that we are migrating from an *older* version. This is useful when running this script from a +# release branch (e.g. when testing patch release candidates). +CURRENT_VERSION=$(head -n1 $ROOT_DIR/VERSION) +CURRENT_VERSION=${CURRENT_VERSION:1} # strip leading 'v' +CURRENT_VERSION=${CURRENT_VERSION%-*} # strip "-dev" suffix if present + +# Exclude peeled tags and release candidates from the version list. +VERSIONS=$(git ls-remote --tags --ref https://github.com/antrea-io/theia.git | \ + grep -v rc | \ + awk '{print $2}' | awk -F/ '{print $3}' | \ + sort --version-sort -r) + +if [ ! -z "$THEIA_FROM_TAG" ]; then + rc=0 + echo "$VERSIONS" | grep -q "$THEIA_FROM_TAG" || rc=$? + if [ $rc -ne 0 ]; then + echoerr "$THEIA_FROM_TAG is not a valid Theia tag" + exit 1 + fi +else # Set THEIA_FROM_TAG using the provided FROM_VERSION_N_MINUS value + arr=( ${CURRENT_VERSION//./ } ) # x.y.z -> (x y z) + minor_version="${arr[0]}.${arr[1]}" + count= + for version in $VERSIONS; do + version_nums=${version:1} # strip leading 'v' + arr=( ${version_nums//./ } ) # x.y.z -> (x y z) + new_minor_version="${arr[0]}.${arr[1]}" + if version_lt $new_minor_version $minor_version; then # change in minor version, increase $count + ((count+=1)) + minor_version=$new_minor_version + if [ "$count" == "$FROM_VERSION_N_MINUS" ]; then # we went back enough, use this version + THEIA_FROM_TAG="${version%-*}" # strip "-alpha" suffix if present + break + fi + fi + done + + if [ -z "$THEIA_FROM_TAG" ]; then + echoerr "Cannot determine tag for provided --from-version-n-minus value" + exit 1 + fi +fi + +echo "Running migrate test for tag $THEIA_FROM_TAG" +ANTREA_FROM_TAG=$(grep "$THEIA_FROM_TAG" < $ROOT_DIR/VERSION_MAP | awk '{print $2}') + +# From v0.3.0, ClickHouse Image is labeled with Theia version +if [[ $THEIA_FROM_TAG == "v0.2.0" || $THEIA_FROM_TAG == "v0.1.0" ]]; then + CLICKHOUSE_FROM_TAG="21.11" +else + CLICKHOUSE_FROM_TAG=$THEIA_FROM_TAG +fi + +# theiaTagArray and chOperatorTagArray are expected to be updated together +# when we migrate ClickHouse Operator version +declare -a theiaTagArray=( +[0]="0.1.0" +[1]="0.7.0" +) + +declare -a chOperatorTagArray=( +[0]="0.18.2" +[1]="0.21.0" +) + +# find the index of tag less than or equal to $THEIA_FROM_TAG, then get the +# corresponding ClickHouse Operator tag +INDEX=0 +for i in "${!theiaTagArray[@]}" +do +if version_lt ${theiaTagArray[$i]} ${THEIA_FROM_TAG:1}; then + INDEX=$i +else + break +fi +done + +CH_OPERATOR_FROM_TAG=${chOperatorTagArray[$INDEX]} + +DOCKER_IMAGES=("registry.k8s.io/e2e-test-images/agnhost:2.29" \ + "projects.registry.vmware.com/antrea/busybox" \ + "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" \ + "projects.registry.vmware.com/antrea/perftool" \ + "projects.registry.vmware.com/antrea/clickhouse-operator:$CH_OPERATOR_FROM_TAG" \ + "projects.registry.vmware.com/antrea/metrics-exporter:$CH_OPERATOR_FROM_TAG" \ + "projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0" \ + "projects.registry.vmware.com/antrea/metrics-exporter:0.21.0" \ + "projects.registry.vmware.com/antrea/theia-zookeeper:3.8.0" \ + "projects.registry.vmware.com/antrea/theia-grafana:8.3.3" \ + "projects.registry.vmware.com/antrea/antrea-ubuntu:$ANTREA_FROM_TAG" \ + "projects.registry.vmware.com/antrea/theia-clickhouse-monitor:$THEIA_FROM_TAG" \ + "projects.registry.vmware.com/antrea/theia-clickhouse-server:$CLICKHOUSE_FROM_TAG" \ + "antrea/antrea-ubuntu:latest") + +for img in "${DOCKER_IMAGES[@]}"; do + echo "Pulling $img" + for i in `seq 3`; do + docker pull $img > /dev/null && break + sleep 1 + done +done + +DOCKER_IMAGES+=("projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest\ + projects.registry.vmware.com/antrea/theia-clickhouse-server:latest") + +echo "Creating Kind cluster" +IMAGES="${DOCKER_IMAGES[@]}" +$THIS_DIR/kind-setup.sh create kind --images "$IMAGES" + +# When running this script as part of a Github Action, we do *not* want to use +# the pre-installed version of kustomize, as it is a snap and cannot access +# /tmp. See: +# * https://github.com/actions/virtual-environments/issues/1514 +# * https://forum.snapcraft.io/t/interfaces-allow-access-tmp-directory/5129 +unset KUSTOMIZE + +# Load latest Antrea yaml files +TMP_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) +curl -o $TMP_DIR/antrea.yml https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/antrea.yml + +sed -i -e "s|image: \"projects.registry.vmware.com/antrea/antrea-ubuntu:latest\"|image: \"antrea/antrea-ubuntu:latest\"|g" $TMP_DIR/antrea.yml +sed -i -e "s|type: RollingUpdate|type: OnDelete|g" $TMP_DIR/antrea.yml +# Load latest Theia yaml files +kubectl apply -f $TMP_DIR/antrea.yml +docker exec -i kind-control-plane dd of=/root/antrea-new.yml < $TMP_DIR/antrea.yml +$ROOT_DIR/hack/generate-manifest.sh --local /data/clickhouse --no-grafana | docker exec -i kind-control-plane dd of=/root/flow-visibility-new.yml +docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle-new.yaml < $ROOT_DIR/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml +rm -rf $TMP_DIR + +# Load previous version yaml files from Antrea repo +TMP_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) +git clone --branch $ANTREA_FROM_TAG --depth 1 https://github.com/antrea-io/antrea.git $TMP_DIR + +pushd $TMP_DIR > /dev/null +export IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu +export IMG_TAG=$ANTREA_FROM_TAG +./hack/generate-manifest.sh --mode release | docker exec -i kind-control-plane dd of=/root/antrea.yml +popd +rm -rf $TMP_DIR + +# Load previous version yaml files from Theia repo +TMP_THEIA_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) +git clone --branch $THEIA_FROM_TAG --depth 1 https://github.com/antrea-io/theia.git $TMP_THEIA_DIR + +pushd $TMP_THEIA_DIR > /dev/null +export IMG_NAME=projects.registry.vmware.com/antrea/theia-clickhouse-monitor +export IMG_TAG=$THEIA_FROM_TAG +# In Theia v0.1.0, we do not support --local option when generating manifest, +# Copy the latest script for release v0.1.0 to generate manifest. +if [[ $THEIA_FROM_TAG == "v0.1.0" ]]; then + cp $ROOT_DIR/hack/generate-manifest.sh hack/generate-manifest.sh +fi +./hack/generate-manifest.sh --mode release --local /data/clickhouse --no-grafana | docker exec -i kind-control-plane dd of=/root/flow-visibility-ch-only.yml +docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yaml < build/charts/theia/crds/clickhouse-operator-install-bundle.yaml + +popd +rm -rf $TMP_THEIA_DIR + +rc=0 +go test -v -timeout=15m -run=TestMigrate antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --migrate.toVersion=$CURRENT_VERSION --migrate.fromVersion=$THEIA_FROM_TAG || rc=$? + +$THIS_DIR/kind-setup.sh destroy kind + +exit $rc diff --git a/ci/kind/test-upgrade-theia.sh b/ci/kind/test-upgrade-theia.sh index 65f7fd2a..09c46fcf 100755 --- a/ci/kind/test-upgrade-theia.sh +++ b/ci/kind/test-upgrade-theia.sh @@ -165,7 +165,7 @@ declare -a chOperatorTagArray=( # find the index of tag less than or equal to $THEIA_FROM_TAG, then get the # corresponding ClickHouse Operator tag INDEX=0 -for i in ${!theiaTagArray[@]} +for i in "${!theiaTagArray[@]}" do if version_lt ${theiaTagArray[$i]} ${THEIA_FROM_TAG:1}; then INDEX=$i @@ -232,6 +232,7 @@ git clone --branch $ANTREA_FROM_TAG --depth 1 https://github.com/antrea-io/antre pushd $TMP_DIR > /dev/null export IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu export IMG_TAG=$ANTREA_FROM_TAG +./hack/generate-manifest.sh --mode release | kubectl apply -f - ./hack/generate-manifest.sh --mode release | docker exec -i kind-control-plane dd of=/root/antrea.yml popd rm -rf $TMP_DIR @@ -248,14 +249,14 @@ export IMG_TAG=$THEIA_FROM_TAG if [[ $THEIA_FROM_TAG == "v0.1.0" ]]; then cp $ROOT_DIR/hack/generate-manifest.sh hack/generate-manifest.sh fi -./hack/generate-manifest.sh --mode release --local /data/clickhouse --no-grafana | docker exec -i kind-control-plane dd of=/root/flow-visibility-ch-only.yml docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yaml < build/charts/theia/crds/clickhouse-operator-install-bundle.yaml +./hack/generate-manifest.sh --mode release --local /data/clickhouse --no-grafana | docker exec -i kind-control-plane dd of=/root/flow-visibility-ch-only.yml popd rm -rf $TMP_THEIA_DIR rc=0 -go test -v -timeout=15m -run=TestUpgrade antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --upgrade.toVersion=$CURRENT_VERSION --upgrade.fromVersion=$THEIA_FROM_TAG || rc=$? +go test -v -timeout=15m -run=TestUpgrade antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --upgrade.toVersion=$CURRENT_VERSION || rc=$? $THIS_DIR/kind-setup.sh destroy kind diff --git a/test/e2e/clickhouse_migrate_test.go b/test/e2e/clickhouse_migrate_test.go new file mode 100644 index 00000000..4631c570 --- /dev/null +++ b/test/e2e/clickhouse_migrate_test.go @@ -0,0 +1,177 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + networkingv1 "k8s.io/api/networking/v1" + "sigs.k8s.io/yaml" + + antreav1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + antreav1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" +) + +var ( + migrateToVersion = flag.String("migrate.toVersion", "", "Version migrated to") + migrateFromVersion = flag.String("migrate.fromVersion", "", "Version migrated from") +) + +const ( + migrateFromFlowVisibilityYML = "flow-visibility-ch-only.yml" + migrateFromChOperatorYML = "clickhouse-operator-install-bundle.yaml" + latestAntreaYML = "antrea-new.yml" + migrateToFlowVisibilityYML = "flow-visibility-new.yml" + migrateToChOperatorYML = "clickhouse-operator-install-bundle-new.yaml" +) + +func skipIfNotMigrateTest(t *testing.T) { + if *migrateToVersion == "" { + t.Skipf("Skipping test as we are not testing for migrate") + } +} + +// TestUpgrade tests that some basic functionalities are not broken when +// upgrading from one version of Theia to another. At the moment it checks +// that: +// - ClickHouse data schema version +// - Consistency of recommendations stored in ClickHouse +// +// To run the test, provide the -upgrade.toVersion flag. +func TestMigrate(t *testing.T) { + skipIfNotMigrateTest(t) + config := FlowVisibilitySetUpConfig{ + withSparkOperator: false, + withGrafana: false, + withClickHouseLocalPv: true, + withFlowAggregator: false, + } + data, _, _, err := setupTestForFlowVisibility(t, config) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer func() { + teardownTest(t, data) + teardownFlowVisibility(t, data, config) + data.deleteClickHouseOperator(migrateToChOperatorYML) + }() + checkClickHouseVersionTable(t, data, *migrateFromVersion) + if needCheckRecommendationsSchema(*migrateFromVersion) { + insertRecommendations(t, data) + } + // upgrade and check + ApplyNewVersion(t, data, latestAntreaYML, migrateToChOperatorYML, migrateToFlowVisibilityYML) + checkClickHouseVersionTable(t, data, *migrateToVersion) + // This check only works when upgrading from v0.3.0 to v0.4.0 for now + // as the recommendations schema only changes between these 2 version. + // More versions can be added when we add other changes to recommendations + // table. + if needCheckRecommendationsSchema(*migrateFromVersion) { + checkRecommendations(t, data, *migrateToVersion) + } + // downgrade and check + ApplyNewVersion(t, data, latestAntreaYML, migrateFromChOperatorYML, migrateFromFlowVisibilityYML) + checkClickHouseVersionTable(t, data, *migrateFromVersion) + if needCheckRecommendationsSchema(*migrateFromVersion) { + checkRecommendations(t, data, *migrateFromVersion) + } +} + +func checkClickHouseVersionTable(t *testing.T, data *TestData, version string) { + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SHOW TABLES\""}) + require.NoErrorf(t, err, "Fail to get tables from ClickHouse: %v", stderr) + if version != "v0.1.0" { + require.Contains(t, queryOutput, "flows") + require.Contains(t, queryOutput, "flows_local") + if version == "v0.2.0" { + require.Contains(t, queryOutput, "migrate_version") + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SELECT version FROM migrate_version\""}) + require.NoErrorf(t, err, "Fail to get version from ClickHouse: %v", stderr) + // strip leading 'v' + assert.Contains(t, queryOutput, version[1:]) + } else { + require.Contains(t, queryOutput, "schema_migrations") + } + } +} + +func needCheckRecommendationsSchema(fromVersion string) bool { + return fromVersion == "v0.3.0" +} + +func insertRecommendations(t *testing.T, data *TestData) { + command := fmt.Sprintf("clickhouse client -q \"INSERT INTO recommendations (id, yamls) VALUES ('%s', '%s'), ('%s', '%s')\"", + id1, strings.Join([]string{getNetworkPolicyYaml("knp"), getNetworkPolicyYaml("anp")}, "\n---\n"), + id2, strings.Join([]string{getNetworkPolicyYaml("acg"), getNetworkPolicyYaml("acnp")}, "\n---\n"), + ) + stdout, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) + require.NoErrorf(t, err, "Fail to get tables from ClickHouse, stdout: %v, stderr: %v", stdout, stderr) +} + +func checkRecommendations(t *testing.T, data *TestData, version string) { + if version == "v0.3.0" { + checkRecommendationsVersion3(t, data, id1, []string{"knp", "anp"}) + checkRecommendationsVersion3(t, data, id2, []string{"acnp", "acg"}) + } else if version == "v0.4.0" { + checkRecommendationsVersion4(t, data, id1, "knp") + checkRecommendationsVersion4(t, data, id1, "anp") + checkRecommendationsVersion4(t, data, id2, "acnp") + checkRecommendationsVersion4(t, data, id2, "acg") + } +} + +func checkRecommendationsVersion3(t *testing.T, data *TestData, id string, kinds []string) { + // Get the recommendationed policy and check if it is equal to the original yaml + command := fmt.Sprintf("clickhouse client -q \"select yamls from recommendations where id='%s'\"", id) + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) + require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) + queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") + for _, kind := range kinds { + assert.Contains(t, queryOutput, getNetworkPolicyYaml(kind)) + } +} + +func checkRecommendationsVersion4(t *testing.T, data *TestData, id string, kind string) { + // Get the recommendationed policy and check if it is equal to the original yaml + command := fmt.Sprintf("clickhouse client -q \"select policy from recommendations where id='%s' and kind='%s'\"", id, kind) + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) + require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) + queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") + assert.Contains(t, queryOutput, getNetworkPolicyYaml(kind)) + // Parse the recommendationed policy to corresponding type to verify there is no error + switch kind { + case "acnp": + var acnp antreav1alpha1.ClusterNetworkPolicy + err = yaml.Unmarshal([]byte(queryOutput), &acnp) + require.NoErrorf(t, err, "failed to parse the policy with kind acnp, yaml: %s", queryOutput) + case "anp": + var anp antreav1alpha1.NetworkPolicy + err = yaml.Unmarshal([]byte(queryOutput), &anp) + require.NoErrorf(t, err, "failed to parse the policy with kind anp, yaml: %s", queryOutput) + case "acg": + var acg antreav1alpha2.ClusterGroup + err = yaml.Unmarshal([]byte(queryOutput), &acg) + require.NoErrorf(t, err, "failed to parse the policy with kind acg, yaml: %s", queryOutput) + case "knp": + var knp networkingv1.NetworkPolicy + err = yaml.Unmarshal([]byte(queryOutput), &knp) + require.NoErrorf(t, err, "failed to parse the policy with kind knp, yaml: %s", queryOutput) + } +} diff --git a/test/e2e/fixture.go b/test/e2e/fixture.go index f56c2ac4..e83c6a25 100644 --- a/test/e2e/fixture.go +++ b/test/e2e/fixture.go @@ -281,7 +281,7 @@ func setupTest(tb testing.TB) (*TestData, error) { return testData, nil } -func setupTestForFlowVisibility(tb testing.TB, config FlowVisibiltiySetUpConfig) (*TestData, bool, bool, error) { +func setupTestForFlowVisibility(tb testing.TB, config FlowVisibilitySetUpConfig) (*TestData, bool, bool, error) { v4Enabled := clusterInfo.podV4NetworkCIDR != "" v6Enabled := clusterInfo.podV6NetworkCIDR != "" testData, err := setupTest(tb) diff --git a/test/e2e/flowvisibility_test.go b/test/e2e/flowvisibility_test.go index 2f57a5ef..740dd78d 100644 --- a/test/e2e/flowvisibility_test.go +++ b/test/e2e/flowvisibility_test.go @@ -140,7 +140,7 @@ type testFlow struct { } func TestFlowVisibility(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: true, withClickHouseLocalPv: false, @@ -1280,7 +1280,7 @@ type ClickHouseFullRow struct { } func failOnError(err error, t *testing.T, data *TestData) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: true, withClickHouseLocalPv: false, diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 559eb3b9..c4f9e1b9 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -165,7 +165,7 @@ var ( clickHousePodName = fmt.Sprintf("%s-0-0-0", clickHousePodNamePrefix) ) -type FlowVisibiltiySetUpConfig struct { +type FlowVisibilitySetUpConfig struct { withSparkOperator bool withGrafana bool withClickHouseLocalPv bool @@ -246,11 +246,11 @@ func (p PodIPs) String() string { func (data *TestData) deployAntreaCommon(yamlFile string, extraOptions string, waitForAgentRollout bool) error { // TODO: use the K8s apiserver when server side apply is available? // See https://kubernetes.io/docs/reference/using-api/api-concepts/#server-side-apply - rc, _, _, err := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl apply %s -f %s", extraOptions, yamlFile)) + rc, stdout, stderr, err := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl apply %s -f %s", extraOptions, yamlFile)) if err != nil || rc != 0 { - return fmt.Errorf("error when deploying Antrea; is %s available on the control-plane Node?", yamlFile) + return fmt.Errorf("error when deploying Antrea; is %s available on the control-plane Node? - rc: %v - stdout: %v - stderr: %v - err: %v", yamlFile, rc, stdout, stderr, err) } - rc, stdout, stderr, err := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s rollout status deploy/%s --timeout=%v", antreaNamespace, antreaDeployment, defaultTimeout)) + rc, stdout, stderr, err = data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s rollout status deploy/%s --timeout=%v", antreaNamespace, antreaDeployment, defaultTimeout)) if err != nil || rc != 0 { return fmt.Errorf("error when waiting for antrea-controller rollout to complete - rc: %v - stdout: %v - stderr: %v - err: %v", rc, stdout, stderr, err) } @@ -1175,7 +1175,7 @@ func (data *TestData) createTestNamespace() error { // deployFlowVisibility deploys ClickHouse Operator and DB. // Other applications will be deployed according to config. -func (data *TestData) deployFlowVisibility(config FlowVisibiltiySetUpConfig) (chSvcIP string, err error) { +func (data *TestData) deployFlowVisibility(config FlowVisibilitySetUpConfig) (chSvcIP string, err error) { if config.withClickHouseLocalPv { // Label one of the worker Node to fulfill the Local PersistentVolume affinity requirement. node, err := data.clientset.CoreV1().Nodes().Get(context.TODO(), workerNodeName(1), metav1.GetOptions{}) @@ -1392,7 +1392,7 @@ func (data *TestData) deleteClickHouseOperator(chOperatorYML string) error { return nil } -func teardownFlowVisibility(tb testing.TB, data *TestData, config FlowVisibiltiySetUpConfig) { +func teardownFlowVisibility(tb testing.TB, data *TestData, config FlowVisibilitySetUpConfig) { if config.withFlowAggregator { if err := data.DeleteNamespace(flowAggregatorNamespace, defaultTimeout); err != nil { tb.Logf("Error when tearing down flow aggregator: %v", err) @@ -1406,7 +1406,7 @@ func teardownFlowVisibility(tb testing.TB, data *TestData, config FlowVisibiltiy } } -func (data *TestData) deleteFlowVisibility(config FlowVisibiltiySetUpConfig) error { +func (data *TestData) deleteFlowVisibility(config FlowVisibilitySetUpConfig) error { var flowVisibilityManifest string if !config.withGrafana && !config.withSparkOperator { flowVisibilityManifest = flowVisibilityChOnlyYML @@ -1529,7 +1529,7 @@ func (data *TestData) Cleanup(namespaces []string) { } } -func flowVisibilityCleanup(tb testing.TB, data *TestData, config FlowVisibiltiySetUpConfig) { +func flowVisibilityCleanup(tb testing.TB, data *TestData, config FlowVisibilitySetUpConfig) { teardownTest(tb, data) teardownFlowVisibility(tb, data, config) } diff --git a/test/e2e/policyrecommendation_test.go b/test/e2e/policyrecommendation_test.go index 7f50bf52..5f0933a0 100644 --- a/test/e2e/policyrecommendation_test.go +++ b/test/e2e/policyrecommendation_test.go @@ -42,7 +42,7 @@ const ( ) func TestPolicyRecommendation(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: true, withGrafana: false, withClickHouseLocalPv: false, diff --git a/test/e2e/theia_clickhouse_test.go b/test/e2e/theia_clickhouse_test.go index fcc69cbe..b1102a7c 100644 --- a/test/e2e/theia_clickhouse_test.go +++ b/test/e2e/theia_clickhouse_test.go @@ -53,7 +53,7 @@ var tableColumnNumberMap = map[string]string{ } func TestTheiaClickHouseStatusCommand(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: false, withClickHouseLocalPv: false, diff --git a/test/e2e/throughputanomalydetection_test.go b/test/e2e/throughputanomalydetection_test.go index dbbd766e..537364c0 100644 --- a/test/e2e/throughputanomalydetection_test.go +++ b/test/e2e/throughputanomalydetection_test.go @@ -39,7 +39,7 @@ const ( ) func TestAnomalyDetection(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: true, withGrafana: false, withClickHouseLocalPv: false, diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 6295ad6d..b5c82080 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -16,33 +16,17 @@ package e2e import ( "flag" - "fmt" - "strings" "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - networkingv1 "k8s.io/api/networking/v1" - "sigs.k8s.io/yaml" - - antreav1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" - antreav1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" ) var ( - upgradeToVersion = flag.String("upgrade.toVersion", "", "Version updated to") - upgradeFromVersion = flag.String("upgrade.fromVersion", "", "Version updated from") + upgradeToVersion = flag.String("upgrade.toVersion", "", "Version updated to") ) const ( - upgradeFromAntreaYML = "antrea.yml" - upgradeFromFlowVisibilityYML = "flow-visibility-ch-only.yml" - upgradeFromChOperatorYML = "clickhouse-operator-install-bundle.yaml" - upgradeToAntreaYML = "antrea-new.yml" - upgradeToFlowVisibilityYML = "flow-visibility-new.yml" - upgradeToChOperatorYML = "clickhouse-operator-install-bundle-new.yaml" - id1 = "86b1f47b-7864-4351-bb73-1ffb8bacb2e7" - id2 = "13465fa5-99d7-430b-91df-272ce6b9704c" + upgradeToAntreaYML = "antrea-new.yml" + upgradeToFlowVisibilityYML = "flow-visibility-new.yml" + upgradeToChOperatorYML = "clickhouse-operator-install-bundle-new.yaml" ) func skipIfNotUpgradeTest(t *testing.T) { @@ -60,7 +44,7 @@ func skipIfNotUpgradeTest(t *testing.T) { // To run the test, provide the -upgrade.toVersion flag. func TestUpgrade(t *testing.T) { skipIfNotUpgradeTest(t) - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: false, withClickHouseLocalPv: true, @@ -75,127 +59,6 @@ func TestUpgrade(t *testing.T) { teardownFlowVisibility(t, data, config) data.deleteClickHouseOperator(upgradeToChOperatorYML) }() - checkClickHouseVersionTable(t, data, *upgradeFromVersion) - if needCheckRecommendationsSchema(*upgradeFromVersion) { - insertRecommendations(t, data) - } // upgrade and check - applyNewVersion(t, data, upgradeToAntreaYML, upgradeToChOperatorYML, upgradeToFlowVisibilityYML) - checkClickHouseVersionTable(t, data, *upgradeToVersion) - // This check only works when upgrading from v0.3.0 to v0.4.0 for now - // as the recommendations schema only changes between these 2 version. - // More versions can be added when we add other changes to recommendations - // table. - if needCheckRecommendationsSchema(*upgradeFromVersion) { - checkRecommendations(t, data, *upgradeToVersion) - } - // downgrade and check - applyNewVersion(t, data, upgradeFromAntreaYML, upgradeFromChOperatorYML, upgradeFromFlowVisibilityYML) - checkClickHouseVersionTable(t, data, *upgradeFromVersion) - if needCheckRecommendationsSchema(*upgradeFromVersion) { - checkRecommendations(t, data, *upgradeFromVersion) - } -} - -func applyNewVersion(t *testing.T, data *TestData, antreaYML, chOperatorYML, flowVisibilityYML string) { - t.Logf("Changing Antrea YAML to %s", antreaYML) - // Do not wait for agent rollout as its updateStrategy is set to OnDelete for upgrade test. - if err := data.deployAntreaCommon(antreaYML, "", false); err != nil { - t.Fatalf("Error upgrading Antrea: %v", err) - } - t.Logf("Restarting all Antrea DaemonSet Pods") - if err := data.restartAntreaAgentPods(defaultTimeout); err != nil { - t.Fatalf("Error when restarting Antrea: %v", err) - } - - t.Logf("Changing ClickHouse Operator YAML to %s,\nFlow Visibility YAML to %s", chOperatorYML, flowVisibilityYML) - if err := data.deployFlowVisibilityCommon(chOperatorYML, flowVisibilityYML); err != nil { - t.Fatalf("Error upgrading Flow Visibility: %v", err) - } - t.Logf("Waiting for the ClickHouse Pod restarting") - if err := data.waitForClickHousePod(); err != nil { - t.Fatalf("Error when waiting for the ClickHouse Pod restarting: %v", err) - } -} - -func checkClickHouseVersionTable(t *testing.T, data *TestData, version string) { - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SHOW TABLES\""}) - require.NoErrorf(t, err, "Fail to get tables from ClickHouse: %v", stderr) - if version != "v0.1.0" { - require.Contains(t, queryOutput, "flows") - require.Contains(t, queryOutput, "flows_local") - if version == "v0.2.0" { - require.Contains(t, queryOutput, "migrate_version") - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SELECT version FROM migrate_version\""}) - require.NoErrorf(t, err, "Fail to get version from ClickHouse: %v", stderr) - // strip leading 'v' - assert.Contains(t, queryOutput, version[1:]) - } else { - require.Contains(t, queryOutput, "schema_migrations") - } - } -} - -func needCheckRecommendationsSchema(fromVersion string) bool { - return fromVersion == "v0.3.0" -} - -func insertRecommendations(t *testing.T, data *TestData) { - command := fmt.Sprintf("clickhouse client -q \"INSERT INTO recommendations (id, yamls) VALUES ('%s', '%s'), ('%s', '%s')\"", - id1, strings.Join([]string{getNetowrkPolicyYaml("knp"), getNetowrkPolicyYaml("anp")}, "\n---\n"), - id2, strings.Join([]string{getNetowrkPolicyYaml("acg"), getNetowrkPolicyYaml("acnp")}, "\n---\n"), - ) - stdout, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) - require.NoErrorf(t, err, "Fail to get tables from ClickHouse, stdout: %v, stderr: %v", stdout, stderr) -} - -func checkRecommendations(t *testing.T, data *TestData, version string) { - if version == "v0.3.0" { - checkRecommendationsVersion3(t, data, id1, []string{"knp", "anp"}) - checkRecommendationsVersion3(t, data, id2, []string{"acnp", "acg"}) - } else if version == "v0.4.0" { - checkRecommendationsVersion4(t, data, id1, "knp") - checkRecommendationsVersion4(t, data, id1, "anp") - checkRecommendationsVersion4(t, data, id2, "acnp") - checkRecommendationsVersion4(t, data, id2, "acg") - } -} - -func checkRecommendationsVersion3(t *testing.T, data *TestData, id string, kinds []string) { - // Get the recommendationed policy and check if it is equal to the original yaml - command := fmt.Sprintf("clickhouse client -q \"select yamls from recommendations where id='%s'\"", id) - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) - require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) - queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") - for _, kind := range kinds { - assert.Contains(t, queryOutput, getNetowrkPolicyYaml(kind)) - } -} - -func checkRecommendationsVersion4(t *testing.T, data *TestData, id string, kind string) { - // Get the recommendationed policy and check if it is equal to the original yaml - command := fmt.Sprintf("clickhouse client -q \"select policy from recommendations where id='%s' and kind='%s'\"", id, kind) - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) - require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) - queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") - assert.Contains(t, queryOutput, getNetowrkPolicyYaml(kind)) - // Parse the recommendationed policy to corresponding type to verify there is no error - switch kind { - case "acnp": - var acnp antreav1alpha1.ClusterNetworkPolicy - err = yaml.Unmarshal([]byte(queryOutput), &acnp) - require.NoErrorf(t, err, "failed to parse the policy with kind acnp, yaml: %s", queryOutput) - case "anp": - var anp antreav1alpha1.NetworkPolicy - err = yaml.Unmarshal([]byte(queryOutput), &anp) - require.NoErrorf(t, err, "failed to parse the policy with kind anp, yaml: %s", queryOutput) - case "acg": - var acg antreav1alpha2.ClusterGroup - err = yaml.Unmarshal([]byte(queryOutput), &acg) - require.NoErrorf(t, err, "failed to parse the policy with kind acg, yaml: %s", queryOutput) - case "knp": - var knp networkingv1.NetworkPolicy - err = yaml.Unmarshal([]byte(queryOutput), &knp) - require.NoErrorf(t, err, "failed to parse the policy with kind knp, yaml: %s", queryOutput) - } + ApplyNewVersion(t, data, upgradeToAntreaYML, upgradeToChOperatorYML, upgradeToFlowVisibilityYML) } diff --git a/test/e2e/util.go b/test/e2e/util.go index 48fb3415..e0cc0c2c 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -49,6 +49,8 @@ const ( nodeToNodeDashboardUid = "1F56RJh7z" networkPolicyDashboardUid = "KJNMOwQnk" networkTopologyDashboardUid = "yRVDEad4k" + id1 = "abcd1234-abcd-1234-ab12-abcd12345678" + id2 = "1234abcd-1234-abcd-12ab-12345678abcd" ) var ( @@ -337,7 +339,7 @@ var grafanaTestCases = []struct { }, } -func getNetowrkPolicyYaml(kind string) string { +func getNetworkPolicyYaml(kind string) string { switch kind { case "acnp": return ` @@ -545,3 +547,24 @@ func VerifyJobCleaned(t *testing.T, data *TestData, jobName string, tablename st } return nil } + +func ApplyNewVersion(t *testing.T, data *TestData, antreaYML, chOperatorYML, flowVisibilityYML string) { + t.Logf("Changing Antrea YAML to %s", antreaYML) + // Do not wait for agent rollout as its updateStrategy is set to OnDelete for upgrade test. + if err := data.deployAntreaCommon(antreaYML, "", false); err != nil { + t.Fatalf("Error upgrading Antrea: %v", err) + } + t.Logf("Restarting all Antrea DaemonSet Pods") + if err := data.restartAntreaAgentPods(defaultTimeout); err != nil { + t.Fatalf("Error when restarting Antrea: %v", err) + } + + t.Logf("Changing ClickHouse Operator YAML to %s,\nFlow Visibility YAML to %s", chOperatorYML, flowVisibilityYML) + if err := data.deployFlowVisibilityCommon(chOperatorYML, flowVisibilityYML); err != nil { + t.Fatalf("Error upgrading Flow Visibility: %v", err) + } + t.Logf("Waiting for the ClickHouse Pod restarting") + if err := data.waitForClickHousePod(); err != nil { + t.Fatalf("Error when waiting for the ClickHouse Pod restarting: %v", err) + } +}