diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index d7c2b66e..bf0932ed 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -222,6 +222,63 @@ jobs: name: upgrade-from-theia-version-n-1.tar.gz path: log.tar.gz retention-days: 30 + + test-ClickHouse-migrate-from-N-1: + name: Migrate ClickHouse from Theia version N-1 + needs: + - build-clickhouse-monitor-image + - build-clickhouse-server-image + runs-on: [ubuntu-latest] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "/usr/local/lib/android" + df -h + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + - name: Download ClickHouse monitor images from previous jobs + uses: actions/download-artifact@v3 + with: + name: clickhouse-monitor + - name: Load Theia image + run: | + docker load -i clickhouse-monitor.tar + docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest + - name: Download ClickHouse server images from previous jobs + uses: actions/download-artifact@v3 + with: + name: clickhouse-server + - name: Load Theia image + run: | + docker load -i clickhouse-server.tar + docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest + - name: Install Kind + run: | + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run migrate test + run: | + mkdir log + ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-migrate-clickhouse.sh --from-version-n-minus 1 + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: migrate-clickhouse-from-theia-version-n-1.tar.gz + path: log.tar.gz + retention-days: 30 # Runs after all other jobs in the workflow succeed and deletes Theia Docker images uploaded as temporary # artifacts. It uses a third-party, MIT-licensed action (geekyeggo/delete-artifact). While Github @@ -236,6 +293,7 @@ jobs: - build-theia-manager-image - test-e2e-encap - test-upgrade-from-N-1 + - test-ClickHouse-migrate-from-N-1 runs-on: [ubuntu-latest] steps: - name: Delete spark-jobs diff --git a/ci/kind/test-migrate-clickhouse.sh b/ci/kind/test-migrate-clickhouse.sh new file mode 100755 index 00000000..6981fb1b --- /dev/null +++ b/ci/kind/test-migrate-clickhouse.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash + +# Copyright 2022 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function echoerr { + >&2 echo "$@" +} + +THEIA_FROM_TAG= +FROM_VERSION_N_MINUS= + +_usage="Usage: $0 [--from-tag ] [--from-version-n-minus ] +Perform some basic tests to make sure that Theia can be migrated from the provided version to the +current checked-out version. One of [--from-tag ] or [--from-version-n-minus ] must be +provided. + --from-tag Migrate from this version of Theia (pulled from upstream + Antrea) to the current version. + --from-version-n-minus Get all the released versions of Theia and run the migrate + test from the latest bug fix release for *minor* version + N-{COUNT}. N-1 designates the latest minor release. If this + script is run from a release branch, it will only consider + releases which predate that release branch. + --help, -h Print this message and exit +" + +function print_usage { + echoerr "$_usage" +} + +function print_help { + echoerr "Try '$0 --help' for more information." +} + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR=$THIS_DIR/../.. + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --from-tag) + THEIA_FROM_TAG="$2" + shift 2 + ;; + --from-version-n-minus) + FROM_VERSION_N_MINUS="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) # unknown option + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +if [ -z "$THEIA_FROM_TAG" ] && [ -z "$FROM_VERSION_N_MINUS" ]; then + echoerr "One of --from-tag or --from-version-n-minus must be provided" + print_help + exit 1 +fi + +case $FROM_VERSION_N_MINUS in + ''|*[!0-9]*) + echoerr "--from-version-n-minus must be a number greater than 0" + print_help + exit 1 + ;; + *) + ;; +esac + +if [ ! "$FROM_VERSION_N_MINUS" -gt "0" ]; then + echoerr "--from-version-n-minus must be a number greater than 0" + print_help + exit 1 +fi + +function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; } + +# We want to ignore all minor versions greater than the current version, as an migrate test implies +# that we are migrating from an *older* version. This is useful when running this script from a +# release branch (e.g. when testing patch release candidates). +CURRENT_VERSION=$(head -n1 $ROOT_DIR/VERSION) +CURRENT_VERSION=${CURRENT_VERSION:1} # strip leading 'v' +CURRENT_VERSION=${CURRENT_VERSION%-*} # strip "-dev" suffix if present + +# Exclude peeled tags and release candidates from the version list. +VERSIONS=$(git ls-remote --tags --ref https://github.com/antrea-io/theia.git | \ + grep -v rc | \ + awk '{print $2}' | awk -F/ '{print $3}' | \ + sort --version-sort -r) + +if [ ! -z "$THEIA_FROM_TAG" ]; then + rc=0 + echo "$VERSIONS" | grep -q "$THEIA_FROM_TAG" || rc=$? + if [ $rc -ne 0 ]; then + echoerr "$THEIA_FROM_TAG is not a valid Theia tag" + exit 1 + fi +else # Set THEIA_FROM_TAG using the provided FROM_VERSION_N_MINUS value + arr=( ${CURRENT_VERSION//./ } ) # x.y.z -> (x y z) + minor_version="${arr[0]}.${arr[1]}" + count= + for version in $VERSIONS; do + version_nums=${version:1} # strip leading 'v' + arr=( ${version_nums//./ } ) # x.y.z -> (x y z) + new_minor_version="${arr[0]}.${arr[1]}" + if version_lt $new_minor_version $minor_version; then # change in minor version, increase $count + ((count+=1)) + minor_version=$new_minor_version + if [ "$count" == "$FROM_VERSION_N_MINUS" ]; then # we went back enough, use this version + THEIA_FROM_TAG="${version%-*}" # strip "-alpha" suffix if present + break + fi + fi + done + + if [ -z "$THEIA_FROM_TAG" ]; then + echoerr "Cannot determine tag for provided --from-version-n-minus value" + exit 1 + fi +fi + +echo "Running migrate test for tag $THEIA_FROM_TAG" +ANTREA_FROM_TAG=$(grep "$THEIA_FROM_TAG" < $ROOT_DIR/VERSION_MAP | awk '{print $2}') + +# From v0.3.0, ClickHouse Image is labeled with Theia version +if [[ $THEIA_FROM_TAG == "v0.2.0" || $THEIA_FROM_TAG == "v0.1.0" ]]; then + CLICKHOUSE_FROM_TAG="21.11" +else + CLICKHOUSE_FROM_TAG=$THEIA_FROM_TAG +fi + +# theiaTagArray and chOperatorTagArray are expected to be updated together +# when we migrate ClickHouse Operator version +declare -a theiaTagArray=( +[0]="0.1.0" +[1]="0.7.0" +) + +declare -a chOperatorTagArray=( +[0]="0.18.2" +[1]="0.21.0" +) + +# find the index of tag less than or equal to $THEIA_FROM_TAG, then get the +# corresponding ClickHouse Operator tag +INDEX=0 +for i in "${!theiaTagArray[@]}" +do +if version_lt ${theiaTagArray[$i]} ${THEIA_FROM_TAG:1}; then + INDEX=$i +else + break +fi +done + +CH_OPERATOR_FROM_TAG=${chOperatorTagArray[$INDEX]} + +DOCKER_IMAGES=("registry.k8s.io/e2e-test-images/agnhost:2.29" \ + "projects.registry.vmware.com/antrea/busybox" \ + "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" \ + "projects.registry.vmware.com/antrea/perftool" \ + "projects.registry.vmware.com/antrea/clickhouse-operator:$CH_OPERATOR_FROM_TAG" \ + "projects.registry.vmware.com/antrea/metrics-exporter:$CH_OPERATOR_FROM_TAG" \ + "projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0" \ + "projects.registry.vmware.com/antrea/metrics-exporter:0.21.0" \ + "projects.registry.vmware.com/antrea/theia-zookeeper:3.8.0" \ + "projects.registry.vmware.com/antrea/antrea-ubuntu:$ANTREA_FROM_TAG" \ + "projects.registry.vmware.com/antrea/theia-clickhouse-monitor:$THEIA_FROM_TAG" \ + "projects.registry.vmware.com/antrea/theia-clickhouse-server:$CLICKHOUSE_FROM_TAG" \ + "antrea/antrea-ubuntu:latest") + +for img in "${DOCKER_IMAGES[@]}"; do + echo "Pulling $img" + for i in `seq 3`; do + docker pull $img > /dev/null && break + sleep 1 + done +done + +DOCKER_IMAGES+=("projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest\ + projects.registry.vmware.com/antrea/theia-clickhouse-server:latest") + +echo "Creating Kind cluster" +IMAGES="${DOCKER_IMAGES[@]}" +$THIS_DIR/kind-setup.sh create kind --images "$IMAGES" + +# When running this script as part of a Github Action, we do *not* want to use +# the pre-installed version of kustomize, as it is a snap and cannot access +# /tmp. See: +# * https://github.com/actions/virtual-environments/issues/1514 +# * https://forum.snapcraft.io/t/interfaces-allow-access-tmp-directory/5129 +unset KUSTOMIZE + +# Load latest Antrea yaml files +TMP_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) +curl -o $TMP_DIR/antrea.yml https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/antrea.yml + +sed -i -e "s|image: \"projects.registry.vmware.com/antrea/antrea-ubuntu:latest\"|image: \"antrea/antrea-ubuntu:latest\"|g" $TMP_DIR/antrea.yml +sed -i -e "s|type: RollingUpdate|type: OnDelete|g" $TMP_DIR/antrea.yml +# Load latest Theia yaml files +docker exec -i kind-control-plane dd of=/root/antrea-new.yml < $TMP_DIR/antrea.yml +$ROOT_DIR/hack/generate-manifest.sh --local /data/clickhouse --ch-only | docker exec -i kind-control-plane dd of=/root/flow-visibility-new.yml +docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle-new.yaml < $ROOT_DIR/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml +rm -rf $TMP_DIR + +# Load previous version yaml files from Antrea repo +TMP_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) +git clone --branch $ANTREA_FROM_TAG --depth 1 https://github.com/antrea-io/antrea.git $TMP_DIR + +pushd $TMP_DIR > /dev/null +export IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu +export IMG_TAG=$ANTREA_FROM_TAG +./hack/generate-manifest.sh --mode release | docker exec -i kind-control-plane dd of=/root/antrea.yml +popd +rm -rf $TMP_DIR + +# Load previous version yaml files from Theia repo +TMP_THEIA_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) +git clone --branch $THEIA_FROM_TAG --depth 1 https://github.com/antrea-io/theia.git $TMP_THEIA_DIR + +pushd $TMP_THEIA_DIR > /dev/null +export IMG_NAME=projects.registry.vmware.com/antrea/theia-clickhouse-monitor +export IMG_TAG=$THEIA_FROM_TAG +# In Theia v0.1.0, we do not support --local option when generating manifest, +# Copy the latest script for release v0.1.0 to generate manifest. +if [[ $THEIA_FROM_TAG == "v0.1.0" ]]; then + cp $ROOT_DIR/hack/generate-manifest.sh hack/generate-manifest.sh +fi +# In Theia v0.7.0, we support --ch-only option when generating manifest, +# which is not present in earlier versions +# Copy the latest script for release < v0.7.0 to generate manifest. +export RELEASE_VERSION=${THEIA_FROM_TAG#"v"} +export RELEASE_VERSION_NUMBER=$(echo "$RELEASE_VERSION" | awk -F '.' '{printf "%.1f", $1 + $2/10 + $3/100}') +if (( $(echo "$RELEASE_VERSION_NUMBER < 0.7" | bc -l) )); then + cp $ROOT_DIR/hack/generate-manifest.sh hack/generate-manifest.sh +fi +./hack/generate-manifest.sh --mode release --local /data/clickhouse --ch-only | docker exec -i kind-control-plane dd of=/root/flow-visibility-ch-only.yml +docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yaml < build/charts/theia/crds/clickhouse-operator-install-bundle.yaml + +popd +rm -rf $TMP_THEIA_DIR + +rc=0 +go test -v -timeout=15m -run=TestMigrate antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --migrate.toVersion=$CURRENT_VERSION --migrate.fromVersion=$THEIA_FROM_TAG || rc=$? + +$THIS_DIR/kind-setup.sh destroy kind + +exit $rc diff --git a/ci/kind/test-upgrade-theia.sh b/ci/kind/test-upgrade-theia.sh index 65f7fd2a..46e10a6c 100755 --- a/ci/kind/test-upgrade-theia.sh +++ b/ci/kind/test-upgrade-theia.sh @@ -165,7 +165,7 @@ declare -a chOperatorTagArray=( # find the index of tag less than or equal to $THEIA_FROM_TAG, then get the # corresponding ClickHouse Operator tag INDEX=0 -for i in ${!theiaTagArray[@]} +for i in "${!theiaTagArray[@]}" do if version_lt ${theiaTagArray[$i]} ${THEIA_FROM_TAG:1}; then INDEX=$i @@ -221,7 +221,7 @@ sed -i -e "s|image: \"projects.registry.vmware.com/antrea/antrea-ubuntu:latest\" sed -i -e "s|type: RollingUpdate|type: OnDelete|g" $TMP_DIR/antrea.yml # Load latest Theia yaml files docker exec -i kind-control-plane dd of=/root/antrea-new.yml < $TMP_DIR/antrea.yml -$ROOT_DIR/hack/generate-manifest.sh --local /data/clickhouse --no-grafana | docker exec -i kind-control-plane dd of=/root/flow-visibility-new.yml +$ROOT_DIR/hack/generate-manifest.sh --local /data/clickhouse --spark-operator --theia-manager | docker exec -i kind-control-plane dd of=/root/flow-visibility-new.yml docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle-new.yaml < $ROOT_DIR/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml rm -rf $TMP_DIR @@ -248,14 +248,14 @@ export IMG_TAG=$THEIA_FROM_TAG if [[ $THEIA_FROM_TAG == "v0.1.0" ]]; then cp $ROOT_DIR/hack/generate-manifest.sh hack/generate-manifest.sh fi -./hack/generate-manifest.sh --mode release --local /data/clickhouse --no-grafana | docker exec -i kind-control-plane dd of=/root/flow-visibility-ch-only.yml docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yaml < build/charts/theia/crds/clickhouse-operator-install-bundle.yaml +./hack/generate-manifest.sh --mode release --local /data/clickhouse --spark-operator --theia-manager | docker exec -i kind-control-plane dd of=/root/flow-visibility-with-spark.yml popd rm -rf $TMP_THEIA_DIR rc=0 -go test -v -timeout=15m -run=TestUpgrade antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --upgrade.toVersion=$CURRENT_VERSION --upgrade.fromVersion=$THEIA_FROM_TAG || rc=$? +go test -v -timeout=15m -run=TestUpgrade antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --upgrade.toVersion=$CURRENT_VERSION || rc=$? $THIS_DIR/kind-setup.sh destroy kind diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 797dfa07..d09a124f 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -30,6 +30,7 @@ Kustomize, and print it to stdout. --spark-operator Generate a manifest with Spark Operator enabled. --theia-manager Generate a manifest with Theia Manager enabled. --no-grafana Generate a manifest with Grafana disabled. + --ch-only Generate a manifest with ClickHouse only. --ch-size Deploy the ClickHouse with a specific storage size. Can be a plain integer or as a fixed-point number using one of these quantity suffixes: E, P, T, G, M, K. Or the power-of-two equivalents: @@ -56,6 +57,7 @@ MODE="dev" SPARK_OP=false THEIA_MANAGER=false GRAFANA=true +CH_ONLY=false CH_SIZE="8Gi" CH_THRESHOLD=0.5 LOCALPATH="" @@ -81,6 +83,10 @@ case $key in GRAFANA=false shift 1 ;; + --ch-only) + CH_ONLY=true + shift 1 + ;; --ch-size) CH_SIZE="$2" shift 2 @@ -161,6 +167,9 @@ fi if [ "$GRAFANA" == false ]; then HELM_VALUES+=("grafana.enable=false") fi +if [ "$CH_ONLY" == true ]; then + HELM_VALUES+=("grafana.enable=false" "theiaManager.enable=false" "sparkOperator.enable=false" "clickhouse.monitor.enable=true") +fi if [[ $LOCALPATH != "" ]]; then HELM_VALUES+=("clickhouse.storage.createPersistentVolume.type=Local" "clickhouse.storage.createPersistentVolume.local.path=$LOCALPATH") fi diff --git a/test/e2e/fixture.go b/test/e2e/fixture.go index f56c2ac4..e83c6a25 100644 --- a/test/e2e/fixture.go +++ b/test/e2e/fixture.go @@ -281,7 +281,7 @@ func setupTest(tb testing.TB) (*TestData, error) { return testData, nil } -func setupTestForFlowVisibility(tb testing.TB, config FlowVisibiltiySetUpConfig) (*TestData, bool, bool, error) { +func setupTestForFlowVisibility(tb testing.TB, config FlowVisibilitySetUpConfig) (*TestData, bool, bool, error) { v4Enabled := clusterInfo.podV4NetworkCIDR != "" v6Enabled := clusterInfo.podV6NetworkCIDR != "" testData, err := setupTest(tb) diff --git a/test/e2e/flowvisibility_test.go b/test/e2e/flowvisibility_test.go index 2f57a5ef..740dd78d 100644 --- a/test/e2e/flowvisibility_test.go +++ b/test/e2e/flowvisibility_test.go @@ -140,7 +140,7 @@ type testFlow struct { } func TestFlowVisibility(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: true, withClickHouseLocalPv: false, @@ -1280,7 +1280,7 @@ type ClickHouseFullRow struct { } func failOnError(err error, t *testing.T, data *TestData) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: true, withClickHouseLocalPv: false, diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 559eb3b9..c4f9e1b9 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -165,7 +165,7 @@ var ( clickHousePodName = fmt.Sprintf("%s-0-0-0", clickHousePodNamePrefix) ) -type FlowVisibiltiySetUpConfig struct { +type FlowVisibilitySetUpConfig struct { withSparkOperator bool withGrafana bool withClickHouseLocalPv bool @@ -246,11 +246,11 @@ func (p PodIPs) String() string { func (data *TestData) deployAntreaCommon(yamlFile string, extraOptions string, waitForAgentRollout bool) error { // TODO: use the K8s apiserver when server side apply is available? // See https://kubernetes.io/docs/reference/using-api/api-concepts/#server-side-apply - rc, _, _, err := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl apply %s -f %s", extraOptions, yamlFile)) + rc, stdout, stderr, err := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl apply %s -f %s", extraOptions, yamlFile)) if err != nil || rc != 0 { - return fmt.Errorf("error when deploying Antrea; is %s available on the control-plane Node?", yamlFile) + return fmt.Errorf("error when deploying Antrea; is %s available on the control-plane Node? - rc: %v - stdout: %v - stderr: %v - err: %v", yamlFile, rc, stdout, stderr, err) } - rc, stdout, stderr, err := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s rollout status deploy/%s --timeout=%v", antreaNamespace, antreaDeployment, defaultTimeout)) + rc, stdout, stderr, err = data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s rollout status deploy/%s --timeout=%v", antreaNamespace, antreaDeployment, defaultTimeout)) if err != nil || rc != 0 { return fmt.Errorf("error when waiting for antrea-controller rollout to complete - rc: %v - stdout: %v - stderr: %v - err: %v", rc, stdout, stderr, err) } @@ -1175,7 +1175,7 @@ func (data *TestData) createTestNamespace() error { // deployFlowVisibility deploys ClickHouse Operator and DB. // Other applications will be deployed according to config. -func (data *TestData) deployFlowVisibility(config FlowVisibiltiySetUpConfig) (chSvcIP string, err error) { +func (data *TestData) deployFlowVisibility(config FlowVisibilitySetUpConfig) (chSvcIP string, err error) { if config.withClickHouseLocalPv { // Label one of the worker Node to fulfill the Local PersistentVolume affinity requirement. node, err := data.clientset.CoreV1().Nodes().Get(context.TODO(), workerNodeName(1), metav1.GetOptions{}) @@ -1392,7 +1392,7 @@ func (data *TestData) deleteClickHouseOperator(chOperatorYML string) error { return nil } -func teardownFlowVisibility(tb testing.TB, data *TestData, config FlowVisibiltiySetUpConfig) { +func teardownFlowVisibility(tb testing.TB, data *TestData, config FlowVisibilitySetUpConfig) { if config.withFlowAggregator { if err := data.DeleteNamespace(flowAggregatorNamespace, defaultTimeout); err != nil { tb.Logf("Error when tearing down flow aggregator: %v", err) @@ -1406,7 +1406,7 @@ func teardownFlowVisibility(tb testing.TB, data *TestData, config FlowVisibiltiy } } -func (data *TestData) deleteFlowVisibility(config FlowVisibiltiySetUpConfig) error { +func (data *TestData) deleteFlowVisibility(config FlowVisibilitySetUpConfig) error { var flowVisibilityManifest string if !config.withGrafana && !config.withSparkOperator { flowVisibilityManifest = flowVisibilityChOnlyYML @@ -1529,7 +1529,7 @@ func (data *TestData) Cleanup(namespaces []string) { } } -func flowVisibilityCleanup(tb testing.TB, data *TestData, config FlowVisibiltiySetUpConfig) { +func flowVisibilityCleanup(tb testing.TB, data *TestData, config FlowVisibilitySetUpConfig) { teardownTest(tb, data) teardownFlowVisibility(tb, data, config) } diff --git a/test/e2e/migrate_clickhouse_test.go b/test/e2e/migrate_clickhouse_test.go new file mode 100644 index 00000000..4631c570 --- /dev/null +++ b/test/e2e/migrate_clickhouse_test.go @@ -0,0 +1,177 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + networkingv1 "k8s.io/api/networking/v1" + "sigs.k8s.io/yaml" + + antreav1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + antreav1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" +) + +var ( + migrateToVersion = flag.String("migrate.toVersion", "", "Version migrated to") + migrateFromVersion = flag.String("migrate.fromVersion", "", "Version migrated from") +) + +const ( + migrateFromFlowVisibilityYML = "flow-visibility-ch-only.yml" + migrateFromChOperatorYML = "clickhouse-operator-install-bundle.yaml" + latestAntreaYML = "antrea-new.yml" + migrateToFlowVisibilityYML = "flow-visibility-new.yml" + migrateToChOperatorYML = "clickhouse-operator-install-bundle-new.yaml" +) + +func skipIfNotMigrateTest(t *testing.T) { + if *migrateToVersion == "" { + t.Skipf("Skipping test as we are not testing for migrate") + } +} + +// TestUpgrade tests that some basic functionalities are not broken when +// upgrading from one version of Theia to another. At the moment it checks +// that: +// - ClickHouse data schema version +// - Consistency of recommendations stored in ClickHouse +// +// To run the test, provide the -upgrade.toVersion flag. +func TestMigrate(t *testing.T) { + skipIfNotMigrateTest(t) + config := FlowVisibilitySetUpConfig{ + withSparkOperator: false, + withGrafana: false, + withClickHouseLocalPv: true, + withFlowAggregator: false, + } + data, _, _, err := setupTestForFlowVisibility(t, config) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer func() { + teardownTest(t, data) + teardownFlowVisibility(t, data, config) + data.deleteClickHouseOperator(migrateToChOperatorYML) + }() + checkClickHouseVersionTable(t, data, *migrateFromVersion) + if needCheckRecommendationsSchema(*migrateFromVersion) { + insertRecommendations(t, data) + } + // upgrade and check + ApplyNewVersion(t, data, latestAntreaYML, migrateToChOperatorYML, migrateToFlowVisibilityYML) + checkClickHouseVersionTable(t, data, *migrateToVersion) + // This check only works when upgrading from v0.3.0 to v0.4.0 for now + // as the recommendations schema only changes between these 2 version. + // More versions can be added when we add other changes to recommendations + // table. + if needCheckRecommendationsSchema(*migrateFromVersion) { + checkRecommendations(t, data, *migrateToVersion) + } + // downgrade and check + ApplyNewVersion(t, data, latestAntreaYML, migrateFromChOperatorYML, migrateFromFlowVisibilityYML) + checkClickHouseVersionTable(t, data, *migrateFromVersion) + if needCheckRecommendationsSchema(*migrateFromVersion) { + checkRecommendations(t, data, *migrateFromVersion) + } +} + +func checkClickHouseVersionTable(t *testing.T, data *TestData, version string) { + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SHOW TABLES\""}) + require.NoErrorf(t, err, "Fail to get tables from ClickHouse: %v", stderr) + if version != "v0.1.0" { + require.Contains(t, queryOutput, "flows") + require.Contains(t, queryOutput, "flows_local") + if version == "v0.2.0" { + require.Contains(t, queryOutput, "migrate_version") + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SELECT version FROM migrate_version\""}) + require.NoErrorf(t, err, "Fail to get version from ClickHouse: %v", stderr) + // strip leading 'v' + assert.Contains(t, queryOutput, version[1:]) + } else { + require.Contains(t, queryOutput, "schema_migrations") + } + } +} + +func needCheckRecommendationsSchema(fromVersion string) bool { + return fromVersion == "v0.3.0" +} + +func insertRecommendations(t *testing.T, data *TestData) { + command := fmt.Sprintf("clickhouse client -q \"INSERT INTO recommendations (id, yamls) VALUES ('%s', '%s'), ('%s', '%s')\"", + id1, strings.Join([]string{getNetworkPolicyYaml("knp"), getNetworkPolicyYaml("anp")}, "\n---\n"), + id2, strings.Join([]string{getNetworkPolicyYaml("acg"), getNetworkPolicyYaml("acnp")}, "\n---\n"), + ) + stdout, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) + require.NoErrorf(t, err, "Fail to get tables from ClickHouse, stdout: %v, stderr: %v", stdout, stderr) +} + +func checkRecommendations(t *testing.T, data *TestData, version string) { + if version == "v0.3.0" { + checkRecommendationsVersion3(t, data, id1, []string{"knp", "anp"}) + checkRecommendationsVersion3(t, data, id2, []string{"acnp", "acg"}) + } else if version == "v0.4.0" { + checkRecommendationsVersion4(t, data, id1, "knp") + checkRecommendationsVersion4(t, data, id1, "anp") + checkRecommendationsVersion4(t, data, id2, "acnp") + checkRecommendationsVersion4(t, data, id2, "acg") + } +} + +func checkRecommendationsVersion3(t *testing.T, data *TestData, id string, kinds []string) { + // Get the recommendationed policy and check if it is equal to the original yaml + command := fmt.Sprintf("clickhouse client -q \"select yamls from recommendations where id='%s'\"", id) + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) + require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) + queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") + for _, kind := range kinds { + assert.Contains(t, queryOutput, getNetworkPolicyYaml(kind)) + } +} + +func checkRecommendationsVersion4(t *testing.T, data *TestData, id string, kind string) { + // Get the recommendationed policy and check if it is equal to the original yaml + command := fmt.Sprintf("clickhouse client -q \"select policy from recommendations where id='%s' and kind='%s'\"", id, kind) + queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) + require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) + queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") + assert.Contains(t, queryOutput, getNetworkPolicyYaml(kind)) + // Parse the recommendationed policy to corresponding type to verify there is no error + switch kind { + case "acnp": + var acnp antreav1alpha1.ClusterNetworkPolicy + err = yaml.Unmarshal([]byte(queryOutput), &acnp) + require.NoErrorf(t, err, "failed to parse the policy with kind acnp, yaml: %s", queryOutput) + case "anp": + var anp antreav1alpha1.NetworkPolicy + err = yaml.Unmarshal([]byte(queryOutput), &anp) + require.NoErrorf(t, err, "failed to parse the policy with kind anp, yaml: %s", queryOutput) + case "acg": + var acg antreav1alpha2.ClusterGroup + err = yaml.Unmarshal([]byte(queryOutput), &acg) + require.NoErrorf(t, err, "failed to parse the policy with kind acg, yaml: %s", queryOutput) + case "knp": + var knp networkingv1.NetworkPolicy + err = yaml.Unmarshal([]byte(queryOutput), &knp) + require.NoErrorf(t, err, "failed to parse the policy with kind knp, yaml: %s", queryOutput) + } +} diff --git a/test/e2e/policyrecommendation_test.go b/test/e2e/policyrecommendation_test.go index 7f50bf52..5f0933a0 100644 --- a/test/e2e/policyrecommendation_test.go +++ b/test/e2e/policyrecommendation_test.go @@ -42,7 +42,7 @@ const ( ) func TestPolicyRecommendation(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: true, withGrafana: false, withClickHouseLocalPv: false, diff --git a/test/e2e/theia_clickhouse_test.go b/test/e2e/theia_clickhouse_test.go index fcc69cbe..b1102a7c 100644 --- a/test/e2e/theia_clickhouse_test.go +++ b/test/e2e/theia_clickhouse_test.go @@ -53,7 +53,7 @@ var tableColumnNumberMap = map[string]string{ } func TestTheiaClickHouseStatusCommand(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: false, withGrafana: false, withClickHouseLocalPv: false, diff --git a/test/e2e/throughputanomalydetection_test.go b/test/e2e/throughputanomalydetection_test.go index dbbd766e..537364c0 100644 --- a/test/e2e/throughputanomalydetection_test.go +++ b/test/e2e/throughputanomalydetection_test.go @@ -39,7 +39,7 @@ const ( ) func TestAnomalyDetection(t *testing.T) { - config := FlowVisibiltiySetUpConfig{ + config := FlowVisibilitySetUpConfig{ withSparkOperator: true, withGrafana: false, withClickHouseLocalPv: false, diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 6295ad6d..6e4da196 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -16,33 +16,17 @@ package e2e import ( "flag" - "fmt" - "strings" "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - networkingv1 "k8s.io/api/networking/v1" - "sigs.k8s.io/yaml" - - antreav1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" - antreav1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" ) var ( - upgradeToVersion = flag.String("upgrade.toVersion", "", "Version updated to") - upgradeFromVersion = flag.String("upgrade.fromVersion", "", "Version updated from") + upgradeToVersion = flag.String("upgrade.toVersion", "", "Version updated to") ) const ( - upgradeFromAntreaYML = "antrea.yml" - upgradeFromFlowVisibilityYML = "flow-visibility-ch-only.yml" - upgradeFromChOperatorYML = "clickhouse-operator-install-bundle.yaml" - upgradeToAntreaYML = "antrea-new.yml" - upgradeToFlowVisibilityYML = "flow-visibility-new.yml" - upgradeToChOperatorYML = "clickhouse-operator-install-bundle-new.yaml" - id1 = "86b1f47b-7864-4351-bb73-1ffb8bacb2e7" - id2 = "13465fa5-99d7-430b-91df-272ce6b9704c" + upgradeToAntreaYML = "antrea-new.yml" + upgradeToFlowVisibilityYML = "flow-visibility-new.yml" + upgradeToChOperatorYML = "clickhouse-operator-install-bundle-new.yaml" ) func skipIfNotUpgradeTest(t *testing.T) { @@ -60,9 +44,9 @@ func skipIfNotUpgradeTest(t *testing.T) { // To run the test, provide the -upgrade.toVersion flag. func TestUpgrade(t *testing.T) { skipIfNotUpgradeTest(t) - config := FlowVisibiltiySetUpConfig{ - withSparkOperator: false, - withGrafana: false, + config := FlowVisibilitySetUpConfig{ + withSparkOperator: true, + withGrafana: true, withClickHouseLocalPv: true, withFlowAggregator: false, } @@ -75,127 +59,6 @@ func TestUpgrade(t *testing.T) { teardownFlowVisibility(t, data, config) data.deleteClickHouseOperator(upgradeToChOperatorYML) }() - checkClickHouseVersionTable(t, data, *upgradeFromVersion) - if needCheckRecommendationsSchema(*upgradeFromVersion) { - insertRecommendations(t, data) - } // upgrade and check - applyNewVersion(t, data, upgradeToAntreaYML, upgradeToChOperatorYML, upgradeToFlowVisibilityYML) - checkClickHouseVersionTable(t, data, *upgradeToVersion) - // This check only works when upgrading from v0.3.0 to v0.4.0 for now - // as the recommendations schema only changes between these 2 version. - // More versions can be added when we add other changes to recommendations - // table. - if needCheckRecommendationsSchema(*upgradeFromVersion) { - checkRecommendations(t, data, *upgradeToVersion) - } - // downgrade and check - applyNewVersion(t, data, upgradeFromAntreaYML, upgradeFromChOperatorYML, upgradeFromFlowVisibilityYML) - checkClickHouseVersionTable(t, data, *upgradeFromVersion) - if needCheckRecommendationsSchema(*upgradeFromVersion) { - checkRecommendations(t, data, *upgradeFromVersion) - } -} - -func applyNewVersion(t *testing.T, data *TestData, antreaYML, chOperatorYML, flowVisibilityYML string) { - t.Logf("Changing Antrea YAML to %s", antreaYML) - // Do not wait for agent rollout as its updateStrategy is set to OnDelete for upgrade test. - if err := data.deployAntreaCommon(antreaYML, "", false); err != nil { - t.Fatalf("Error upgrading Antrea: %v", err) - } - t.Logf("Restarting all Antrea DaemonSet Pods") - if err := data.restartAntreaAgentPods(defaultTimeout); err != nil { - t.Fatalf("Error when restarting Antrea: %v", err) - } - - t.Logf("Changing ClickHouse Operator YAML to %s,\nFlow Visibility YAML to %s", chOperatorYML, flowVisibilityYML) - if err := data.deployFlowVisibilityCommon(chOperatorYML, flowVisibilityYML); err != nil { - t.Fatalf("Error upgrading Flow Visibility: %v", err) - } - t.Logf("Waiting for the ClickHouse Pod restarting") - if err := data.waitForClickHousePod(); err != nil { - t.Fatalf("Error when waiting for the ClickHouse Pod restarting: %v", err) - } -} - -func checkClickHouseVersionTable(t *testing.T, data *TestData, version string) { - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SHOW TABLES\""}) - require.NoErrorf(t, err, "Fail to get tables from ClickHouse: %v", stderr) - if version != "v0.1.0" { - require.Contains(t, queryOutput, "flows") - require.Contains(t, queryOutput, "flows_local") - if version == "v0.2.0" { - require.Contains(t, queryOutput, "migrate_version") - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", "clickhouse client -q \"SELECT version FROM migrate_version\""}) - require.NoErrorf(t, err, "Fail to get version from ClickHouse: %v", stderr) - // strip leading 'v' - assert.Contains(t, queryOutput, version[1:]) - } else { - require.Contains(t, queryOutput, "schema_migrations") - } - } -} - -func needCheckRecommendationsSchema(fromVersion string) bool { - return fromVersion == "v0.3.0" -} - -func insertRecommendations(t *testing.T, data *TestData) { - command := fmt.Sprintf("clickhouse client -q \"INSERT INTO recommendations (id, yamls) VALUES ('%s', '%s'), ('%s', '%s')\"", - id1, strings.Join([]string{getNetowrkPolicyYaml("knp"), getNetowrkPolicyYaml("anp")}, "\n---\n"), - id2, strings.Join([]string{getNetowrkPolicyYaml("acg"), getNetowrkPolicyYaml("acnp")}, "\n---\n"), - ) - stdout, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) - require.NoErrorf(t, err, "Fail to get tables from ClickHouse, stdout: %v, stderr: %v", stdout, stderr) -} - -func checkRecommendations(t *testing.T, data *TestData, version string) { - if version == "v0.3.0" { - checkRecommendationsVersion3(t, data, id1, []string{"knp", "anp"}) - checkRecommendationsVersion3(t, data, id2, []string{"acnp", "acg"}) - } else if version == "v0.4.0" { - checkRecommendationsVersion4(t, data, id1, "knp") - checkRecommendationsVersion4(t, data, id1, "anp") - checkRecommendationsVersion4(t, data, id2, "acnp") - checkRecommendationsVersion4(t, data, id2, "acg") - } -} - -func checkRecommendationsVersion3(t *testing.T, data *TestData, id string, kinds []string) { - // Get the recommendationed policy and check if it is equal to the original yaml - command := fmt.Sprintf("clickhouse client -q \"select yamls from recommendations where id='%s'\"", id) - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) - require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) - queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") - for _, kind := range kinds { - assert.Contains(t, queryOutput, getNetowrkPolicyYaml(kind)) - } -} - -func checkRecommendationsVersion4(t *testing.T, data *TestData, id string, kind string) { - // Get the recommendationed policy and check if it is equal to the original yaml - command := fmt.Sprintf("clickhouse client -q \"select policy from recommendations where id='%s' and kind='%s'\"", id, kind) - queryOutput, stderr, err := data.RunCommandFromPod(flowVisibilityNamespace, clickHousePodName, "clickhouse", []string{"bash", "-c", command}) - require.NoErrorf(t, err, "Fail to get recommendations from ClickHouse, stderr: %v", stderr) - queryOutput = strings.ReplaceAll(queryOutput, "\\n", "\n") - assert.Contains(t, queryOutput, getNetowrkPolicyYaml(kind)) - // Parse the recommendationed policy to corresponding type to verify there is no error - switch kind { - case "acnp": - var acnp antreav1alpha1.ClusterNetworkPolicy - err = yaml.Unmarshal([]byte(queryOutput), &acnp) - require.NoErrorf(t, err, "failed to parse the policy with kind acnp, yaml: %s", queryOutput) - case "anp": - var anp antreav1alpha1.NetworkPolicy - err = yaml.Unmarshal([]byte(queryOutput), &anp) - require.NoErrorf(t, err, "failed to parse the policy with kind anp, yaml: %s", queryOutput) - case "acg": - var acg antreav1alpha2.ClusterGroup - err = yaml.Unmarshal([]byte(queryOutput), &acg) - require.NoErrorf(t, err, "failed to parse the policy with kind acg, yaml: %s", queryOutput) - case "knp": - var knp networkingv1.NetworkPolicy - err = yaml.Unmarshal([]byte(queryOutput), &knp) - require.NoErrorf(t, err, "failed to parse the policy with kind knp, yaml: %s", queryOutput) - } + ApplyNewVersion(t, data, upgradeToAntreaYML, upgradeToChOperatorYML, upgradeToFlowVisibilityYML) } diff --git a/test/e2e/util.go b/test/e2e/util.go index 48fb3415..e0cc0c2c 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -49,6 +49,8 @@ const ( nodeToNodeDashboardUid = "1F56RJh7z" networkPolicyDashboardUid = "KJNMOwQnk" networkTopologyDashboardUid = "yRVDEad4k" + id1 = "abcd1234-abcd-1234-ab12-abcd12345678" + id2 = "1234abcd-1234-abcd-12ab-12345678abcd" ) var ( @@ -337,7 +339,7 @@ var grafanaTestCases = []struct { }, } -func getNetowrkPolicyYaml(kind string) string { +func getNetworkPolicyYaml(kind string) string { switch kind { case "acnp": return ` @@ -545,3 +547,24 @@ func VerifyJobCleaned(t *testing.T, data *TestData, jobName string, tablename st } return nil } + +func ApplyNewVersion(t *testing.T, data *TestData, antreaYML, chOperatorYML, flowVisibilityYML string) { + t.Logf("Changing Antrea YAML to %s", antreaYML) + // Do not wait for agent rollout as its updateStrategy is set to OnDelete for upgrade test. + if err := data.deployAntreaCommon(antreaYML, "", false); err != nil { + t.Fatalf("Error upgrading Antrea: %v", err) + } + t.Logf("Restarting all Antrea DaemonSet Pods") + if err := data.restartAntreaAgentPods(defaultTimeout); err != nil { + t.Fatalf("Error when restarting Antrea: %v", err) + } + + t.Logf("Changing ClickHouse Operator YAML to %s,\nFlow Visibility YAML to %s", chOperatorYML, flowVisibilityYML) + if err := data.deployFlowVisibilityCommon(chOperatorYML, flowVisibilityYML); err != nil { + t.Fatalf("Error upgrading Flow Visibility: %v", err) + } + t.Logf("Waiting for the ClickHouse Pod restarting") + if err := data.waitForClickHousePod(); err != nil { + t.Fatalf("Error when waiting for the ClickHouse Pod restarting: %v", err) + } +}