From 27c6a96518a37f0e9ed108abfa583acb5d9ddec4 Mon Sep 17 00:00:00 2001 From: Ryan Cook Date: Mon, 10 Jun 2024 20:29:20 -0400 Subject: [PATCH] clean out jobs Signed-off-by: Ryan Cook --- .../workflows/network-latency-chaos-test.yaml | 197 ------------------ .../network-partition-chaos-test.yaml | 95 --------- .github/workflows/pod-failure-chaos-test.yaml | 192 ----------------- .../pod-kill-chaos-test-kafka-version.yaml | 188 ----------------- .github/workflows/pod-kill-chaos-test.yaml | 188 ----------------- .github/workflows/publish-builder.yaml | 90 -------- .github/workflows/publish-gpu-builder.yaml | 84 -------- .github/workflows/publish-krte-images.yaml | 85 -------- .github/workflows/publish-test-images.yaml | 97 --------- .github/workflows/remote-rhel-build.yaml | 1 + .github/workflows/rerun-failure-checks.yaml | 47 ----- .../workflows/simd-compatibility-test.yaml | 87 -------- .github/workflows/update-knowhere-commit.yaml | 75 ------- .github/workflows/weekly-release.yml | 71 ------- 14 files changed, 1 insertion(+), 1496 deletions(-) delete mode 100644 .github/workflows/network-latency-chaos-test.yaml delete mode 100644 .github/workflows/network-partition-chaos-test.yaml delete mode 100644 .github/workflows/pod-failure-chaos-test.yaml delete mode 100644 .github/workflows/pod-kill-chaos-test-kafka-version.yaml delete mode 100644 .github/workflows/pod-kill-chaos-test.yaml delete mode 100644 .github/workflows/publish-builder.yaml delete mode 100644 .github/workflows/publish-gpu-builder.yaml delete mode 100644 .github/workflows/publish-krte-images.yaml delete mode 100644 .github/workflows/publish-test-images.yaml delete mode 100644 .github/workflows/rerun-failure-checks.yaml delete mode 100644 .github/workflows/simd-compatibility-test.yaml delete mode 100644 .github/workflows/update-knowhere-commit.yaml delete mode 100644 .github/workflows/weekly-release.yml diff --git a/.github/workflows/network-latency-chaos-test.yaml b/.github/workflows/network-latency-chaos-test.yaml deleted file mode 100644 index 813d5415d2e93..0000000000000 --- a/.github/workflows/network-latency-chaos-test.yaml +++ /dev/null @@ -1,197 +0,0 @@ -name: Network Latency Chaos Test - -on: - workflow_dispatch: -jobs: - - test-network-latency-chaos: - - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - pod: [datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio] - - steps: - - - name: Set env param - run: | - echo "RELEASE=test-${{ matrix.pod }}-network-latency" >> $GITHUB_ENV - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependency - uses: nick-invision/retry@v2 - with: - timeout_minutes: 5 - max_attempts: 3 - retry_on: error - shell: bash - command: | - pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org - pip install --upgrade protobuf - - - name: Deploy Chaos Mesh - shell: bash - run: | - helm repo add chaos-mesh https://charts.chaos-mesh.org - helm search repo chaos-mesh - kubectl create ns chaos-testing - helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock - kubectl get po -n chaos-testing - - - name: Deploy Milvus - shell: bash - working-directory: tests/python_client/chaos - run: | - echo "latest tag:" - bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml -n=chaos-testing; fi - if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f standalone-values.yaml -n=chaos-testing; fi - kubectl get pods -n chaos-testing - sleep 20s - kubectl get pods -n chaos-testing - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - # check whether milvus server is healthy - python scripts/hello_milvus.py - - - name: Chaos Test - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - # replace chaos object - sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/network_latency\/'/g" constants.py - sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_network_latency.yaml\'/g" constants.py - sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py - cat constants.py - timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed" - - - name: Result Analysis - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos/reports - run: | - echo "result analysis" - cat ${{ env.RELEASE }}.log || echo "no log file" - - - name: Milvus E2E Test - timeout-minutes: 10 - if: ${{ always() }} - shell: bash - working-directory: tests/python_client - run: | - kubectl get networkchaos -n chaos-testing - kubectl get pod -n chaos-testing - # wait all pod to be ready - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl get pod -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - - sleep 20s - nc -vz 127.0.0.1 19530 - - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - python chaos/scripts/hello_milvus.py --host 127.0.0.1 - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test - - - name: Deploy Milvus Again If Previous E2E Test Failed - timeout-minutes: 15 - if: ${{ failure() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - kubectl config set-context --current --namespace=chaos-testing - bash scripts/uninstall_milvus.sh ${{ env.RELEASE }} - if [ ${{ matrix.pod }} != "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml -n=chaos-testing; fi - if [ ${{ matrix.pod }} == "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set cluster.enabled=false --set etcd.replicaCount=1 --set minio.mode=standalone --set pulsar.enabled=false -n=chaos-testing; fi - kubectl get pods -n chaos-testing - sleep 20s - kubectl get pods -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - # check whether milvus server is healthy - python scripts/hello_milvus.py - - - name: Data Consist Test - timeout-minutes: 5 - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "data consist chaos test failed" - - - name: Milvus E2E Test - timeout-minutes: 10 - if: ${{ always() }} - shell: bash - working-directory: tests/python_client - run: | - kubectl get pod -n chaos-testing - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl get pod -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - nc -vz 127.0.0.1 19530 - - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - python chaos/scripts/hello_milvus.py --host 127.0.0.1 - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/data-consist-test - bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon k8s_logs/chaos-mesh-daemon - - - name: Upload logs - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.pod }} - path: | - tests/python_client/chaos/k8s_logs - tests/python_client/chaos/reports diff --git a/.github/workflows/network-partition-chaos-test.yaml b/.github/workflows/network-partition-chaos-test.yaml deleted file mode 100644 index b9331155fd050..0000000000000 --- a/.github/workflows/network-partition-chaos-test.yaml +++ /dev/null @@ -1,95 +0,0 @@ -name: Network Partition Chaos Test - -on: - workflow_dispatch: - -jobs: - - test-network-partition-chaos: - - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - pod: [datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio] - - steps: - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependency - shell: bash - working-directory: tests/python_client/chaos - run: | - pip install -r ../requirements.txt - pip install --upgrade protobuf - - - name: Deploy Chaos Mesh - shell: bash - run: | - helm repo add chaos-mesh https://charts.chaos-mesh.org - helm search repo chaos-mesh - kubectl create ns chaos-testing - helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-testing --version v2.0.1 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock - # wait for all pod running - sleep 60s - kubectl get po -n chaos-testing - - - name: Deploy Milvus - shell: bash - run: | - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - # install milvus with latest image - helm install --wait --timeout 360s chaos-testing milvus/milvus --set cluster.enabled=true --set image.all.repository=milvusdb/milvus --set image.all.tag=master-latest -n=chaos-testing - kubectl get pods -n chaos-testing - sleep 20s - kubectl get pods -n chaos-testing - kubectl port-forward service/chaos-testing-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - - name: Chaos Test - shell: bash - working-directory: tests/python_client/chaos - run: | - # replace chaos object - sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/network_partition\/'/g" constants.py - sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_network_partition.yaml\'/g" constants.py - cat constants.py - pytest -s -v test_chaos.py --host 127.0.0.1 - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing chaos-testing - bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon - - - name: Upload logs - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.pod }} - path: tests/python_client/chaos/k8s_logs \ No newline at end of file diff --git a/.github/workflows/pod-failure-chaos-test.yaml b/.github/workflows/pod-failure-chaos-test.yaml deleted file mode 100644 index 916f427e8744a..0000000000000 --- a/.github/workflows/pod-failure-chaos-test.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: Cluster N node Chaos Test - -on: - workflow_dispatch: - inputs: - image_tag: - description: The image tag to use for the chaos test - required: true - default: 'master-latest' - image_repo: - description: The image repo to use for the chaos test - required: true - default: 'milvusdb/milvus' - schedule: - - cron: "30 19 * * *" -jobs: - - test-cluster-n-node-chaos: - - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - chaos_type: [pod_failure, pod_kill] - pod: [querynode, datanode, indexnode, proxy] - - steps: - - name: Set env param - env: - DEFAULT_IMAGE_TAG: master-latest - DEFAULT_IMAGE_REPO: milvusdb/milvus - run: | - chaos_type=${{ matrix.chaos_type }} - release="test"-${{ matrix.pod }}-${chaos_type/_/-} - echo "RELEASE=$release" >> $GITHUB_ENV - echo "IMAGE_REPO=${{ github.event.inputs.image_repo || env.DEFAULT_IMAGE_REPO}}" >> $GITHUB_ENV - echo "IMAGE_TAG=${{ github.event.inputs.image_tag || env.DEFAULT_IMAGE_TAG}}" >> $GITHUB_ENV - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependency - uses: nick-invision/retry@v2 - with: - timeout_minutes: 5 - max_attempts: 3 - retry_on: error - shell: bash - command: | - pip install -r tests/python_client/requirements.txt - - - name: Deploy Chaos Mesh - shell: bash - run: | - helm repo add chaos-mesh https://charts.chaos-mesh.org - helm search repo chaos-mesh - kubectl create ns chaos-testing - helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock - sleep 60s - kubectl get po -n chaos-testing - - - name: Deploy Milvus - shell: bash - working-directory: tests/python_client/chaos - run: | - echo "latest tag:" - bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q - declare -A pod_map=( ["querynode"]="queryNode" ["indexnode"]="indexNode" ["datanode"]="dataNode" ["proxy"]="proxy") - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} --set ${pod_map[${{ matrix.pod }}]}.replicas=2 -f cluster-values.yaml -n=chaos-testing - kubectl get pods -n chaos-testing - sleep 20s - kubectl get pods -n chaos-testing - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - # check whether milvus server is healthy - pytest -s -v ../testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - python scripts/hello_milvus.py --host 127.0.0.1 - - - name: Chaos Test - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - # replace chaos object - sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${{ matrix.chaos_type }}\/'/g" constants.py - sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_${{ matrix.chaos_type }}.yaml\'/g" constants.py - sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py - sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py - cat constants.py - timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed" - - - name: Result Analysis - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos/reports - run: | - echo "result analysis" - cat ${{ env.RELEASE }}.log || echo "no log file" - - - name: Milvus E2E Test - timeout-minutes: 10 - if: ${{ always() }} - shell: bash - working-directory: tests/python_client - run: | - kubectl get pod -n chaos-testing - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl get pod -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - - sleep 20s - nc -vz 127.0.0.1 19530 - - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - python chaos/scripts/hello_milvus.py --host 127.0.0.1 - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test - - - name: Data Consist Test - timeout-minutes: 5 - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO || echo "data consist chaos test failed" - - - name: Milvus E2E Test - timeout-minutes: 5 - if: ${{ always() }} - shell: bash - working-directory: tests/python_client - run: | - kubectl get pod -n chaos-testing - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl get pod -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - - sleep 20s - nc -vz 127.0.0.1 19530 - - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - python chaos/scripts/hello_milvus.py --host 127.0.0.1 - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/data-consist-test - bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon - - - name: Upload logs - if: ${{ ! success() }} - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.pod }}-${{ matrix.chaos_type }} - path: tests/python_client/chaos/k8s_logs diff --git a/.github/workflows/pod-kill-chaos-test-kafka-version.yaml b/.github/workflows/pod-kill-chaos-test-kafka-version.yaml deleted file mode 100644 index 7cc6e965af8d3..0000000000000 --- a/.github/workflows/pod-kill-chaos-test-kafka-version.yaml +++ /dev/null @@ -1,188 +0,0 @@ -name: Pod Kill Chaos Test For Kafka Version - -on: - workflow_dispatch: - inputs: - image_tag: - description: The image tag to use for the chaos test - required: true - default: 'master-latest' - image_repo: - description: The image repo to use for the chaos test - required: true - default: 'milvusdb/milvus' - schedule: - - cron: "30 17 * * *" -jobs: - - test-pod-kill-chaos: - - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - pod: [allstandalone, allcluster, standalone, datacoord, datanode, indexcoord, indexnode, proxy, kafka, querycoord, querynode, rootcoord, etcd, minio] - - steps: - - - name: Set env param - env: - DEFAULT_IMAGE_TAG: master-latest - DEFAULT_IMAGE_REPO: milvusdb/milvus - run: | - echo "RELEASE=test-${{ matrix.pod }}-pod-kill" >> $GITHUB_ENV - echo "IMAGE_REPO=${{ github.event.inputs.image_repo || env.DEFAULT_IMAGE_REPO}}" >> $GITHUB_ENV - echo "IMAGE_TAG=${{ github.event.inputs.image_tag || env.DEFAULT_IMAGE_TAG}}" >> $GITHUB_ENV - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependency - uses: nick-invision/retry@v2 - with: - timeout_minutes: 5 - max_attempts: 3 - retry_on: error - shell: bash - command: | - pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org - - - name: Deploy Chaos Mesh - timeout-minutes: 2 - shell: bash - run: | - helm repo add chaos-mesh https://charts.chaos-mesh.org - helm search repo chaos-mesh - kubectl create ns chaos-testing - helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock - kubectl get po -n chaos-testing - - - name: Deploy Milvus - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - echo "latest tag:" - bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set pulsar.enabled=false --set kafka.enabled=true --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f cluster-values.yaml -n=chaos-testing; fi - if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set pulsar.enabled=false --set kafka.enabled=true --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f standalone-values.yaml -n=chaos-testing; fi - kubectl get pods -n chaos-testing - sleep 20s - kubectl get pods -n chaos-testing - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - - - name: Run e2e test before chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client - run: | - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Run data presistence test before chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client/chaos - run: | - pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Chaos Test - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - # replace chaos object - sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py - sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py - sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py - cat constants.py - timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed" - - - name: Result Analysis - timeout-minutes: 1 - shell: bash - working-directory: tests/python_client/chaos/reports - run: | - echo "result analysis" - cat ${{ env.RELEASE }}.log || echo "no log file" - - - name: Wait all pods ready - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client - run: | - kubectl get pod -n chaos-testing - # wait all pod to be ready - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl get pod -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - - sleep 20s - nc -vz 127.0.0.1 19530 - - - name: Run e2e test after chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client - run: | - - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Run data presistence test after chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client/chaos - run: | - pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Verify all collections after chaos - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - - pytest -s -v testcases/test_get_collections.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - sleep 2s - pytest -s -v testcases/test_all_collections_after_chaos.py --host 127.0.0.1 -n 4 --log-cli-level=INFO --capture=no - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test - - - name: Upload logs - if: ${{ ! success() }} - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.pod }} - path: | - tests/python_client/chaos/k8s_logs - tests/python_client/chaos/reports diff --git a/.github/workflows/pod-kill-chaos-test.yaml b/.github/workflows/pod-kill-chaos-test.yaml deleted file mode 100644 index cb5ab9cda4979..0000000000000 --- a/.github/workflows/pod-kill-chaos-test.yaml +++ /dev/null @@ -1,188 +0,0 @@ -name: Pod Kill Chaos Test - -on: - workflow_dispatch: - inputs: - image_tag: - description: The image tag to use for the chaos test - required: true - default: 'master-latest' - image_repo: - description: The image repo to use for the chaos test - required: true - default: 'milvusdb/milvus' - schedule: - - cron: "30 18 * * *" -jobs: - - test-pod-kill-chaos: - - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - pod: [allstandalone, allcluster, standalone, datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio] - - steps: - - - name: Set env param - env: - DEFAULT_IMAGE_TAG: master-latest - DEFAULT_IMAGE_REPO: milvusdb/milvus - run: | - echo "RELEASE=test-${{ matrix.pod }}-pod-kill" >> $GITHUB_ENV - echo "IMAGE_REPO=${{ github.event.inputs.image_repo || env.DEFAULT_IMAGE_REPO}}" >> $GITHUB_ENV - echo "IMAGE_TAG=${{ github.event.inputs.image_tag || env.DEFAULT_IMAGE_TAG}}" >> $GITHUB_ENV - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependency - uses: nick-invision/retry@v2 - with: - timeout_minutes: 5 - max_attempts: 3 - retry_on: error - shell: bash - command: | - pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org - - - name: Deploy Chaos Mesh - timeout-minutes: 2 - shell: bash - run: | - helm repo add chaos-mesh https://charts.chaos-mesh.org - helm search repo chaos-mesh - kubectl create ns chaos-testing - helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock - kubectl get po -n chaos-testing - - - name: Deploy Milvus - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - echo "latest tag:" - bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f cluster-values.yaml -n=chaos-testing; fi - if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f standalone-values.yaml -n=chaos-testing; fi - kubectl get pods -n chaos-testing - sleep 20s - kubectl get pods -n chaos-testing - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - - - name: Run e2e test before chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client - run: | - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Run data presistence test before chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client/chaos - run: | - pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Chaos Test - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - # replace chaos object - sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py - sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py - sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py - cat constants.py - timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed" - - - name: Result Analysis - timeout-minutes: 1 - shell: bash - working-directory: tests/python_client/chaos/reports - run: | - echo "result analysis" - cat ${{ env.RELEASE }}.log || echo "no log file" - - - name: Wait all pods ready - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client - run: | - kubectl get pod -n chaos-testing - # wait all pod to be ready - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s - kubectl get pod -n chaos-testing - ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9 - kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 & - - sleep 20s - nc -vz 127.0.0.1 19530 - - - name: Run e2e test after chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client - run: | - - pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Run data presistence test after chaos - timeout-minutes: 5 - shell: bash - working-directory: tests/python_client/chaos - run: | - pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - - - name: Verify all collections after chaos - timeout-minutes: 15 - shell: bash - working-directory: tests/python_client/chaos - run: | - - pytest -s -v testcases/test_get_collections.py --host 127.0.0.1 --log-cli-level=INFO --capture=no - sleep 2s - pytest -s -v testcases/test_all_collections_after_chaos.py --host 127.0.0.1 -n 4 --log-cli-level=INFO --capture=no - - - name: Export logs - if: ${{ always() }} - shell: bash - working-directory: tests/python_client/chaos - run: | - #in this step, verify whether pod has been killed by pod's age - kubectl get po -n chaos-testing - # export k8s log for chaos mesh and milvus - bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test - - - name: Upload logs - if: ${{ ! success() }} - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.pod }} - path: | - tests/python_client/chaos/k8s_logs - tests/python_client/chaos/reports diff --git a/.github/workflows/publish-builder.yaml b/.github/workflows/publish-builder.yaml deleted file mode 100644 index d8d0029fd44d5..0000000000000 --- a/.github/workflows/publish-builder.yaml +++ /dev/null @@ -1,90 +0,0 @@ -name: Publish Builder -# TODO: do not trigger action for some document file update - -# This workflow is triggered on pushes or pull request to the repository. -on: - push: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/builder/cpu/**' - - '.github/workflows/publish-builder.yaml' - - '!**.md' - pull_request: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/builder/cpu/**' - - '.github/workflows/publish-builder.yaml' - - '!**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - publish-builder: - name: ${{ matrix.arch }} ${{ matrix.os }} - runs-on: ubuntu-latest - timeout-minutes: 500 - strategy: - fail-fast: false - matrix: - os: [ubuntu20.04, amazonlinux2023, rockylinux8] - env: - OS_NAME: ${{ matrix.os }} - IMAGE_ARCH: ${{ matrix.arch }} - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner - with: - root-reserve-mb: 20480 - # overprovision-lvm: 'true' - swap-size-mb: 1024 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - - name: Checkout - uses: actions/checkout@v2 - - name: Get version from system time after release step - id: extracter - run: | - echo "::set-output name=version::$(date +%Y%m%d)" - echo "::set-output name=sha_short::$(git rev-parse --short=7 HEAD)" - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - milvusdb/milvus-env - tags: | - type=raw,enable=true,value=${{ matrix.os }}-{{date 'YYYYMMDD'}}-{{sha}} - type=raw,enable=true,value=${{ matrix.os }}-latest - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - file: build/docker/builder/cpu/${{ matrix.os }}/Dockerfile - - name: Bump Builder Version - uses: ./.github/actions/bump-builder-version - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04' - with: - tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}" - type: cpu - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} diff --git a/.github/workflows/publish-gpu-builder.yaml b/.github/workflows/publish-gpu-builder.yaml deleted file mode 100644 index 73ff250c020f3..0000000000000 --- a/.github/workflows/publish-gpu-builder.yaml +++ /dev/null @@ -1,84 +0,0 @@ -name: Publish Gpu Builder -# TODO: do not trigger action for some document file update - -# This workflow is triggered on pushes or pull request to the repository. -on: - push: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/builder/gpu/**' - - '.github/workflows/publish-gpu-builder.yaml' - - '!**.md' - pull_request: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/builder/gpu/**' - - '.github/workflows/publish-gpu-builder.yaml' - - '!**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - publish-gpu-builder: - runs-on: ubuntu-latest - timeout-minutes: 500 - env: - OS: ubuntu22.04 - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner - with: - root-reserve-mb: 20480 - # overprovision-lvm: 'true' - swap-size-mb: 1024 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - - name: Checkout - uses: actions/checkout@v2 - - name: Get version from system time after release step - id: extracter - run: | - echo "::set-output name=version::$(date +%Y%m%d)" - echo "::set-output name=sha_short::$(git rev-parse --short=7 HEAD)" - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - milvusdb/milvus-env - tags: | - type=raw,enable=true,value=gpu-${{ env.OS }}-{{date 'YYYYMMDD'}}-{{sha}} - type=raw,enable=true,value=gpu-${{ env.OS }}-latest - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - file: build/docker/builder/gpu/${{ env.OS }}/Dockerfile - - name: Bump Builder Version - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && ${{ env.OS == 'ubuntu22.04' }} - uses: ./.github/actions/bump-builder-version - with: - tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}" - type: gpu - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} diff --git a/.github/workflows/publish-krte-images.yaml b/.github/workflows/publish-krte-images.yaml deleted file mode 100644 index a9579dfaec3a7..0000000000000 --- a/.github/workflows/publish-krte-images.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: Publish KRTE Images -# TODO: do not trigger action for some document file update - -# This workflow is triggered on pushes or pull request to the repository. -on: - workflow_dispatch: - push: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/krte/**' - - '.github/workflows/publish-krte-images.yaml' - - '!**.md' - pull_request: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/krte/**' - - '.github/workflows/publish-krte-images.yaml' - - '!**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - publish-krte-images: - name: KRTE - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get version from system time after release step - id: extracter - run: | - echo "::set-output name=version::$(date +%Y%m%d)" - echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - - name: Docker Build - shell: bash - working-directory: build/docker/krte - run: | - export TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} - docker build --build-arg IMAGE_ARG="milvusdb/krte:$TAG" --build-arg GO_VERSION=1.15.8 -t "milvusdb/krte:$TAG" . - - name: Docker Push - if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' )&& github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - run: | - docker login -u ${{ secrets.DOCKERHUB_USER }} \ - -p ${{ secrets.DOCKERHUB_TOKEN }} - export TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} - docker push "milvusdb/krte:$TAG" - echo "Push krte image Succeeded" - - name: Update KRTE Image Changes - if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - working-directory: build/ci/jenkins/pod - run: | - export TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} - sed -i "s#krte:.*#krte:${TAG}#g" ./build/ci/jenkins/pod/rte.yaml - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add rte.yaml - git commit -m "Update KRTE image changes" - - name: Create Pull Request - id: cpr - if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && github.repository == 'milvus-io/milvus' - continue-on-error: true - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: update_krte_${{ github.sha }} - delete-branch: true - title: '[automated] Update KRTE image changes' - body: | - Update KRTE image changes - - Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com - - name: Check outputs - if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && github.repository == 'milvus-io/milvus' - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/publish-test-images.yaml b/.github/workflows/publish-test-images.yaml deleted file mode 100644 index 5587389d63f42..0000000000000 --- a/.github/workflows/publish-test-images.yaml +++ /dev/null @@ -1,97 +0,0 @@ -name: Publish Test Images -# TODO: do not trigger action for some document file update - -# This workflow is triggered on pushes or pull request to the repository. -on: - push: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'tests/docker/Dockerfile' - - 'tests/python_client/requirements.txt' - - '.github/workflows/publish-test-images.yaml' - - '!**.md' - pull_request: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'tests/docker/Dockerfile' - - 'tests/python_client/requirements.txt' - - '.github/workflows/publish-test-images.yaml' - - '!**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - publish-pytest-images: - name: PyTest - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get version from system time after release step - id: extracter - run: | - echo "::set-output name=version::$(date +%Y%m%d)" - echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - - name: Docker Pull - shell: bash - working-directory: tests/docker - run: | - docker-compose pull --ignore-pull-failures pytest - - name: Docker Build - shell: bash - working-directory: tests/docker - run: | - IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} docker-compose build pytest - export LATEST_IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} - IMAGE_TAG=latest docker-compose build pytest - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - working-directory: tests/docker - run: | - docker login -u ${{ secrets.DOCKERHUB_USER }} \ - -p ${{ secrets.DOCKERHUB_TOKEN }} - IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} docker-compose push pytest - IMAGE_TAG=latest docker-compose push pytest - echo "Push pytest image Succeeded" - - name: Update Pytest Image Changes - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - working-directory: . - run: | - sed -i "s#^IMAGE_TAG=.*#IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./tests/docker/.env - sed -i "s#^LATEST_IMAGE_TAG=.*#LATEST_IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./tests/docker/.env - sed -i "s#pytest:.*#pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./ci/jenkins/pod/rte.yaml - sed -i "s#pytest:.*#pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./ci/jenkins/pod/e2e.yaml - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add ./tests/docker/.env - git add ./ci/jenkins/pod/rte.yaml - git add ./ci/jenkins/pod/e2e.yaml - git commit -m "Update Pytest image changes" - - name: Create Pull Request - id: cpr - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: true - branch: update_pytest_${{ github.sha }} - delete-branch: true - title: '[automated] Update Pytest image changes' - body: | - Update Pytest image changes - See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }} - Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com - - name: Check outputs - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/remote-rhel-build.yaml b/.github/workflows/remote-rhel-build.yaml index 58c24f2166bd3..b1937e1bd404a 100644 --- a/.github/workflows/remote-rhel-build.yaml +++ b/.github/workflows/remote-rhel-build.yaml @@ -39,6 +39,7 @@ jobs: run: | PUBLIC_IP=$(terraform output -json | jq -r '.public_ip.value') podman system connection add terraform --identity ~/.ssh/id_rsa ssh://$PUBLIC_IP/run/user/1000/podman/podman.sock + podman system connection list - name: Build image run: | diff --git a/.github/workflows/rerun-failure-checks.yaml b/.github/workflows/rerun-failure-checks.yaml deleted file mode 100644 index fa779cfd46304..0000000000000 --- a/.github/workflows/rerun-failure-checks.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Rerun Failure Checks -on: - issue_comment: - types: [created] -jobs: - rerun-checks: - if: "github.event_name == 'issue_comment'&& startsWith(github.event.comment.body, 'rerun ut')" - runs-on: ubuntu-latest - steps: - - name: Is Organization Member - shell: bash - if: "github.event_name == 'issue_comment'" - id: is_organization_member - run: | - response_code=$(curl -sIL -w %{http_code} -H "Authorization: token ${{ secrets.RERUN_BOT_TOKEN }}" -H "Accept: application/vnd.github.v3+json" \ - -o /dev/null https://api.github.com/orgs/${GITHUB_REPOSITORY_OWNER}/members/${{ github.event.sender.login }}) - echo " response code is ${response_code} " - if [[ ${response_code} == '204' ]];then - echo "::set-output name=is-member::true" - elif [[ ${response_code} == '404' ]]; then - echo "::set-output name=is-member::false" - else - echo "Please check if the repository secret RERUN_BOT_TOKEN still exists and have the permission to read organization membership." - exit 1 - fi - - name: Create Comment for Non-Org Member - if: "github.event_name == 'issue_comment' && steps.is_organization_member.outputs.is-member == 'false'" - uses: actions-cool/issues-helper@v2.5.0 - with: - actions: 'create-comment' - token: ${{ secrets.GITHUB_TOKEN }} - issue-number: ${{ github.event.issue.number }} - body: | - Hello ${{ github.event.sender.login }}, you are not in the organization, so you do not have the permission to rerun the workflow, please contact `@milvus-io/milvus-maintainers` for help. - - uses: actions/checkout@v2 - - name: Rerun Failure Checks - if: "github.event_name == 'issue_comment' && steps.is_organization_member.outputs.is-member == 'true'" - uses: zymap/bot@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # replace here to your token - with: - repo_owner: milvus-io # replace here to your repo owner - repo_name: milvus # replace here to your repo name - rerun_cmd: rerun ut - comment: ${{ github.event.comment.body }} - - \ No newline at end of file diff --git a/.github/workflows/simd-compatibility-test.yaml b/.github/workflows/simd-compatibility-test.yaml deleted file mode 100644 index ee26169de6f4e..0000000000000 --- a/.github/workflows/simd-compatibility-test.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: SIMD Compatibility Test - -on: - workflow_dispatch: - schedule: - - cron: "30 18 * * 0" - -jobs: - test-simd-compatibility: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - simd_type: ["sse4_2", "avx", "avx2", "avx512"] - steps: - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependency - shell: bash - working-directory: tests/python_client/deploy - run: | - pip install -r requirements.txt - pip install --upgrade protobuf - - - name: install milvus operator and milvus cluster - shell: bash - working-directory: tests/python_client/customize/template - run: | - kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.3/cert-manager.yaml - sleep 30s - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=360s - kubectl get pods -A - - #install milvus operator - kubectl apply -f https://raw.githubusercontent.com/milvus-io/milvus-operator/main/deploy/manifests/deployment.yaml - sleep 30s - kubectl wait --for=condition=Ready pod --all -n milvus-operator --timeout=360s - kubectl get pods -A - #install milvus cluster - - # change simdType - sed -i "s/simdType.*/simdType\: ${{ matrix.simd_type }}/g" minimum.yaml - kubectl apply -f minimum.yaml - sleep 60s - # wait all pod running - kubectl get pods -A - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release-minio --timeout=360s - kubectl get pods -A - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release-etcd --timeout=360s - kubectl get pods -A - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release-pulsar --timeout=360s - kubectl get pods -A - sleep 60s - kubectl get pods -A - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release --timeout=360s - sleep 30s - kubectl get pods -A - - # port-forward - kubectl port-forward service/my-release-milvus 19530 >/dev/null 2>&1 & - sleep 20s - # check whether port-forward success - nc -vz 127.0.0.1 19530 - - - name: Run E2E test - shell: bash - working-directory: tests/python_client/chaos - run: | - python scripts/hello_milvus.py diff --git a/.github/workflows/update-knowhere-commit.yaml b/.github/workflows/update-knowhere-commit.yaml deleted file mode 100644 index 051f3b8d49290..0000000000000 --- a/.github/workflows/update-knowhere-commit.yaml +++ /dev/null @@ -1,75 +0,0 @@ -name: Update Knowhere Commit - -on: - workflow_dispatch: - schedule: - # * is a special character in YAML so you have to quote this string - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - # │ │ │ │ │ - - cron: '0 15 * * *' - - -jobs: - update-knowhere-commits: - name: update-knowhere-commit - runs-on: ubuntu-latest - env: - OS_NAME: ubuntu20.04 - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: get knowhere latest commit - id: get-knowhere-latest-commit - run: | - cd .. - git clone https://github.com/zilliztech/Knowhere.git - cd Knowhere - export commit=$(git rev-parse --short HEAD) - echo $commit - echo "::set-output name=knowhere-commit::$commit" - cd ../milvus - - name: Update Commit Changes - continue-on-error: true - shell: bash - run: | - sed -i "0,/(\ KNOWHERE_VERSION/ s#( KNOWHERE_VERSION.*#( KNOWHERE_VERSION ${{ steps.get-knowhere-latest-commit.outputs.knowhere-commit }} )#g" internal/core/thirdparty/knowhere/CMakeLists.txt - head -n 17 internal/core/thirdparty/knowhere/CMakeLists.txt - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add internal/core/thirdparty/knowhere/CMakeLists.txt - git commit -m "Update knowhere commit" - - name: Create Pull Request - id: cpr - continue-on-error: true - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} - author: sre-ci-robot - signoff: true - branch: update_knowhere_commit_${{ github.sha }} - delete-branch: true - title: '[automated] Update Knowhere Commit' - body: | - Update Knowhere Commit - Signed-off-by: sre-ci-robot sre-ci-robot@users.noreply.github.com - - name: Check outputs - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" - - name: Send Message To Feishu - env: - ACTIONS_FEISHU_TAG: 'v1.3.1' - INPUT_WEBHOOK: "${{ secrets.FEISHU_WEBHOOK_KNOWHERE_COMMIT_BOT }}" - INPUT_MESSAGE_TYPE: text - INPUT_CONTENT: "Pr url by rebot: ${{ steps.cpr.outputs.pull-request-url }}" - run: | - wget -q https://github.com/xiachufang/actions-feishu/releases/download/${{ env.ACTIONS_FEISHU_TAG }}/linux-amd64-actions-feishu.tar.gz - tar zxf linux-amd64-actions-feishu.tar.gz feishu - ./feishu diff --git a/.github/workflows/weekly-release.yml b/.github/workflows/weekly-release.yml deleted file mode 100644 index ee9ceb1647206..0000000000000 --- a/.github/workflows/weekly-release.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Weekly Release -on: - schedule: - # * is a special character in YAML so you have to quote this string - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - # │ │ │ │ │ - - cron: '0 0 * * 2,5' - -jobs: - nightly: - name: Run Weekly Release - if: github.repository == 'milvus-io/milvus' - runs-on: ubuntu-latest - env: - IMAGE_REPO: "milvusdb" - DEV: "milvus" - WEEKLY: "weekly-build" - TAG_PREFIX: "master-" - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - - name: Get the latest of Milvus dev image tag - shell: bash - working-directory: scripts - run: echo "tag=$(./docker_image_find_tag.sh -n ${IMAGE_REPO}/${DEV} -t ${TAG_PREFIX}latest -f ${TAG_PREFIX} -F -L -q)" >> $GITHUB_ENV - - - - name: Pull latest milvus image with tag prefix master- - run: | - docker pull "${IMAGE_REPO}/${DEV}:${{ env.tag }}" - docker tag "${IMAGE_REPO}/${DEV}:${{ env.tag }}" "${IMAGE_REPO}/${WEEKLY}:${{ env.tag }}" - - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push Docker image - run: | - docker push "${IMAGE_REPO}/${WEEKLY}:${{ env.tag }}" - - - name: Set release build - shell: bash - run: | - tag=${{ env.tag }} - IFS=- read branch date sha <<< "$tag" - echo "build=$date" >> $GITHUB_ENV - echo "sha=$(git rev-parse $sha)" >> $GITHUB_ENV - - - name: Create a weekly release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - body: "Image: ${{ env.IMAGE_REPO}}/${{ env.WEEKLY }}:${{ env.tag }}" - prerelease: true - tag_name: "v2.2-testing-${{ env.build }}" - release_name: "milvus-2.2-testing-${{ env.build }}" - commitish: "${{ env.sha }}" - -