Skip to content
Test

restore: skip existing RBAC information during restoration #1390

Sign in to view logs

restore: skip existing RBAC information during restoration

restore: skip existing RBAC information during restoration #1390

Workflow file for this run

.github/workflows/main.yaml at fb646bb
name: Test
on:
push:
branches:
- main
pull_request:
branches:
- main
workflow_dispatch:
schedule:
- cron: '0 0 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test-backup-restore-cross-version:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
another_milvus_mode: [standalone]
source_image_tag: [2.3-latest, 2.4-latest]
target_image_tag: [master-latest, 2.4-latest]
exclude:
- source_image_tag: 2.4-latest
target_image_tag: 2.4-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v3
with:
go-version: '1.18.0'
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.source_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup delete -n my_backup
./milvus-backup list
- name: Uninstall Milvus
timeout-minutes: 5
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.target_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
shell: bash
run: |
pwd
ls -l
tree
- name: Restore backup from another Milvus
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data from another Milvus
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
test-backup-restore-after-upgrade:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
another_milvus_mode: [standalone]
source_image_tag: [v2.4.17, 2.4-latest]
target_image_tag: [master-latest, 2.4-latest]
exclude:
- source_image_tag: 2.4-latest
target_image_tag: 2.4-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v3
with:
go-version: '1.18.0'
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.source_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Upgrade Milvus
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
docker-compose down
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.target_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
test-backup-restore-cli:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
another_milvus_mode: [standalone]
image_tag: [master-latest]
# mq_type: [pulsar, kafka] # TODO: add pulsar and kafka
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v3
with:
go-version: '1.18.0'
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup delete -n my_backup
./milvus-backup list
- name: Uninstall Milvus
timeout-minutes: 5
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
shell: bash
run: |
pwd
ls -l
tree
- name: Restore backup from another Milvus
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data from another Milvus
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
test-backup-restore-with-rbac-config:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [helm]
milvus_mode: [standalone]
image_tag: [master-latest]
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v3
with:
go-version: '1.18.0'
cache: "true"
- name: Creating kind cluster
if: ${{ matrix.deploy_tools == 'helm' }}
uses: helm/[email protected]
- name: Build
timeout-minutes: 5
shell: bash
run: |
yq -i '.log.level = "debug"' configs/backup.yaml
yq -i '.milvus.authorizationEnabled = true' configs/backup.yaml
yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml
yq -i '.minio.rootPath = "file"' configs/backup.yaml
yq -i '.minio.backupPort = 9010' configs/backup.yaml
yq -i '.minio.backupBucketName = "milvus-bucket"' configs/backup.yaml
yq -i '.minio.backupRootPath = "backup"' configs/backup.yaml
yq -i '.minio.crossStorage = true' configs/backup.yaml
cat configs/backup.yaml || true
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
helm repo add milvus https://zilliztech.github.io/milvus-helm
helm repo update
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".image.all.tag=\"${tag}\"" rbac-values.yaml
helm install --wait --debug --timeout 600s milvus-backup milvus/milvus -f rbac-values.yaml
helm install --wait --debug --timeout 600s milvus-restore milvus/milvus -f rbac-values.yaml
kubectl get pods
kubectl port-forward service/milvus-backup 19530 >/dev/null 2>&1 &
kubectl port-forward service/milvus-backup-minio 9000 >/dev/null 2>&1 &
kubectl port-forward service/milvus-restore 19531:19530 >/dev/null 2>&1 &
kubectl port-forward service/milvus-restore-minio 9010:9000 >/dev/null 2>&1 &
sleep 10
nc -vz 127.0.0.1 19530
nc -vz 127.0.0.1 9000
nc -vz 127.0.0.1 19531
nc -vz 127.0.0.1 9010
sleep 10
kubectl get pods -n default | grep milvus-backup
kubectl get pods -n default | grep milvus-restore
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
fi
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py --uri http://127.0.0.1:19530
python example/prepare_rbac.py --uri http://127.0.0.1:19530
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup --rbac
./milvus-backup list
- name: Modify Restore Config
timeout-minutes: 5
shell: bash
run: |
yq -i '.milvus.port = 19531' configs/backup.yaml
yq -i '.minio.port = 9010' configs/backup.yaml
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover --rbac
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py --uri http://127.0.0.1:19531
python example/verify_rbac.py --uri http://127.0.0.1:19531
- name: Export logs
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
kubectl get pods
bash ../../scripts/export_log_k8s.sh default milvus-backup logs
bash ../../scripts/export_log_k8s.sh default milvus-restore logs
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v4
with:
name: custom-config-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
path: |
./logs
./server.log
/tmp/ci_logs
deployment/${{ matrix.milvus_mode }}/logs
test-backup-restore-api:
runs-on: ubuntu-latest
needs: test-backup-restore-cli
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
image_tag: [master-latest, 2.4-latest]
case_tag: [L0, L1, L2, MASTER]
exclude:
- image_tag: 2.4-latest
case_tag: MASTER
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v3
with:
go-version: '1.18.0'
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
yq -i '.log.level = "debug"' configs/backup.yaml
cat configs/backup.yaml || true
go get
go build
bash build_image.sh
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
docker-compose up -d
bash ../../scripts/check_healthy.sh
docker-compose ps -a
- name: Start backup server
timeout-minutes: 5
shell: bash
working-directory: build/deployment
run: |
docker-compose up -d
sleep 10
docker-compose ps -a
- name: Run test
timeout-minutes: 120
shell: bash
working-directory: tests
run: |
pytest -s -v --tags ${{ matrix.case_tag }} -n 4
- name: Get Milvus status
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
docker-compose ps -a || true
- name: Export Milvus logs
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
- name: Export Backup Server logs
if: ${{ always() }}
shell: bash
working-directory: build/deployment
run: |
docker-compose logs > /tmp/ci_logs/backup_server.log 2>&1 || true
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v4
with:
name: api-test-logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}-${{ matrix.image_tag }}-${{ matrix.case_tag }}
path: |
/tmp/ci_logs
deployment/${{ matrix.milvus_mode }}/logs