Test

add api integration test #618

Workflow file for this run

	name: Test

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main

	jobs:
	milvus-cdc-system-test:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- uses: actions/setup-go@v4
	with:
	go-version: '1.21'
	cache-dependency-path: server/go.sum
	cache: true

	- name: set up python
	uses: actions/setup-python@v2
	with:
	python-version: '3.8'
	cache: 'pip'

	- name: Build CDC
	timeout-minutes: 15
	shell: bash
	run: \|
	docker build --build-arg GIT_COMMIT_ARG=$(git rev-parse --short HEAD) -t milvus-cdc:latest .
	docker tag milvus-cdc:latest milvusdb/milvus-cdc:latest

	- name: Creating kind cluster
	uses: helm/[email protected]

	- name: Print cluster information
	run: \|
	kubectl config view
	kubectl cluster-info
	kubectl get nodes
	kubectl get pods -o wide -n kube-system
	helm version
	kubectl version

	- name: Deploy Source Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/upstream/
	run: \|
	docker compose up -d
	bash ../../../scripts/check_healthy.sh
	docker compose ps -a

	- name: Deploy Downstream Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/downstream
	run: \|
	helm repo add milvus https://zilliztech.github.io/milvus-helm
	helm repo update
	helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values.yaml
	kubectl get pods
	kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 &
	sleep 20s
	nc -vz 127.0.0.1 19500

	- name: Deploy Milvus CDC
	timeout-minutes: 15
	working-directory: deployment/docker
	shell: bash
	run: \|
	docker compose --verbose up -d
	sleep 20s
	docker compose ps
	docker compose logs \|\| true

	- name: Create CDC task
	timeout-minutes: 15
	run: \|
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "create",
	"request_data": {
	"milvus_connect_param": {
	"host": "127.0.0.1",
	"port": 19500,
	"username": "",
	"password": "",
	"enable_tls": false,
	"connect_timeout": 120
	},
	"extra_info": {
	"enable_user_role": true
	},
	"db_collections": {
	"*": [
	{
	"name": "*"
	}
	]
	}
	}
	}'

	- name: Checkout Milvus Repo
	uses: actions/checkout@v2
	with:
	repository: zhuwenxing/milvus
	ref: refine_checker
	path: milvus

	- name: Run test
	timeout-minutes: 15
	shell: bash
	working-directory: milvus/tests/python_client
	run: \|
	pip install -r requirements.txt --trusted-host https://test.pypi.org
	cd chaos && pytest testcases/test_concurrent_operation.py --host 127.0.0.1 --log-cli-level=INFO --capture=no --request_duration 10m

	- name: List CDC task
	timeout-minutes: 15
	working-directory: deployment/docker
	shell: bash
	run: \|
	docker compose logs
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "list"
	}'

	- name: Diff upstream and downstream
	if: ${{ always() }}
	timeout-minutes: 10
	shell: bash
	working-directory: milvus/tests/python_client/chaos/scripts
	run: \|
	python diff_upstream_downstream.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500


	- name: Export milvus cdc logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: deployment/docker
	run: \|
	docker compose ps -a
	docker stats --no-stream
	docker compose logs > server.log

	- name: Export upstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: tests/deployment/upstream
	run: \|
	docker compose ps -a
	docker stats --no-stream
	bash ../../../scripts/export_log_docker.sh
	- name: Export downstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: scripts
	run: \|
	kubectl get pods \|\| true
	bash export_log_k8s.sh default cdc-downstream k8s_logs

	- name: Upload logs
	if: ${{ always() }}
	uses: actions/upload-artifact@v4
	with:
	name: system-test-logs
	path: \|
	scripts/k8s_logs
	tests/deployment/upstream/logs
	deployment/docker/server.log

	milvus-cdc-function-test:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- uses: actions/setup-go@v4
	with:
	go-version: '1.21'
	cache-dependency-path: server/go.sum
	cache: true

	- name: set up python
	uses: actions/setup-python@v2
	with:
	python-version: '3.8'
	cache: 'pip'

	- name: Build CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	make build
	ls -l

	- name: Creating kind cluster
	uses: helm/[email protected]

	- name: Print cluster information
	run: \|
	kubectl config view
	kubectl cluster-info
	kubectl get nodes
	kubectl get pods -o wide -n kube-system
	helm version
	kubectl version

	- name: Deploy Source Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/upstream/
	run: \|
	docker compose up -d
	bash ../../../scripts/check_healthy.sh
	docker compose ps -a

	- name: Deploy Downstream Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/downstream
	run: \|
	helm repo add milvus https://zilliztech.github.io/milvus-helm
	helm repo update
	helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values-auth.yaml
	kubectl get pods
	kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 &
	sleep 20s
	nc -vz 127.0.0.1 19500

	- name: Deploy Milvus CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cp ../deployment/docker/cdc.yaml configs/cdc.yaml
	../bin/cdc > server.log 2>&1 &
	sleep 20s

	- name: Create CDC task
	timeout-minutes: 15
	run: \|
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "create",
	"request_data": {
	"milvus_connect_param": {
	"uri": "http://127.0.0.1:19500",
	"token": "root:Milvus",
	"connect_timeout": 120
	},
	"extra_info": {
	"enable_user_role": true
	},
	"db_collections": {
	"*": [
	{
	"name": "*"
	}
	]
	}
	}
	}'

	- name: Run test
	timeout-minutes: 15
	shell: bash
	working-directory: tests
	run: \|
	pip install -r requirements.txt --trusted-host https://test.pypi.org
	pytest testcases/test_cdc_sync_requests.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500

	- name: List CDC task
	if: ${{ always() }}
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cat server.log \| tail -n 100
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "list"
	}'

	- name: Export upstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: tests/deployment/upstream
	run: \|
	docker compose ps -a
	docker stats --no-stream
	bash ../../../scripts/export_log_docker.sh
	- name: Export downstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: scripts
	run: \|
	kubectl get pods \|\| true
	bash export_log_k8s.sh default cdc-downstream k8s_logs

	- name: Upload logs
	if: ${{ always() }}
	uses: actions/upload-artifact@v4
	with:
	name: func-test-logs
	path: \|
	scripts/k8s_logs
	tests/deployment/upstream/logs
	server/server.log

	milvus-cdc-function-test-4-16:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- uses: actions/setup-go@v4
	with:
	go-version: '1.21'
	cache-dependency-path: server/go.sum
	cache: true

	- name: set up python
	uses: actions/setup-python@v2
	with:
	python-version: '3.8'
	cache: 'pip'

	- name: Build CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	make build
	ls -l

	- name: Creating kind cluster
	uses: helm/[email protected]

	- name: Print cluster information
	run: \|
	kubectl config view
	kubectl cluster-info
	kubectl get nodes
	kubectl get pods -o wide -n kube-system
	helm version
	kubectl version

	- name: Deploy Source Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/channel_4/
	run: \|
	docker compose up -d
	bash ../../../scripts/check_healthy.sh
	docker compose ps -a

	- name: Deploy Downstream Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/downstream
	run: \|
	helm repo add milvus https://zilliztech.github.io/milvus-helm
	helm repo update
	helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values.yaml
	kubectl get pods
	kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 &
	sleep 20s
	nc -vz 127.0.0.1 19500

	- name: Deploy Milvus CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cp ../deployment/docker/cdc-4.yaml configs/cdc.yaml
	../bin/cdc > server.log 2>&1 &
	sleep 20s

	- name: Create CDC task
	timeout-minutes: 15
	run: \|
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "create",
	"request_data": {
	"milvus_connect_param": {
	"uri": "http://127.0.0.1:19500",
	"token": "root:Milvus",
	"connect_timeout": 120,
	"channel_num": 16
	},
	"extra_info": {
	"enable_user_role": true
	},
	"db_collections": {
	"*": [
	{
	"name": "*"
	}
	]
	}
	}
	}'

	- name: Run test
	timeout-minutes: 15
	shell: bash
	working-directory: tests
	run: \|
	pip install -r requirements.txt --trusted-host https://test.pypi.org
	pytest testcases/test_cdc_sync_requests.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500

	- name: List CDC task
	if: ${{ always() }}
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cat server.log \| tail -n 100
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "list"
	}'

	- name: Export upstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: tests/deployment/channel_4/
	run: \|
	docker compose ps -a
	docker stats --no-stream
	bash ../../../scripts/export_log_docker.sh
	- name: Export downstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: scripts
	run: \|
	kubectl get pods \|\| true
	bash export_log_k8s.sh default cdc-downstream k8s_logs

	- name: Upload logs
	if: ${{ always() }}
	uses: actions/upload-artifact@v4
	with:
	name: func-test-logs-4-16
	path: \|
	scripts/k8s_logs
	tests/deployment/channel_4/logs
	server/server.log

	milvus-cdc-function-test-16-8:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- uses: actions/setup-go@v4
	with:
	go-version: '1.21'
	cache-dependency-path: server/go.sum
	cache: true

	- name: set up python
	uses: actions/setup-python@v2
	with:
	python-version: '3.8'
	cache: 'pip'

	- name: Build CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	make build
	ls -l

	- name: Creating kind cluster
	uses: helm/[email protected]

	- name: Print cluster information
	run: \|
	kubectl config view
	kubectl cluster-info
	kubectl get nodes
	kubectl get pods -o wide -n kube-system
	helm version
	kubectl version

	- name: Deploy Source Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/upstream/
	run: \|
	docker compose up -d
	bash ../../../scripts/check_healthy.sh
	docker compose ps -a

	- name: Deploy Downstream Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/downstream
	run: \|
	helm repo add milvus https://zilliztech.github.io/milvus-helm
	helm repo update
	helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values-8.yaml
	kubectl get pods
	kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 &
	sleep 20s
	nc -vz 127.0.0.1 19500

	- name: Deploy Milvus CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cp ../deployment/docker/cdc-16.yaml configs/cdc.yaml
	../bin/cdc > server.log 2>&1 &
	sleep 20s

	- name: Create CDC task
	timeout-minutes: 15
	run: \|
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "create",
	"request_data": {
	"milvus_connect_param": {
	"uri": "http://127.0.0.1:19500",
	"token": "root:Milvus",
	"connect_timeout": 120,
	"channel_num": 8
	},
	"extra_info": {
	"enable_user_role": true
	},
	"db_collections": {
	"*": [
	{
	"name": "*"
	}
	]
	}
	}
	}'

	- name: Run test
	timeout-minutes: 15
	shell: bash
	working-directory: tests
	run: \|
	pip install -r requirements.txt --trusted-host https://test.pypi.org
	pytest testcases/test_cdc_sync_requests.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500

	- name: List CDC task
	if: ${{ always() }}
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cat server.log \| tail -n 100
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "list"
	}'

	- name: Export upstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: tests/deployment/upstream
	run: \|
	docker compose ps -a
	docker stats --no-stream
	bash ../../../scripts/export_log_docker.sh
	- name: Export downstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: scripts
	run: \|
	kubectl get pods \|\| true
	bash export_log_k8s.sh default cdc-downstream k8s_logs

	- name: Upload logs
	if: ${{ always() }}
	uses: actions/upload-artifact@v4
	with:
	name: func-test-logs-16-8
	path: \|
	scripts/k8s_logs
	tests/deployment/upstream/logs
	server/server.log
	milvus-cdc-function-test-with-db:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- uses: actions/setup-go@v4
	with:
	go-version: '1.21'
	cache-dependency-path: server/go.sum
	cache: true

	- name: set up python
	uses: actions/setup-python@v2
	with:
	python-version: '3.8'
	cache: 'pip'

	- name: Build CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	make build
	ls -l

	- name: Creating kind cluster
	uses: helm/[email protected]

	- name: Print cluster information
	run: \|
	kubectl config view
	kubectl cluster-info
	kubectl get nodes
	kubectl get pods -o wide -n kube-system
	helm version
	kubectl version

	- name: Deploy Source Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/upstream/
	run: \|
	docker compose up -d
	bash ../../../scripts/check_healthy.sh
	docker compose ps -a

	- name: Deploy Downstream Milvus
	timeout-minutes: 15
	shell: bash
	working-directory: tests/deployment/downstream
	run: \|
	helm repo add milvus https://zilliztech.github.io/milvus-helm
	helm repo update
	helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values-auth.yaml
	kubectl get pods
	kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 &
	sleep 20s
	nc -vz 127.0.0.1 19500

	- name: Deploy Milvus CDC
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cp ../deployment/docker/cdc.yaml configs/cdc.yaml
	../bin/cdc > server.log 2>&1 &
	sleep 20s

	- name: Create CDC task
	timeout-minutes: 15
	run: \|
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "create",
	"request_data": {
	"milvus_connect_param": {
	"uri": "http://127.0.0.1:19500",
	"token": "root:Milvus",
	"connect_timeout": 120
	},
	"extra_info": {
	"enable_user_role": true
	},
	"db_collections": {
	"foo": [
	{
	"name": "*"
	}
	]
	}
	}
	}'

	- name: Run test
	timeout-minutes: 15
	shell: bash
	working-directory: tests
	run: \|
	pip install -r requirements.txt --trusted-host https://test.pypi.org
	pytest testcases/test_cdc_database.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
	pytest testcases/test_cdc_get.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
	pytest testcases/test_cdc_list.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
	pytest testcases/test_cdc_delete.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
	pytest testcases/test_cdc_pause.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
	pytest testcases/test_cdc_resume.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500

	- name: List CDC task
	if: ${{ always() }}
	timeout-minutes: 15
	working-directory: server
	shell: bash
	run: \|
	cat server.log \| tail -n 100
	curl --location '127.0.0.1:8444/cdc' \
	--header 'Content-Type: application/json' \
	--data '{
	"request_type": "list"
	}'

	- name: Export upstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: tests/deployment/upstream
	run: \|
	docker compose ps -a
	docker stats --no-stream
	bash ../../../scripts/export_log_docker.sh
	- name: Export downstream milvus logs
	if: ${{ always() }}
	timeout-minutes: 5
	working-directory: scripts
	run: \|
	kubectl get pods \|\| true
	bash export_log_k8s.sh default cdc-downstream k8s_logs

	- name: Upload logs
	if: ${{ always() }}
	uses: actions/upload-artifact@v4
	with:
	name: func-test-logs-with-db
	path: \|
	scripts/k8s_logs
	tests/deployment/upstream/logs
	server/server.log

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

add api integration test #618

Workflow file

add api integration test #618

Jobs

Run details

Workflow file for this run