From 93dc7f60a8cbb8378f734be2d8fc381c7378ec8b Mon Sep 17 00:00:00 2001 From: Shinya Hayashi Date: Fri, 14 Jun 2024 00:56:05 +0000 Subject: [PATCH] split test --- .../deploy-rook-ceph-on-two-k8s-clusters.yaml | 72 ++++++++++++++++ .github/workflows/e2e.yaml | 42 +--------- .../workflows/inter-cluster-connection.yaml | 83 +++++++++++++++++++ test/e2e/Makefile | 78 +++++++++++------ 4 files changed, 212 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/deploy-rook-ceph-on-two-k8s-clusters.yaml create mode 100644 .github/workflows/inter-cluster-connection.yaml diff --git a/.github/workflows/deploy-rook-ceph-on-two-k8s-clusters.yaml b/.github/workflows/deploy-rook-ceph-on-two-k8s-clusters.yaml new file mode 100644 index 00000000..e233f09e --- /dev/null +++ b/.github/workflows/deploy-rook-ceph-on-two-k8s-clusters.yaml @@ -0,0 +1,72 @@ +name: "two k8s cluster" +on: + pull_request: + paths-ignore: + - "**/*.md" + - "CODEOWNERS" + push: + paths-ignore: + - "**/*.md" + - "CODEOWNERS" + branches: + - "main" + +jobs: + test: + runs-on: "ubuntu-22.04" + env: + MINIKUBE_HOME: ${{ github.workspace }}/test/e2e/bin + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: "go.mod" + - run: env | grep MINIKUBE_HOME + - run: sudo apt-get update + - run: sudo apt-get purge -y apport apport-symptoms fwupd nano netplan.io popularity-contest unattended-upgrades update-manager-core snapd + - run: sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + - run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup /usr/lib/jvm || true + - run: sudo apt-get purge -y $(dpkg-query -W | grep nginx | awk '{print $1}') + - run: sudo apt-get purge -y aria2 ansible azure-cli shellcheck rpm xorriso zsync firefox gfortran-9 google-chrome-stable google-cloud-sdk + - run: sudo apt-get purge -y imagemagick libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional mercurial apt-transport-https + - run: sudo apt-get purge -y mono-complete unixodbc-dev yarn chrpath libxft-dev libfreetype6-dev libfontconfig1 libfontconfig1-dev + - run: sudo apt-get purge -y snmp pollinate libpq-dev postgresql-client powershell ruby-full sphinxsearch subversion azure-cli + - run: sudo apt-get purge -y $(dpkg-query -W | grep dotnet | awk '{print $1}') + - run: sudo apt-get autoremove -y >/dev/null 2>&1 + - run: sudo apt-get autoclean -y >/dev/null 2>&1 + - name: "KVM setup" + run: |- + VIRTUALIZATION_SUPPORT=$(grep -E -q 'vmx|svm' /proc/cpuinfo && echo yes || echo no) + echo ${VIRTUALIZATION_SUPPORT} + if [ "${VIRTUALIZATION_SUPPORT}" != "yes" ]; then + exit 1 + fi + sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils + kvm-ok + sudo adduser `id -un` libvirt + sudo adduser `id -un` kvm + virsh list --all + sudo ls -la /var/run/libvirt/libvirt-sock + sudo chmod 777 /var/run/libvirt/libvirt-sock + sudo ls -la /var/run/libvirt/libvirt-sock + ls -l /dev/kvm + sudo rmmod kvm_amd + sudo rmmod kvm + sudo modprobe -a kvm + sudo modprobe -a kvm_amd + - run: |- + make -C test/e2e setup + ln -s $(pwd)/test/e2e/bin/kubectl-1.28.3 test/e2e/bin/kubectl + - run: |- + make -C test/e2e launch-minikube PROFILE=profile1 + make -C test/e2e launch-minikube PROFILE=profile2 + - name: "Deploy primary Rook/Ceph cluster" + run: |- + ./test/e2e/bin/kubectl config use-context profile1 + make -C test/e2e launch-single-rook-ceph PROFILE=profile1 + make -C test/e2e setup-single-components + - name: "Deploy secondary Rook/Ceph cluster" + run: |- + ./test/e2e/bin/kubectl config use-context profile2 + make -C test/e2e launch-single-rook-ceph PROFILE=profile1 + make -C test/e2e setup-single-components diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6179ae5d..f9328040 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,7 +12,7 @@ on: - "main" jobs: - build: + test: runs-on: "ubuntu-22.04" env: MINIKUBE_HOME: ${{ github.workspace }}/test/e2e/bin @@ -54,41 +54,5 @@ jobs: sudo rmmod kvm sudo modprobe -a kvm sudo modprobe -a kvm_amd - - run: |- - make -C test/e2e setup - ln -s $(pwd)/test/e2e/bin/kubectl-1.28.3 test/e2e/bin/kubectl - - run: |- - make -C test/e2e launch-minikube PROFILE=profile1 - make -C test/e2e launch-minikube PROFILE=profile2 - - name: "deploy components" - id: deploy-components - run: |- - ./test/e2e/bin/kubectl --context profile2 create ns test - ./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/nginx.yaml - ./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/svcnp.yaml - ./test/e2e/bin/minikube -p profile2 service list - SERVICE=$(./test/e2e/bin/minikube -p profile2 service list | grep my-nginx-np | cut -d "|" -f 5 | tr -d "[[:space:]]" | sed 's/\//\\\//g') - echo "service name: ${SERVICE}" - echo "service-name=${SERVICE}" >> "$GITHUB_OUTPUT" - ./test/e2e/bin/kubectl wait deploy -n test --timeout=2m --for=condition=available my-nginx - - name: "Run and wait curl job" - run: |- - sed -i "s/placeholder/${{ steps.deploy-components.outputs.service-name }}/" test/e2e/job.yaml - ./test/e2e/bin/kubectl --context profile1 apply -f test/e2e/job.yaml - ./test/e2e/bin/kubectl --context profile1 wait job --for=condition=complete --timeout=2m curl - PODS=$(./test/e2e/bin/kubectl --context profile1 get pod | grep curl | awk '{print $1}') - for POD in ${PODS}; do - echo "POD: ${POD}" - ./test/e2e/bin/kubectl --context profile1 logs ${POD} - done - - name: "Run existing e2e test" - run: |- - ./test/e2e/bin/kubectl config use-context profile1 - make -C test/e2e launch-rook-ceph - make -C test/e2e setup-components - make -C test/e2e do_test - - name: "Deploy secondary Rook/Ceph cluster" - run: |- - ./test/e2e/bin/kubectl config use-context profile2 - make -C test/e2e launch-rook-ceph - make -C test/e2e setup-components + - run: make -C test/e2e setup + - run: make -C test/e2e test diff --git a/.github/workflows/inter-cluster-connection.yaml b/.github/workflows/inter-cluster-connection.yaml new file mode 100644 index 00000000..37b05686 --- /dev/null +++ b/.github/workflows/inter-cluster-connection.yaml @@ -0,0 +1,83 @@ +name: "inter-cluster connection" +on: + pull_request: + paths-ignore: + - "**/*.md" + - "CODEOWNERS" + push: + paths-ignore: + - "**/*.md" + - "CODEOWNERS" + branches: + - "main" + +jobs: + test: + runs-on: "ubuntu-22.04" + env: + MINIKUBE_HOME: ${{ github.workspace }}/test/e2e/bin + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: "go.mod" + - run: env | grep MINIKUBE_HOME + - run: sudo apt-get update + - run: sudo apt-get purge -y apport apport-symptoms fwupd nano netplan.io popularity-contest unattended-upgrades update-manager-core snapd + - run: sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + - run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup /usr/lib/jvm || true + - run: sudo apt-get purge -y $(dpkg-query -W | grep nginx | awk '{print $1}') + - run: sudo apt-get purge -y aria2 ansible azure-cli shellcheck rpm xorriso zsync firefox gfortran-9 google-chrome-stable google-cloud-sdk + - run: sudo apt-get purge -y imagemagick libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional mercurial apt-transport-https + - run: sudo apt-get purge -y mono-complete unixodbc-dev yarn chrpath libxft-dev libfreetype6-dev libfontconfig1 libfontconfig1-dev + - run: sudo apt-get purge -y snmp pollinate libpq-dev postgresql-client powershell ruby-full sphinxsearch subversion azure-cli + - run: sudo apt-get purge -y $(dpkg-query -W | grep dotnet | awk '{print $1}') + - run: sudo apt-get autoremove -y >/dev/null 2>&1 + - run: sudo apt-get autoclean -y >/dev/null 2>&1 + - name: "KVM setup" + run: |- + VIRTUALIZATION_SUPPORT=$(grep -E -q 'vmx|svm' /proc/cpuinfo && echo yes || echo no) + echo ${VIRTUALIZATION_SUPPORT} + if [ "${VIRTUALIZATION_SUPPORT}" != "yes" ]; then + exit 1 + fi + sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils + kvm-ok + sudo adduser `id -un` libvirt + sudo adduser `id -un` kvm + virsh list --all + sudo ls -la /var/run/libvirt/libvirt-sock + sudo chmod 777 /var/run/libvirt/libvirt-sock + sudo ls -la /var/run/libvirt/libvirt-sock + ls -l /dev/kvm + sudo rmmod kvm_amd + sudo rmmod kvm + sudo modprobe -a kvm + sudo modprobe -a kvm_amd + - run: |- + make -C test/e2e setup + ln -s $(pwd)/test/e2e/bin/kubectl-1.28.3 test/e2e/bin/kubectl + - run: |- + make -C test/e2e launch-minikube PROFILE=profile1 + make -C test/e2e launch-minikube PROFILE=profile2 + - name: "deploy components" + id: deploy-components + run: |- + ./test/e2e/bin/kubectl --context profile2 create ns test + ./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/nginx.yaml + ./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/svcnp.yaml + ./test/e2e/bin/minikube -p profile2 service list + SERVICE=$(./test/e2e/bin/minikube -p profile2 service list | grep my-nginx-np | cut -d "|" -f 5 | tr -d "[[:space:]]" | sed 's/\//\\\//g') + echo "service name: ${SERVICE}" + echo "service-name=${SERVICE}" >> "$GITHUB_OUTPUT" + ./test/e2e/bin/kubectl wait deploy -n test --timeout=2m --for=condition=available my-nginx + - name: "Run and wait curl job" + run: |- + sed -i "s/placeholder/${{ steps.deploy-components.outputs.service-name }}/" test/e2e/job.yaml + ./test/e2e/bin/kubectl --context profile1 apply -f test/e2e/job.yaml + ./test/e2e/bin/kubectl --context profile1 wait job --for=condition=complete --timeout=2m curl + PODS=$(./test/e2e/bin/kubectl --context profile1 get pod | grep curl | awk '{print $1}') + for POD in ${PODS}; do + echo "POD: ${POD}" + ./test/e2e/bin/kubectl --context profile1 logs ${POD} + done diff --git a/test/e2e/Makefile b/test/e2e/Makefile index fd8f1d8c..4c9fea8f 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -2,7 +2,6 @@ include ../../versions.mk SHELL := /bin/bash BINDIR := $(shell pwd)/bin -TMPDIR := $(shell pwd)/tmp CURL := curl -sSLf GINKGO := $(BINDIR)/ginkgo-$(GINKGO_VERSION) GINKGO_FLAGS := @@ -16,8 +15,8 @@ NODE_NAME := minikube-worker SUDO := sudo DD := dd LOSETUP := losetup -LOOP_FILE := $(TMPDIR)/loop.img -LOOP_FILE2 := $(TMPDIR)/loop2.img +LOOP_FILE := loop.img +LOOP_FILE2 := loop2.img CNI_PLUGIN_INSTALL_DIR := /opt/cni/bin POLLING_INTERVAL := 1 TIMEOUT_SECS := 180 @@ -26,10 +25,10 @@ PROFILE ?= minikube export MINIKUBE_HOME define loop-dev -$(shell $(LOSETUP) | grep $(LOOP_FILE) | cut -d ' ' -f 1) +$(shell $(MINIKUBE) ssh -- $(LOSETUP) | grep $(LOOP_FILE) | cut -d ' ' -f 1) endef define loop-dev2 -$(shell $(LOSETUP) | grep $(LOOP_FILE2) | cut -d ' ' -f 1) +$(shell $(MINIKUBE) ssh -- $(LOSETUP) | grep $(LOOP_FILE2) | cut -d ' ' -f 1) endef .PHONY: setup @@ -55,9 +54,6 @@ clean: $(BINDIR): mkdir -p $@ -$(TMPDIR): - mkdir -p $@ - $(GINKGO): | $(BINDIR) GOBIN=$(BINDIR) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION) mv $(BINDIR)/ginkgo $@ @@ -114,26 +110,19 @@ delete-minikube: $(SUDO) rm -rf /tmp/juju-* .PHONY: create-loop-dev -create-loop-dev: | $(TMPDIR) - $(shell \ - if [ -z "$(loop-dev)" ]; then \ - $(DD) if=/dev/zero of=$(LOOP_FILE) bs=1G seek=32 count=0; \ - $(SUDO) $(LOSETUP) $$($(SUDO) $(LOSETUP) -f) $(LOOP_FILE); \ - fi; \ - if [ -z "$(loop-dev2)" ]; then \ - $(DD) if=/dev/zero of=$(LOOP_FILE2) bs=1G seek=32 count=0; \ - $(SUDO) $(LOSETUP) $$($(SUDO) $(LOSETUP) -f) $(LOOP_FILE2); \ - fi\ - ) +create-loop-dev: + $(MINIKUBE) -p $(PROFILE) ssh -- $(DD) if=/dev/zero of=$(LOOP_FILE) bs=1G seek=32 count=0 + $(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) $(LOSETUP) /dev/loop0 $(LOOP_FILE) + $(MINIKUBE) -p $(PROFILE) ssh $(DD) if=/dev/zero of=$(LOOP_FILE2) bs=1G seek=32 count=0 + $(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) $(LOSETUP) /dev/loop1 $(LOOP_FILE2) + $(MINIKUBE) -p $(PROFILE) ssh -- lsblk # suppress "make[1]: Nothing to be done for 'create-loop-dev'." : .PHONY: delete-loop-dev -delete-loop-dev: | $(TMPDIR) - $(SUDO) $(LOSETUP) -d $(loop-dev) || : - $(SUDO) rm -f $(LOOP_FILE) - $(SUDO) $(LOSETUP) -d $(loop-dev2) || : - $(SUDO) rm -f $(LOOP_FILE2) +delete-loop-dev: + $(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) $(LOSETUP) -D || : + $(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) rm -f loop* .PHONY: wait-deploy-ready wait-deploy-ready: NS= @@ -172,9 +161,40 @@ launch-rook-ceph: create-loop-dev --create-namespace --namespace $(CEPH_CLUSTER2_NAMESPACE) -f testdata/values-cluster.yaml \ --set cephClusterSpec.dataDirHostPath=/var/lib/rook2 \ --wait rook-ceph-cluster2 rook-ceph-cluster + free -h + df -h $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator + free -h + df -h $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0 + free -h + df -h $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER2_NAMESPACE) DEPLOY=rook-ceph-osd-0 + free -h + df -h + +.PHONY: launch-single-rook-ceph +launch-single-rook-ceph: create-loop-dev + $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ + --create-namespace --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values.yaml --wait \ + rook-ceph rook-ceph + sed \ + -e "s%{LOOP_DEV}%$(loop-dev)%" \ + -e "s%{LOOP_DEV2}%$(loop-dev2)%" \ + -e "s%{NODE_NAME}%$(NODE_NAME)%" \ + testdata/persistentvolumes-template.yaml \ + > testdata/persistentvolumes.yaml + $(KUBECTL) apply -f testdata/persistentvolumes.yaml + $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ + --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml --wait rook-ceph-cluster rook-ceph-cluster + free -h + df -h + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator + free -h + df -h + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0 + free -h + df -h .PHONY: delete-rook-ceph delete-rook-ceph: @@ -199,6 +219,16 @@ setup-components: $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 ../../charts/mantle/ --wait +.PHONY: setup-single-components +setup-single-components: + $(MAKE) -C ../.. docker-build + # We can't use `minikube image load` due to a minikube's problem. + # ref. https://github.com/kubernetes/minikube/issues/17785 + docker save controller:latest | $(SUDO) ctr -n k8s.io images import - + + $(HELM) upgrade --install mantle-cluster-wide ../../charts/mantle-cluster-wide/ --wait + $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait + .PHONY: delete-components delete-components: $(HELM) uninstall --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 --wait || true