Skip to content

Commit

Permalink
split test
Browse files Browse the repository at this point in the history
  • Loading branch information
peng225 committed Jun 14, 2024
1 parent 572d58c commit 93dc7f6
Show file tree
Hide file tree
Showing 4 changed files with 212 additions and 63 deletions.
72 changes: 72 additions & 0 deletions .github/workflows/deploy-rook-ceph-on-two-k8s-clusters.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: "two k8s cluster"
on:
pull_request:
paths-ignore:
- "**/*.md"
- "CODEOWNERS"
push:
paths-ignore:
- "**/*.md"
- "CODEOWNERS"
branches:
- "main"

jobs:
test:
runs-on: "ubuntu-22.04"
env:
MINIKUBE_HOME: ${{ github.workspace }}/test/e2e/bin
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: "go.mod"
- run: env | grep MINIKUBE_HOME
- run: sudo apt-get update
- run: sudo apt-get purge -y apport apport-symptoms fwupd nano netplan.io popularity-contest unattended-upgrades update-manager-core snapd
- run: sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
- run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup /usr/lib/jvm || true
- run: sudo apt-get purge -y $(dpkg-query -W | grep nginx | awk '{print $1}')
- run: sudo apt-get purge -y aria2 ansible azure-cli shellcheck rpm xorriso zsync firefox gfortran-9 google-chrome-stable google-cloud-sdk
- run: sudo apt-get purge -y imagemagick libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional mercurial apt-transport-https
- run: sudo apt-get purge -y mono-complete unixodbc-dev yarn chrpath libxft-dev libfreetype6-dev libfontconfig1 libfontconfig1-dev
- run: sudo apt-get purge -y snmp pollinate libpq-dev postgresql-client powershell ruby-full sphinxsearch subversion azure-cli
- run: sudo apt-get purge -y $(dpkg-query -W | grep dotnet | awk '{print $1}')
- run: sudo apt-get autoremove -y >/dev/null 2>&1
- run: sudo apt-get autoclean -y >/dev/null 2>&1
- name: "KVM setup"
run: |-
VIRTUALIZATION_SUPPORT=$(grep -E -q 'vmx|svm' /proc/cpuinfo && echo yes || echo no)
echo ${VIRTUALIZATION_SUPPORT}
if [ "${VIRTUALIZATION_SUPPORT}" != "yes" ]; then
exit 1
fi
sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils
kvm-ok
sudo adduser `id -un` libvirt
sudo adduser `id -un` kvm
virsh list --all
sudo ls -la /var/run/libvirt/libvirt-sock
sudo chmod 777 /var/run/libvirt/libvirt-sock
sudo ls -la /var/run/libvirt/libvirt-sock
ls -l /dev/kvm
sudo rmmod kvm_amd
sudo rmmod kvm
sudo modprobe -a kvm
sudo modprobe -a kvm_amd
- run: |-
make -C test/e2e setup
ln -s $(pwd)/test/e2e/bin/kubectl-1.28.3 test/e2e/bin/kubectl
- run: |-
make -C test/e2e launch-minikube PROFILE=profile1
make -C test/e2e launch-minikube PROFILE=profile2
- name: "Deploy primary Rook/Ceph cluster"
run: |-
./test/e2e/bin/kubectl config use-context profile1
make -C test/e2e launch-single-rook-ceph PROFILE=profile1
make -C test/e2e setup-single-components
- name: "Deploy secondary Rook/Ceph cluster"
run: |-
./test/e2e/bin/kubectl config use-context profile2
make -C test/e2e launch-single-rook-ceph PROFILE=profile1
make -C test/e2e setup-single-components
42 changes: 3 additions & 39 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
- "main"

jobs:
build:
test:
runs-on: "ubuntu-22.04"
env:
MINIKUBE_HOME: ${{ github.workspace }}/test/e2e/bin
Expand Down Expand Up @@ -54,41 +54,5 @@ jobs:
sudo rmmod kvm
sudo modprobe -a kvm
sudo modprobe -a kvm_amd
- run: |-
make -C test/e2e setup
ln -s $(pwd)/test/e2e/bin/kubectl-1.28.3 test/e2e/bin/kubectl
- run: |-
make -C test/e2e launch-minikube PROFILE=profile1
make -C test/e2e launch-minikube PROFILE=profile2
- name: "deploy components"
id: deploy-components
run: |-
./test/e2e/bin/kubectl --context profile2 create ns test
./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/nginx.yaml
./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/svcnp.yaml
./test/e2e/bin/minikube -p profile2 service list
SERVICE=$(./test/e2e/bin/minikube -p profile2 service list | grep my-nginx-np | cut -d "|" -f 5 | tr -d "[[:space:]]" | sed 's/\//\\\//g')
echo "service name: ${SERVICE}"
echo "service-name=${SERVICE}" >> "$GITHUB_OUTPUT"
./test/e2e/bin/kubectl wait deploy -n test --timeout=2m --for=condition=available my-nginx
- name: "Run and wait curl job"
run: |-
sed -i "s/placeholder/${{ steps.deploy-components.outputs.service-name }}/" test/e2e/job.yaml
./test/e2e/bin/kubectl --context profile1 apply -f test/e2e/job.yaml
./test/e2e/bin/kubectl --context profile1 wait job --for=condition=complete --timeout=2m curl
PODS=$(./test/e2e/bin/kubectl --context profile1 get pod | grep curl | awk '{print $1}')
for POD in ${PODS}; do
echo "POD: ${POD}"
./test/e2e/bin/kubectl --context profile1 logs ${POD}
done
- name: "Run existing e2e test"
run: |-
./test/e2e/bin/kubectl config use-context profile1
make -C test/e2e launch-rook-ceph
make -C test/e2e setup-components
make -C test/e2e do_test
- name: "Deploy secondary Rook/Ceph cluster"
run: |-
./test/e2e/bin/kubectl config use-context profile2
make -C test/e2e launch-rook-ceph
make -C test/e2e setup-components
- run: make -C test/e2e setup
- run: make -C test/e2e test
83 changes: 83 additions & 0 deletions .github/workflows/inter-cluster-connection.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
name: "inter-cluster connection"
on:
pull_request:
paths-ignore:
- "**/*.md"
- "CODEOWNERS"
push:
paths-ignore:
- "**/*.md"
- "CODEOWNERS"
branches:
- "main"

jobs:
test:
runs-on: "ubuntu-22.04"
env:
MINIKUBE_HOME: ${{ github.workspace }}/test/e2e/bin
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: "go.mod"
- run: env | grep MINIKUBE_HOME
- run: sudo apt-get update
- run: sudo apt-get purge -y apport apport-symptoms fwupd nano netplan.io popularity-contest unattended-upgrades update-manager-core snapd
- run: sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
- run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup /usr/lib/jvm || true
- run: sudo apt-get purge -y $(dpkg-query -W | grep nginx | awk '{print $1}')
- run: sudo apt-get purge -y aria2 ansible azure-cli shellcheck rpm xorriso zsync firefox gfortran-9 google-chrome-stable google-cloud-sdk
- run: sudo apt-get purge -y imagemagick libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional mercurial apt-transport-https
- run: sudo apt-get purge -y mono-complete unixodbc-dev yarn chrpath libxft-dev libfreetype6-dev libfontconfig1 libfontconfig1-dev
- run: sudo apt-get purge -y snmp pollinate libpq-dev postgresql-client powershell ruby-full sphinxsearch subversion azure-cli
- run: sudo apt-get purge -y $(dpkg-query -W | grep dotnet | awk '{print $1}')
- run: sudo apt-get autoremove -y >/dev/null 2>&1
- run: sudo apt-get autoclean -y >/dev/null 2>&1
- name: "KVM setup"
run: |-
VIRTUALIZATION_SUPPORT=$(grep -E -q 'vmx|svm' /proc/cpuinfo && echo yes || echo no)
echo ${VIRTUALIZATION_SUPPORT}
if [ "${VIRTUALIZATION_SUPPORT}" != "yes" ]; then
exit 1
fi
sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils
kvm-ok
sudo adduser `id -un` libvirt
sudo adduser `id -un` kvm
virsh list --all
sudo ls -la /var/run/libvirt/libvirt-sock
sudo chmod 777 /var/run/libvirt/libvirt-sock
sudo ls -la /var/run/libvirt/libvirt-sock
ls -l /dev/kvm
sudo rmmod kvm_amd
sudo rmmod kvm
sudo modprobe -a kvm
sudo modprobe -a kvm_amd
- run: |-
make -C test/e2e setup
ln -s $(pwd)/test/e2e/bin/kubectl-1.28.3 test/e2e/bin/kubectl
- run: |-
make -C test/e2e launch-minikube PROFILE=profile1
make -C test/e2e launch-minikube PROFILE=profile2
- name: "deploy components"
id: deploy-components
run: |-
./test/e2e/bin/kubectl --context profile2 create ns test
./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/nginx.yaml
./test/e2e/bin/kubectl --context profile2 apply -f test/e2e/svcnp.yaml
./test/e2e/bin/minikube -p profile2 service list
SERVICE=$(./test/e2e/bin/minikube -p profile2 service list | grep my-nginx-np | cut -d "|" -f 5 | tr -d "[[:space:]]" | sed 's/\//\\\//g')
echo "service name: ${SERVICE}"
echo "service-name=${SERVICE}" >> "$GITHUB_OUTPUT"
./test/e2e/bin/kubectl wait deploy -n test --timeout=2m --for=condition=available my-nginx
- name: "Run and wait curl job"
run: |-
sed -i "s/placeholder/${{ steps.deploy-components.outputs.service-name }}/" test/e2e/job.yaml
./test/e2e/bin/kubectl --context profile1 apply -f test/e2e/job.yaml
./test/e2e/bin/kubectl --context profile1 wait job --for=condition=complete --timeout=2m curl
PODS=$(./test/e2e/bin/kubectl --context profile1 get pod | grep curl | awk '{print $1}')
for POD in ${PODS}; do
echo "POD: ${POD}"
./test/e2e/bin/kubectl --context profile1 logs ${POD}
done
78 changes: 54 additions & 24 deletions test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ include ../../versions.mk

SHELL := /bin/bash
BINDIR := $(shell pwd)/bin
TMPDIR := $(shell pwd)/tmp
CURL := curl -sSLf
GINKGO := $(BINDIR)/ginkgo-$(GINKGO_VERSION)
GINKGO_FLAGS :=
Expand All @@ -16,8 +15,8 @@ NODE_NAME := minikube-worker
SUDO := sudo
DD := dd
LOSETUP := losetup
LOOP_FILE := $(TMPDIR)/loop.img
LOOP_FILE2 := $(TMPDIR)/loop2.img
LOOP_FILE := loop.img
LOOP_FILE2 := loop2.img
CNI_PLUGIN_INSTALL_DIR := /opt/cni/bin
POLLING_INTERVAL := 1
TIMEOUT_SECS := 180
Expand All @@ -26,10 +25,10 @@ PROFILE ?= minikube
export MINIKUBE_HOME

define loop-dev
$(shell $(LOSETUP) | grep $(LOOP_FILE) | cut -d ' ' -f 1)
$(shell $(MINIKUBE) ssh -- $(LOSETUP) | grep $(LOOP_FILE) | cut -d ' ' -f 1)
endef
define loop-dev2
$(shell $(LOSETUP) | grep $(LOOP_FILE2) | cut -d ' ' -f 1)
$(shell $(MINIKUBE) ssh -- $(LOSETUP) | grep $(LOOP_FILE2) | cut -d ' ' -f 1)
endef

.PHONY: setup
Expand All @@ -55,9 +54,6 @@ clean:
$(BINDIR):
mkdir -p $@

$(TMPDIR):
mkdir -p $@

$(GINKGO): | $(BINDIR)
GOBIN=$(BINDIR) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
mv $(BINDIR)/ginkgo $@
Expand Down Expand Up @@ -114,26 +110,19 @@ delete-minikube:
$(SUDO) rm -rf /tmp/juju-*

.PHONY: create-loop-dev
create-loop-dev: | $(TMPDIR)
$(shell \
if [ -z "$(loop-dev)" ]; then \
$(DD) if=/dev/zero of=$(LOOP_FILE) bs=1G seek=32 count=0; \
$(SUDO) $(LOSETUP) $$($(SUDO) $(LOSETUP) -f) $(LOOP_FILE); \
fi; \
if [ -z "$(loop-dev2)" ]; then \
$(DD) if=/dev/zero of=$(LOOP_FILE2) bs=1G seek=32 count=0; \
$(SUDO) $(LOSETUP) $$($(SUDO) $(LOSETUP) -f) $(LOOP_FILE2); \
fi\
)
create-loop-dev:
$(MINIKUBE) -p $(PROFILE) ssh -- $(DD) if=/dev/zero of=$(LOOP_FILE) bs=1G seek=32 count=0
$(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) $(LOSETUP) /dev/loop0 $(LOOP_FILE)
$(MINIKUBE) -p $(PROFILE) ssh $(DD) if=/dev/zero of=$(LOOP_FILE2) bs=1G seek=32 count=0
$(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) $(LOSETUP) /dev/loop1 $(LOOP_FILE2)
$(MINIKUBE) -p $(PROFILE) ssh -- lsblk
# suppress "make[1]: Nothing to be done for 'create-loop-dev'."
:

.PHONY: delete-loop-dev
delete-loop-dev: | $(TMPDIR)
$(SUDO) $(LOSETUP) -d $(loop-dev) || :
$(SUDO) rm -f $(LOOP_FILE)
$(SUDO) $(LOSETUP) -d $(loop-dev2) || :
$(SUDO) rm -f $(LOOP_FILE2)
delete-loop-dev:
$(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) $(LOSETUP) -D || :
$(MINIKUBE) -p $(PROFILE) ssh -- $(SUDO) rm -f loop*

.PHONY: wait-deploy-ready
wait-deploy-ready: NS=
Expand Down Expand Up @@ -172,9 +161,40 @@ launch-rook-ceph: create-loop-dev
--create-namespace --namespace $(CEPH_CLUSTER2_NAMESPACE) -f testdata/values-cluster.yaml \
--set cephClusterSpec.dataDirHostPath=/var/lib/rook2 \
--wait rook-ceph-cluster2 rook-ceph-cluster
free -h
df -h
$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator
free -h
df -h
$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0
free -h
df -h
$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER2_NAMESPACE) DEPLOY=rook-ceph-osd-0
free -h
df -h

.PHONY: launch-single-rook-ceph
launch-single-rook-ceph: create-loop-dev
$(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \
--create-namespace --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values.yaml --wait \
rook-ceph rook-ceph
sed \
-e "s%{LOOP_DEV}%$(loop-dev)%" \
-e "s%{LOOP_DEV2}%$(loop-dev2)%" \
-e "s%{NODE_NAME}%$(NODE_NAME)%" \
testdata/persistentvolumes-template.yaml \
> testdata/persistentvolumes.yaml
$(KUBECTL) apply -f testdata/persistentvolumes.yaml
$(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \
--namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml --wait rook-ceph-cluster rook-ceph-cluster
free -h
df -h
$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator
free -h
df -h
$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0
free -h
df -h

.PHONY: delete-rook-ceph
delete-rook-ceph:
Expand All @@ -199,6 +219,16 @@ setup-components:
$(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait
$(HELM) upgrade --install --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 ../../charts/mantle/ --wait

.PHONY: setup-single-components
setup-single-components:
$(MAKE) -C ../.. docker-build
# We can't use `minikube image load` due to a minikube's problem.
# ref. https://github.com/kubernetes/minikube/issues/17785
docker save controller:latest | $(SUDO) ctr -n k8s.io images import -

$(HELM) upgrade --install mantle-cluster-wide ../../charts/mantle-cluster-wide/ --wait
$(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait

.PHONY: delete-components
delete-components:
$(HELM) uninstall --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 --wait || true
Expand Down

0 comments on commit 93dc7f6

Please sign in to comment.