Merge pull request #22 from cybozu-go/use-kvm-driver-for-minikube

use kvm2 driver for minikube
cybozu-go · Jun 26, 2024 · 2e47168 · 2e47168
2 parents 3e71c96 + ac5c827
commit 2e47168
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 100 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -13,26 +13,45 @@ on:
 
 jobs:
   build:
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version-file: "go.mod"
-      - run: sudo apt update
-      - run: sudo apt purge -y apport apport-symptoms fwupd nano netplan.io popularity-contest unattended-upgrades update-manager-core snapd
+      - run: sudo apt-get update
+      - run: sudo apt-get purge -y apport apport-symptoms fwupd nano netplan.io popularity-contest unattended-upgrades update-manager-core snapd
       - run: sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
       - run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup /usr/lib/jvm || true
-      - run: sudo apt purge -y $(dpkg-query -W | grep nginx | awk '{print $1}')
-      - run: sudo apt purge -y aria2 ansible azure-cli shellcheck rpm xorriso zsync esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable google-cloud-sdk
-      - run: sudo apt purge -y imagemagick libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional mercurial apt-transport-https
-      - run: sudo apt purge -y mono-complete unixodbc-dev yarn chrpath libxft-dev libfreetype6-dev libfontconfig1 libfontconfig1-dev
-      - run: sudo apt purge -y snmp pollinate libpq-dev postgresql-client powershell ruby-full sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable
-      - run: sudo apt purge -y $(dpkg-query -W | grep dotnet | awk '{print $1}')
-      - run: sudo apt autoremove -y >/dev/null 2>&1
-      - run: sudo apt autoclean -y >/dev/null 2>&1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - run: sudo apt-get purge -y $(dpkg-query -W | grep nginx | awk '{print $1}')
+      - run: sudo apt-get purge -y aria2 ansible azure-cli shellcheck rpm xorriso zsync firefox gfortran-9 google-chrome-stable google-cloud-sdk
+      - run: sudo apt-get purge -y imagemagick libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional mercurial apt-transport-https
+      - run: sudo apt-get purge -y mono-complete unixodbc-dev yarn chrpath libxft-dev libfreetype6-dev libfontconfig1 libfontconfig1-dev
+      - run: sudo apt-get purge -y snmp pollinate libpq-dev postgresql-client powershell ruby-full sphinxsearch subversion azure-cli
+      - run: sudo apt-get purge -y $(dpkg-query -W | grep dotnet | awk '{print $1}')
+      - run: sudo apt-get autoremove -y >/dev/null 2>&1
+      - run: sudo apt-get autoclean -y >/dev/null 2>&1
+      - name: "KVM setup"
+        run: |-
+             VIRTUALIZATION_SUPPORT=$(grep -E -q 'vmx|svm' /proc/cpuinfo && echo yes || echo no)
+             echo ${VIRTUALIZATION_SUPPORT}
+             if [ "${VIRTUALIZATION_SUPPORT}" != "yes" ]; then
+               echo "CPU does not support the virtualization feature."
+               exit 1
+             fi
+             sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils
+             kvm-ok
+             sudo adduser `id -un` libvirt
+             sudo adduser `id -un` kvm
+             virsh list --all
+             sudo ls -la /var/run/libvirt/libvirt-sock
+             sudo chmod 777 /var/run/libvirt/libvirt-sock
+             sudo ls -la /var/run/libvirt/libvirt-sock
+             ls -l /dev/kvm
+             sudo rmmod kvm_amd
+             sudo rmmod kvm
+             sudo modprobe -a kvm
+             sudo modprobe -a kvm_amd
       - name: cache go dependencies
         uses: actions/cache@v4
         with:

diff --git a/test/e2e/Makefile b/test/e2e/Makefile
@@ -2,7 +2,6 @@ include ../../versions.mk
 
 SHELL := /bin/bash
 BINDIR := $(shell pwd)/bin
-TMPDIR := $(shell pwd)/tmp
 CURL := curl -sSLf
 GINKGO := $(BINDIR)/ginkgo-$(GINKGO_VERSION)
 GINKGO_FLAGS :=
@@ -16,21 +15,16 @@ NODE_NAME := minikube-worker
 SUDO := sudo
 DD := dd
 LOSETUP := losetup
-LOOP_FILE := $(TMPDIR)/loop.img
-LOOP_FILE2 := $(TMPDIR)/loop2.img
+LOOP_FILE := loop.img
+LOOP_FILE2 := loop2.img
 CNI_PLUGIN_INSTALL_DIR := /opt/cni/bin
 POLLING_INTERVAL := 1
 TIMEOUT_SECS := 180
+LOOP_DEV := /dev/loop0
+LOOP_DEV2 := /dev/loop1
 
 export MINIKUBE_HOME
 
-define loop-dev
-$(shell $(LOSETUP) | grep $(LOOP_FILE) | cut -d ' ' -f 1)
-endef
-define loop-dev2
-$(shell $(LOSETUP) | grep $(LOOP_FILE2) | cut -d ' ' -f 1)
-endef
-
 .PHONY: setup
 setup:
 	$(MAKE) $(GINKGO)
@@ -41,22 +35,18 @@ setup:
 .PHONY: test
 test:
 	$(MAKE) launch-minikube
+	$(MAKE) image-build
 	$(MAKE) launch-rook-ceph
 	$(MAKE) setup-components
 	$(MAKE) do_test
 
 .PHONY: clean
 clean:
-	$(MAKE) delete-components
-	$(MAKE) delete-rook-ceph
 	$(MAKE) delete-minikube
 
 $(BINDIR):
 	mkdir -p $@
 
-$(TMPDIR):
-	mkdir -p $@
-
 $(GINKGO): | $(BINDIR)
 	GOBIN=$(BINDIR) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
 	mv $(BINDIR)/ginkgo $@
@@ -66,25 +56,9 @@ $(KUBECTL): | $(BINDIR)
 	chmod a+x $@
 
 $(MINIKUBE): | $(BINDIR)
-	$(SUDO) apt update
-	$(SUDO) apt install -y conntrack socat
 	$(CURL) -o $@ https://github.com/kubernetes/minikube/releases/download/$(MINIKUBE_VERSION)/minikube-linux-amd64
 	chmod a+x $@
 
-	$(CURL) -o cri-dockerd.deb https://github.com/Mirantis/cri-dockerd/releases/download/$(CRI_DOCKERD_VERSION)/cri-dockerd_$(CRI_DOCKERD_VERSION:v%=%).3-0.ubuntu-focal_amd64.deb
-	$(SUDO) dpkg -i cri-dockerd.deb
-	rm -f cri-dockerd.deb
-
-	$(CURL) -o crictl.tar.gz https://github.com/kubernetes-sigs/cri-tools/releases/download/$(CRICTL_VERSION)/crictl-$(CRICTL_VERSION)-linux-amd64.tar.gz
-	$(SUDO) tar zxvf crictl.tar.gz -C /usr/local/bin
-	rm -f crictl.tar.gz
-	$(SUDO) sysctl fs.protected_regular=0
-
-	$(CURL) -o cni-plugins.tgz https://github.com/containernetworking/plugins/releases/download/$(CNI_PLUGINS_VERSION)/cni-plugins-linux-amd64-$(CNI_PLUGINS_VERSION).tgz
-	$(SUDO) mkdir -p $(CNI_PLUGIN_INSTALL_DIR)
-	$(SUDO) tar -xf cni-plugins.tgz -C $(CNI_PLUGIN_INSTALL_DIR)
-	rm -f cni-plugins.tgz
-
 $(HELM): | $(BINDIR)
 	$(CURL) https://get.helm.sh/helm-v$(HELM_VERSION)-linux-amd64.tar.gz \
 		| tar xvz -C $(BINDIR) --strip-components 1 linux-amd64/helm
@@ -95,47 +69,26 @@ launch-minikube:
 	kubectl get pod >/dev/null 2>&1; \
 	RET=$$? ; \
 	if [ $$RET -eq 0 ] ; then exit; fi ; \
-	$(SUDO) -E $(MINIKUBE) start \
+	$(MINIKUBE) start \
 		--kubernetes-version="v$(KUBERNETES_VERSION)" \
-		--driver=none \
+		--driver=kvm2 \
 		--memory 6g \
 		--cpus=2 \
-		--addons ingress \
-		--cni=calico \
 		--extra-config=kubeadm.node-name=$(NODE_NAME) \
 		--extra-config=kubelet.hostname-override=$(NODE_NAME)
-	$(SUDO) chown -R $$USER $$HOME/.kube $(MINIKUBE_HOME)/.minikube
-	$(SUDO) chmod -R a+r $$HOME/.kube $(MINIKUBE_HOME)/.minikube
-	$(SUDO) find $(MINIKUBE_HOME)/.minikube -name id_rsa -exec chmod 600 {} ';'
 
 .PHONY: delete-minikube
 delete-minikube:
-	$(SUDO) -E $(MINIKUBE) stop || true
-	$(SUDO) -E $(MINIKUBE) delete || true
-	# Is it really OK to remove these files?
-	$(SUDO) rm -rf /tmp/juju-*
+	$(MINIKUBE) stop || true
+	$(MINIKUBE) delete || true
 
 .PHONY: create-loop-dev
-create-loop-dev: | $(TMPDIR)
-	$(shell \
-		if [ -z "$(loop-dev)" ]; then \
-			$(DD) if=/dev/zero of=$(LOOP_FILE) bs=1G seek=32 count=0; \
-			$(SUDO) $(LOSETUP) $$($(SUDO) $(LOSETUP) -f) $(LOOP_FILE); \
-		fi; \
-		if [ -z "$(loop-dev2)" ]; then \
-			$(DD) if=/dev/zero of=$(LOOP_FILE2) bs=1G seek=32 count=0; \
-			$(SUDO) $(LOSETUP) $$($(SUDO) $(LOSETUP) -f) $(LOOP_FILE2); \
-		fi\
-	)
-# suppress "make[1]: Nothing to be done for 'create-loop-dev'."
-	:
-
-.PHONY: delete-loop-dev
-delete-loop-dev: | $(TMPDIR)
-	$(SUDO) $(LOSETUP) -d $(loop-dev) || :
-	$(SUDO) rm -f $(LOOP_FILE)
-	$(SUDO) $(LOSETUP) -d $(loop-dev2) || :
-	$(SUDO) rm -f $(LOOP_FILE2)
+create-loop-dev:
+	$(MINIKUBE) ssh -- $(DD) if=/dev/zero of=$(LOOP_FILE) bs=1G seek=32 count=0
+	$(MINIKUBE) ssh -- $(SUDO) $(LOSETUP) $(LOOP_DEV) $(LOOP_FILE) || :
+	$(MINIKUBE) ssh $(DD) if=/dev/zero of=$(LOOP_FILE2) bs=1G seek=32 count=0
+	$(MINIKUBE) ssh -- $(SUDO) $(LOSETUP) $(LOOP_DEV2) $(LOOP_FILE2) || :
+	$(MINIKUBE) ssh -- lsblk
 
 .PHONY: wait-deploy-ready
 wait-deploy-ready: NS=
@@ -162,8 +115,8 @@ launch-rook-ceph: create-loop-dev
 		--create-namespace --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values.yaml --wait \
 		rook-ceph rook-ceph
 	sed \
-		-e "s%{LOOP_DEV}%$(loop-dev)%" \
-		-e "s%{LOOP_DEV2}%$(loop-dev2)%" \
+		-e "s%{LOOP_DEV}%$(LOOP_DEV)%" \
+		-e "s%{LOOP_DEV2}%$(LOOP_DEV2)%" \
 		-e "s%{NODE_NAME}%$(NODE_NAME)%" \
 		testdata/persistentvolumes-template.yaml \
 		> testdata/persistentvolumes.yaml
@@ -178,38 +131,21 @@ launch-rook-ceph: create-loop-dev
 	$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0
 	$(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER2_NAMESPACE) DEPLOY=rook-ceph-osd-0
 
-.PHONY: delete-rook-ceph
-delete-rook-ceph:
-	$(HELM) uninstall --namespace $(CEPH_CLUSTER2_NAMESPACE) rook-ceph-cluster2 --wait || :
-	$(HELM) uninstall --namespace $(CEPH_CLUSTER1_NAMESPACE) rook-ceph-cluster --wait || :
-	$(SUDO) rm -rf /var/lib/rook
-	$(SUDO) rm -rf /var/lib/rook2
-	ls /dev/mapper/ceph-* | xargs -I% -- $(SUDO) dmsetup remove %
-	$(SUDO) rm -rf /dev/mapper/ceph-*
-	$(HELM) uninstall --namespace $(CEPH_CLUSTER1_NAMESPACE) rook-ceph --wait || :
-	$(KUBECTL) delete -f testdata/persistentvolumes.yaml || :
-	$(MAKE) delete-loop-dev
+.PHONY: image-build
+image-build:
+	eval $$($(MINIKUBE) docker-env); \
+	$(MAKE) -C ../.. docker-build
+	$(MINIKUBE) ssh -- docker images
 
 .PHONY: setup-components
 setup-components:
-	$(MAKE) -C ../.. docker-build
-	# We can't use `minikube image load` due to a minikube's problem.
-	# ref. https://github.com/kubernetes/minikube/issues/17785
-	docker save controller:latest | $(SUDO) ctr -n k8s.io images import -
-
 	$(HELM) upgrade --install mantle-cluster-wide ../../charts/mantle-cluster-wide/ --wait
 	$(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait
 	$(HELM) upgrade --install --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 ../../charts/mantle/ --wait
 
-.PHONY: delete-components
-delete-components:
-	$(HELM) uninstall --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 --wait || true
-	$(HELM) uninstall --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle --wait || true
-	$(HELM) uninstall mantle-cluster-wide --wait || true
-
 .PHONY: do_test
 do_test: $(GINKGO)
-	$(SUDO) -E env \
+	env \
 	PATH=${PATH} \
 	E2ETEST=1 \
 	KUBECTL=$(KUBECTL) \

diff --git a/test/e2e/testdata/values.yaml b/test/e2e/testdata/values.yaml
@@ -1,2 +1,4 @@
 allowLoopDevices: true
 resources: null
+csi:
+  enableCephfsDriver: false