chore(ci): add tests with longhorn v2 engine

Add tests with longhorn v2 engine. Fixes: #9669 Signed-off-by: Noel Georgi <[email protected]>
siderolabs · Jan 22, 2025 · 82c9ec1 · 82c9ec1
1 parent 689ea1d
commit 82c9ec1
Show file tree

Hide file tree

Showing 16 changed files with 227 additions and 110 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-22T14:28:18Z by kres 3075de9.
+# Generated on 2025-01-22T17:37:55Z by kres 3075de9.
 
 name: default
 concurrency:
@@ -2957,10 +2957,14 @@ jobs:
           EXTRA_TEST_ARGS: -talos.csi=longhorn
           GITHUB_STEP_NAME: ${{ github.job}}-e2e-qemu-csi-longhorn
           IMAGE_REGISTRY: registry.dev.siderolabs.io
-          QEMU_MEMORY_WORKERS: "3072"
+          QEMU_EXTRA_DISKS: "1"
+          QEMU_EXTRA_DISKS_DRIVERS: nvme
+          QEMU_EXTRA_DISKS_SIZE: "12288"
+          QEMU_MEMORY_WORKERS: "4096"
           QEMU_WORKERS: "3"
           SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/longhorn-cp.yaml'
+          WITH_CONFIG_PATCH_WORKER: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
         run: |
           sudo -E make e2e-qemu
       - name: save artifacts
@@ -3067,7 +3071,7 @@ jobs:
           QEMU_MEMORY_WORKERS: "4096"
           QEMU_WORKERS: "3"
           SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/openebs-cp.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/openebs-cp.yaml'
           WITH_CONFIG_PATCH_WORKER: '@hack/test/patches/openebs.yaml'
         run: |
           sudo -E make e2e-qemu
@@ -3175,7 +3179,7 @@ jobs:
           QEMU_MEMORY_WORKERS: "8192"
           QEMU_WORKERS: "3"
           SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/rook-ceph.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/rook-ceph.yaml'
         run: |
           sudo -E make e2e-qemu
       - name: save artifacts

diff --git a/.github/workflows/integration-qemu-csi-longhorn-cron.yaml b/.github/workflows/integration-qemu-csi-longhorn-cron.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-21T10:29:14Z by kres 3075de9.
+# Generated on 2025-01-22T17:37:55Z by kres 3075de9.
 
 name: integration-qemu-csi-longhorn-cron
 concurrency:
@@ -112,10 +112,14 @@ jobs:
           EXTRA_TEST_ARGS: -talos.csi=longhorn
           GITHUB_STEP_NAME: ${{ github.job}}-e2e-qemu-csi-longhorn
           IMAGE_REGISTRY: registry.dev.siderolabs.io
-          QEMU_MEMORY_WORKERS: "3072"
+          QEMU_EXTRA_DISKS: "1"
+          QEMU_EXTRA_DISKS_DRIVERS: nvme
+          QEMU_EXTRA_DISKS_SIZE: "12288"
+          QEMU_MEMORY_WORKERS: "4096"
           QEMU_WORKERS: "3"
           SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/longhorn-cp.yaml'
+          WITH_CONFIG_PATCH_WORKER: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
         run: |
           sudo -E make e2e-qemu
       - name: save artifacts

diff --git a/.github/workflows/integration-qemu-csi-openebs-cron.yaml b/.github/workflows/integration-qemu-csi-openebs-cron.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-21T10:29:14Z by kres 3075de9.
+# Generated on 2025-01-22T14:23:24Z by kres 3075de9.
 
 name: integration-qemu-csi-openebs-cron
 concurrency:
@@ -89,7 +89,7 @@ jobs:
           QEMU_MEMORY_WORKERS: "4096"
           QEMU_WORKERS: "3"
           SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/openebs-cp.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/openebs-cp.yaml'
           WITH_CONFIG_PATCH_WORKER: '@hack/test/patches/openebs.yaml'
         run: |
           sudo -E make e2e-qemu

diff --git a/.github/workflows/integration-qemu-csi-rook-ceph-cron.yaml b/.github/workflows/integration-qemu-csi-rook-ceph-cron.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-21T10:29:14Z by kres 3075de9.
+# Generated on 2025-01-22T14:23:24Z by kres 3075de9.
 
 name: integration-qemu-csi-rook-ceph-cron
 concurrency:
@@ -89,7 +89,7 @@ jobs:
           QEMU_MEMORY_WORKERS: "8192"
           QEMU_WORKERS: "3"
           SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/rook-ceph.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/rook-ceph.yaml'
         run: |
           sudo -E make e2e-qemu
       - name: save artifacts

diff --git a/.kres.yaml b/.kres.yaml
@@ -1352,7 +1352,7 @@ spec:
             QEMU_MEMORY_WORKERS: 8192
             QEMU_EXTRA_DISKS: 1
             QEMU_EXTRA_DISKS_SIZE: 12288
-            WITH_CONFIG_PATCH: "@hack/test/patches/rook-ceph.yaml"
+            WITH_CONFIG_PATCH_CONTROLPLANE: "@hack/test/patches/rook-ceph.yaml"
             EXTRA_TEST_ARGS: -talos.csi=rook-ceph
             IMAGE_REGISTRY: registry.dev.siderolabs.io
         - name: save-fio-benchmark
@@ -1441,8 +1441,12 @@ spec:
             GITHUB_STEP_NAME: ${{ github.job}}-e2e-qemu-csi-longhorn
             SHORT_INTEGRATION_TEST: yes
             QEMU_WORKERS: 3
-            QEMU_MEMORY_WORKERS: 3072
-            WITH_CONFIG_PATCH: "@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml"
+            QEMU_MEMORY_WORKERS: 4096
+            QEMU_EXTRA_DISKS: 1
+            QEMU_EXTRA_DISKS_SIZE: 12288
+            QEMU_EXTRA_DISKS_DRIVERS: nvme
+            WITH_CONFIG_PATCH_CONTROLPLANE: "@hack/test/patches/longhorn-cp.yaml"
+            WITH_CONFIG_PATCH_WORKER: "@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml"
             EXTRA_TEST_ARGS: -talos.csi=longhorn
             IMAGE_REGISTRY: registry.dev.siderolabs.io
         - name: save-fio-benchmark
@@ -1508,7 +1512,7 @@ spec:
             QEMU_MEMORY_WORKERS: 4096
             QEMU_EXTRA_DISKS: 1
             QEMU_EXTRA_DISKS_SIZE: 12288
-            WITH_CONFIG_PATCH: "@hack/test/patches/openebs-cp.yaml"
+            WITH_CONFIG_PATCH_CONTROLPLANE: "@hack/test/patches/openebs-cp.yaml"
             WITH_CONFIG_PATCH_WORKER: "@hack/test/patches/openebs.yaml"
             EXTRA_TEST_ARGS: -talos.csi=openebs
             IMAGE_REGISTRY: registry.dev.siderolabs.io

diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh
@@ -139,6 +139,18 @@ case "${WITH_ISO:-false}" in
     ;;
 esac
 
+case "${WITH_CONFIG_PATCH_CONTROLPLANE:-false}" in
+  false)
+    ;;
+  *)
+    [[ ! ${WITH_CONFIG_PATCH_CONTROLPLANE} =~ ^@ ]] && echo "WITH_CONFIG_PATCH_CONTROLPLANE variable should start with @" && exit 1
+
+    for i in ${WITH_CONFIG_PATCH_CONTROLPLANE//:/ }; do
+      QEMU_FLAGS+=("--config-patch-control-plane=${i}")
+    done
+    ;;
+esac
+
 case "${WITH_CONFIG_PATCH_WORKER:-false}" in
   false)
     ;;

diff --git a/hack/test/patches/longhorn-cp.yaml b/hack/test/patches/longhorn-cp.yaml
@@ -0,0 +1,9 @@
+---
+cluster:
+  apiServer:
+    admissionControl:
+      - name: PodSecurity
+        configuration:
+          exemptions:
+            namespaces:
+              - longhorn-system
diff --git a/hack/test/patches/longhorn.yaml b/hack/test/patches/longhorn.yaml
@@ -1,4 +1,12 @@
+---
 machine:
+  sysctls:
+    vm.nr_hugepages: "1024"
+  kernel:
+    modules:
+      - name: nvme_tcp
+      - name: vfio_pci
+      - name: uio_pci_generic
   kubelet:
     extraMounts:
       - destination: /var/lib/longhorn
@@ -8,11 +16,3 @@ machine:
           - bind
           - rshared
           - rw
-cluster:
-  apiServer:
-    admissionControl:
-      - name: PodSecurity
-        configuration:
-          exemptions:
-            namespaces:
-              - longhorn-system
diff --git a/hack/test/patches/openebs-cp.yaml b/hack/test/patches/openebs-cp.yaml
@@ -1,10 +1,9 @@
+---
 cluster:
   apiServer:
     admissionControl:
       - name: PodSecurity
         configuration:
-          apiVersion: pod-security.admission.config.k8s.io/v1beta1
-          kind: PodSecurityConfiguration
           exemptions:
             namespaces:
               - openebs
diff --git a/hack/test/patches/openebs.yaml b/hack/test/patches/openebs.yaml
@@ -1,3 +1,4 @@
+---
 machine:
   sysctls:
     vm.nr_hugepages: "1024"

diff --git a/internal/integration/base/k8s.go b/internal/integration/base/k8s.go
@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/discovery"
@@ -588,7 +589,7 @@ func (k8sSuite *K8sSuite) WaitForResource(ctx context.Context, namespace, group,
 		}
 
 		if !exists {
-			return true, fmt.Errorf("resource %s/%s/%s/%s not found", group, version, kind, resourceName)
+			return true, errors.NewNotFound(mapping.Resource.GroupResource(), resourceName)
 		}
 
 		return false, nil
@@ -657,15 +658,15 @@ func (k8sSuite *K8sSuite) GetUnstructuredResource(ctx context.Context, namespace
 }
 
 // RunFIOTest runs the FIO test with the given storage class and size using kubestr.
-func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClasss, size string) error {
+func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClass, size string) error {
 	args := []string{
 		"--outfile",
-		fmt.Sprintf("/tmp/fio-%s.json", storageClasss),
+		fmt.Sprintf("/tmp/fio-%s.json", storageClass),
 		"--output",
 		"json",
 		"fio",
 		"--storageclass",
-		storageClasss,
+		storageClass,
 		"--size",
 		size,
 	}
@@ -821,6 +822,24 @@ func (k8sSuite *K8sSuite) DeleteManifests(ctx context.Context, manifests []unstr
 	}
 }
 
+// PatchK8sObject patches the kubernetes object with the given namespace, group, kind, version and name.
+func (k8sSuite *K8sSuite) PatchK8sObject(ctx context.Context, namespace, group, kind, version, resourceName string, patchBytes []byte) {
+	patchBytes, err := yaml.ToJSON(patchBytes)
+	k8sSuite.Require().NoError(err, "error converting patch to JSON")
+
+	mapping, err := k8sSuite.Mapper.RESTMapping(schema.GroupKind{
+		Group: group,
+		Kind:  kind,
+	}, version)
+
+	k8sSuite.Require().NoError(err, "error creating mapping for resource %s/%s/%s", group, kind, version)
+
+	dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(namespace)
+
+	_, err = dr.Patch(ctx, resourceName, types.MergePatchType, patchBytes, metav1.PatchOptions{})
+	k8sSuite.Require().NoError(err, "error patching resource %s/%s/%s/%s", group, version, kind, resourceName)
+}
+
 // ToUnstructured converts the given runtime.Object to unstructured.Unstructured.
 func (k8sSuite *K8sSuite) ToUnstructured(obj runtime.Object) unstructured.Unstructured {
 	unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)

diff --git a/internal/integration/k8s/constants.go b/internal/integration/k8s/constants.go
@@ -13,7 +13,7 @@ const (
 	RookCephHelmChartVersion = "v1.16.0"
 	// LongHornHelmChartVersion is the version of the Longhorn Helm chart to use.
 	// renovate: datasource=helm versioning=helm depName=longhorn registryUrl=https://charts.longhorn.io
-	LongHornHelmChartVersion = "v1.7.2"
+	LongHornHelmChartVersion = "v1.8.0"
 	// OpenEBSChartVersion is the version of the OpenEBS Helm chart to use.
 	// renovate: datasource=helm versioning=helm depName=openebs registryUrl=https://openebs.github.io/openebs
 	OpenEBSChartVersion = "v4.1.2"