From 3c3f847edeabdaea5de50eae2f8919b057bd2379 Mon Sep 17 00:00:00 2001 From: Camila Macedo Date: Sat, 14 Sep 2024 15:45:45 +0100 Subject: [PATCH] test --- test/e2e/deployimage/generate_test.go | 50 +++++- test/e2e/deployimage/plugin_cluster_test.go | 54 ++++++- test/e2e/utils/kustomize.go | 100 ++++++++++++ test/e2e/utils/metrics.go | 153 ++++++++++++++++++ test/e2e/utils/webhooks.go | 12 +- test/e2e/v4/generate_test.go | 107 +------------ test/e2e/v4/plugin_cluster_test.go | 166 ++------------------ 7 files changed, 376 insertions(+), 266 deletions(-) create mode 100644 test/e2e/utils/kustomize.go create mode 100644 test/e2e/utils/metrics.go diff --git a/test/e2e/deployimage/generate_test.go b/test/e2e/deployimage/generate_test.go index 3af5680b764..12144a2ac15 100644 --- a/test/e2e/deployimage/generate_test.go +++ b/test/e2e/deployimage/generate_test.go @@ -17,8 +17,13 @@ limitations under the License. package deployimage import ( + "fmt" + "path/filepath" + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + pluginutil "sigs.k8s.io/kubebuilder/v4/pkg/plugin/util" //nolint:golint // nolint:revive @@ -36,10 +41,18 @@ func GenerateDeployImageWithOptions(kbc *utils.TestContext) { // GenerateDeployImage implements a go/v4 plugin project and scaffold an API using the deploy image plugin func GenerateDeployImage(kbc *utils.TestContext) { initTheProject(kbc) - creatingAPI(kbc) + createAPI(kbc) +} + +// GenerateWithWebhooks +func GenerateWithWebhook(kbc *utils.TestContext) { + initTheProject(kbc) + createAPI(kbc) + createWebhook(kbc) + enableMetricsCertManager(kbc) } -func creatingAPI(kbc *utils.TestContext) { +func createAPI(kbc *utils.TestContext) { By("creating API definition with deploy-image/v1-alpha plugin") err := kbc.CreateAPI( "--group", kbc.Group, @@ -81,3 +94,36 @@ func initTheProject(kbc *utils.TestContext) { ) ExpectWithOffset(1, err).NotTo(HaveOccurred()) } + +func createWebhook(kbc *utils.TestContext) { + By("creating Webhook for the API") + err := kbc.CreateWebhook( + "--group", kbc.Group, + "--version", kbc.Version, + "--kind", kbc.Kind, + "--defaulting", + "--programmatic-validation", + ) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("implementing the mutating and validating webhooks") + webhookFilePath := filepath.Join( + kbc.Dir, "api", kbc.Version, + fmt.Sprintf("%s_webhook.go", strings.ToLower(kbc.Kind))) + err = utils.ImplementWebhooks(webhookFilePath, strings.ToLower(kbc.Kind)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) +} + +func enableMetricsCertManager(kbc *utils.TestContext) { + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- ../certmanager", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- path: webhookcainjection_patch.yaml", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + utils.CertManagerTarget, "#")).To(Succeed()) +} diff --git a/test/e2e/deployimage/plugin_cluster_test.go b/test/e2e/deployimage/plugin_cluster_test.go index 3597d4a1d91..ef8b1d2bdb3 100644 --- a/test/e2e/deployimage/plugin_cluster_test.go +++ b/test/e2e/deployimage/plugin_cluster_test.go @@ -20,6 +20,7 @@ import ( "fmt" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -52,18 +53,23 @@ var _ = Describe("kubebuilder", func() { It("should generate a runnable project with deploy-image/v1-alpha options ", func() { GenerateDeployImageWithOptions(kbc) - Run(kbc) + Run(kbc, false) }) It("should generate a runnable project with deploy-image/v1-alpha without options ", func() { GenerateDeployImage(kbc) - Run(kbc) + Run(kbc, false) + }) + + It("should generate a runnable project with deploy-image/v1-alpha and webhooks for the API", func() { + GenerateWithWebhook(kbc) + Run(kbc, true) }) }) }) // Run runs a set of e2e tests for a scaffolded project defined by a TestContext. -func Run(kbc *utils.TestContext) { +func Run(kbc *utils.TestContext, hasWebhooks bool) { var controllerPodName string var err error @@ -148,6 +154,48 @@ func Run(kbc *utils.TestContext) { } Eventually(verifyAvailableStatus).Should(Succeed()) + if hasWebhooks { + By("configuring metrics and create curl Pod") + utils.EnableMetricsCreateCurlPod(kbc) + + By("creating an instance of the CR not in the manager namespace") + sampleFile := filepath.Join("config", "samples", + fmt.Sprintf("%s_%s_%s.yaml", kbc.Group, kbc.Version, strings.ToLower(kbc.Kind))) + + sampleFilePath, err := filepath.Abs(filepath.Join(fmt.Sprintf("e2e-%s", kbc.TestSuffix), sampleFile)) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func(g Gomega) { + g.Expect(kbc.Kubectl.Apply(false, "-f", sampleFilePath)).Error().NotTo(HaveOccurred()) + }).Should(Succeed()) + + By("validating webhooks not in the default namespace") + cnt, err := kbc.Kubectl.Get( + false, + "-f", sampleFile, + "-o", "go-template={{ .spec.size }}") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + size, err := strconv.Atoi(cnt) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, size).To(BeNumerically("==", 5)) + + By("checking the metrics values gets reconciled") + metricsOutput := utils.GetMetricsOutput(kbc) + ExpectWithOffset(1, metricsOutput).To(ContainSubstring(fmt.Sprintf( + `controller_runtime_reconcile_total{controller="%s",result="success"}`, + strings.ToLower(kbc.Kind), + ))) + + By("removing Curl Pod") + utils.RemoveCurlPod(kbc) + + By("removing cr applied not in the default namespace") + Eventually(func(g Gomega) { + g.Expect(kbc.Kubectl.Delete(false, + "-f", sampleFilePath)).Error().NotTo(HaveOccurred()) + }).Should(Succeed()) + } + By("validating the finalizer") Eventually(func(g Gomega) { g.Expect(kbc.Kubectl.Delete(true, "-f", sampleFilePath)).Error().NotTo(HaveOccurred()) diff --git a/test/e2e/utils/kustomize.go b/test/e2e/utils/kustomize.go new file mode 100644 index 00000000000..e57e32eb20b --- /dev/null +++ b/test/e2e/utils/kustomize.go @@ -0,0 +1,100 @@ +package utils + +//nolint:lll +const CertManagerTarget = `#replacements: +# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: CustomResourceDefinition +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: CustomResourceDefinition +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: # Add cert-manager annotation to the webhook Service +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true` diff --git a/test/e2e/utils/metrics.go b/test/e2e/utils/metrics.go new file mode 100644 index 00000000000..9bbe9bc3aa8 --- /dev/null +++ b/test/e2e/utils/metrics.go @@ -0,0 +1,153 @@ +package utils + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}` +) + +// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type +// that we want to use for extracting the token. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} + +// ServiceAccountToken provides a helper function that can provide you with a service account +// token that you can use to interact with the service. This function leverages the k8s' +// TokenRequest API in raw format in order to make it generic for all version of the k8s that +// is currently being supported in kubebuilder test infra. +// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. +func ServiceAccountToken(kbc *TestContext) (out string, err error) { + secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount) + tokenRequestFile := filepath.Join(kbc.Dir, secretName) + err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return out, err + } + var rawJson string + Eventually(func() error { + // Output of this is already a valid JWT token. No need to covert this from base64 to string format + rawJson, err = kbc.Kubectl.Command( + "create", + "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + kbc.Kubectl.Namespace, + kbc.Kubectl.ServiceAccount, + ), + "-f", tokenRequestFile, + ) + if err != nil { + return err + } + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +func EnableMetricsCreateCurlPod(kbc *TestContext) { + _, err := kbc.Kubectl.Command( + "create", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix), + fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", kbc.TestSuffix), + fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + token, err := ServiceAccountToken(kbc) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("validating that the controller-manager service is available") + _, err = kbc.Kubectl.Get( + true, + "service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + ) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist") + + By("ensuring the service endpoint is ready") + eventuallyCheckServiceEndpoint := func() error { + output, err := kbc.Kubectl.Get( + true, + "endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + "-o", "jsonpath={.subsets[*].addresses[*].ip}", + ) + if err != nil { + return err + } + if output == "" { + return fmt.Errorf("no endpoints found") + } + return nil + } + EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(), + "Service endpoint should be ready") + + By("creating a curl pod to access the metrics endpoint") + cmdOpts := CmdOptsToCreateCurlPod(kbc, token) + _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + + By("validating that the curl pod is running as expected") + verifyCurlUp := func() error { + status, err := kbc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) +} + +// GetMetricsOutput return the metrics output from curl pod +func GetMetricsOutput(kbc *TestContext) string { + By("getting logs from curl") + var metricsOutput string + getCurlLogs := func() string { + metricsOutput, err := kbc.Kubectl.Logs("curl") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + return metricsOutput + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// RemoveCurlPod created to collect the metrics +func RemoveCurlPod(kbc *TestContext) { + By("cleaning up the curl pod") + _, err := kbc.Kubectl.Delete(true, "pods/curl") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) +} + +// CmdOptsToCreateCurlPod return the options to create curl Pod +func CmdOptsToCreateCurlPod(kbc *TestContext, token string) []string { + // nolint:lll + cmdOpts := []string{ + "run", "curl", + "--restart=Never", + "--namespace", kbc.Kubectl.Namespace, + "--image=curlimages/curl:7.78.0", + "--", + "/bin/sh", "-c", fmt.Sprintf("curl -v -k -H 'Authorization: Bearer %s' https://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8443/metrics", + token, kbc.TestSuffix, kbc.Kubectl.Namespace), + } + return cmdOpts +} diff --git a/test/e2e/utils/webhooks.go b/test/e2e/utils/webhooks.go index f338d722054..938c0124320 100644 --- a/test/e2e/utils/webhooks.go +++ b/test/e2e/utils/webhooks.go @@ -43,8 +43,8 @@ func ImplementWebhooks(filename, lowerKind string) error { } // implement defaulting webhook logic - replace := fmt.Sprintf(`if %s.Spec.Count == 0 { - %s.Spec.Count = 5 + replace := fmt.Sprintf(`if %s.Spec.Size == 0 { + %s.Spec.Size = 5 }`, lowerKind, lowerKind) str, err = util.EnsureExistAndReplace( str, @@ -59,8 +59,8 @@ func ImplementWebhooks(filename, lowerKind string) error { str, err = util.EnsureExistAndReplace( str, "// TODO(user): fill in your validation logic upon object creation.", - fmt.Sprintf(`if %s.Spec.Count < 0 { - return nil, errors.New(".spec.count must >= 0") + fmt.Sprintf(`if %s.Spec.Size < 0 { + return nil, errors.New(".spec.size must >= 0") }`, lowerKind)) if err != nil { return err @@ -68,8 +68,8 @@ func ImplementWebhooks(filename, lowerKind string) error { str, err = util.EnsureExistAndReplace( str, "// TODO(user): fill in your validation logic upon object update.", - fmt.Sprintf(`if %s.Spec.Count < 0 { - return nil, errors.New(".spec.count must >= 0") + fmt.Sprintf(`if %s.Spec.Size < 0 { + return nil, errors.New(".spec.size must >= 0") }`, lowerKind)) if err != nil { return err diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index d02f71f4aa2..6a67d3e94bc 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -66,7 +66,7 @@ func GenerateV4(kbc *utils.TestContext) { "#- path: webhookcainjection_patch.yaml", "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - certManagerTarget, "#")).To(Succeed()) + utils.CertManagerTarget, "#")).To(Succeed()) if kbc.IsRestricted { By("uncomment kustomize files to ensure that pods are restricted") @@ -106,7 +106,7 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) { filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- path: webhookcainjection_patch.yaml", "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - certManagerTarget, "#")).To(Succeed()) + utils.CertManagerTarget, "#")).To(Succeed()) // Disable metrics ExpectWithOffset(1, pluginutil.CommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), @@ -178,7 +178,7 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) { "#- ../network-policy", "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - certManagerTarget, "#")).To(Succeed()) + utils.CertManagerTarget, "#")).To(Succeed()) } // GenerateV4WithoutWebhooks implements a go/v4 plugin with APIs and enable Prometheus and CertManager @@ -215,7 +215,7 @@ func creatingAPI(kbc *utils.TestContext) { fmt.Sprintf(`type %sSpec struct { `, kbc.Kind), ` // +optional -Count int `+"`"+`json:"count,omitempty"`+"`"+` +Size int `+"`"+`json:"size,omitempty"`+"`"+` `)).Should(Succeed()) } @@ -233,105 +233,6 @@ const metricsTarget = `- path: manager_metrics_patch.yaml target: kind: Deployment` -//nolint:lll -const certManagerTarget = `#replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true` - func uncommentPodStandards(kbc *utils.TestContext) { configManager := filepath.Join(kbc.Dir, "config", "manager", "manager.yaml") diff --git a/test/e2e/v4/plugin_cluster_test.go b/test/e2e/v4/plugin_cluster_test.go index 2a9082365d3..4d11c66c3eb 100644 --- a/test/e2e/v4/plugin_cluster_test.go +++ b/test/e2e/v4/plugin_cluster_test.go @@ -17,7 +17,6 @@ limitations under the License. package v4 import ( - "encoding/json" "fmt" "os" "os/exec" @@ -33,18 +32,6 @@ import ( "sigs.k8s.io/kubebuilder/v4/test/e2e/utils" ) -const ( - tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}` -) - -// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type -// that we want to use for extracting the token. -type tokenRequest struct { - Status struct { - Token string `json:"token"` - } `json:"status"` -} - var _ = Describe("kubebuilder", func() { Context("plugin go/v4", func() { var kbc *utils.TestContext @@ -287,12 +274,18 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool, }, time.Minute, time.Second).Should(Succeed()) if hasMetrics { + By("configuring metrics and create curl Pod") + utils.EnableMetricsCreateCurlPod(kbc) + By("checking the metrics values to validate that the created resource object gets reconciled") - metricsOutput := getMetricsOutput(kbc) + metricsOutput := utils.GetMetricsOutput(kbc) ExpectWithOffset(1, metricsOutput).To(ContainSubstring(fmt.Sprintf( `controller_runtime_reconcile_total{controller="%s",result="success"} 1`, strings.ToLower(kbc.Kind), ))) + + By("removing Curl Pod") + utils.RemoveCurlPod(kbc) } if !hasMetrics { @@ -305,33 +298,26 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool, cnt, err := kbc.Kubectl.Get( true, "-f", sampleFile, - "-o", "go-template={{ .spec.count }}") + "-o", "go-template={{ .spec.size }}") ExpectWithOffset(1, err).NotTo(HaveOccurred()) - count, err := strconv.Atoi(cnt) + size, err := strconv.Atoi(cnt) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - ExpectWithOffset(1, count).To(BeNumerically("==", 5)) + ExpectWithOffset(1, size).To(BeNumerically("==", 5)) } if hasWebhook && hasNetworkPolicies { By("validating that webhooks from namespace without the label will fail") - - // Define the namespace name and CR sample file path namespace := "test-namespace-without-webhook-label" - sampleFile := "path/to/your/sample-file.yaml" - - // Create the namespace By("creating a namespace without the webhook: enabled label") _, err := kbc.Kubectl.Command("create", "namespace", namespace) Expect(err).NotTo(HaveOccurred(), "namespace should be created successfully") - // Apply the Custom Resource in the new namespace and expect it to fail By("applying the CR in the namespace without the webhook: enabled label and expecting it to fail") EventuallyWithOffset(1, func() error { _, err = kbc.Kubectl.Apply(false, "-n", namespace, "-f", sampleFile) return err }, time.Minute, time.Second).Should(HaveOccurred(), "applying the CR should fail due to webhook call timeout") - // Cleanup: Remove the namespace By("removing the namespace") _, err = kbc.Kubectl.Command("delete", "namespace", namespace) Expect(err).NotTo(HaveOccurred(), "namespace should be removed successfully") @@ -375,73 +361,6 @@ func getControllerName(kbc *utils.TestContext) string { return controllerPodName } -// getMetricsOutput return the metrics output from curl pod -func getMetricsOutput(kbc *utils.TestContext) string { - _, err := kbc.Kubectl.Command( - "create", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix), - fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", kbc.TestSuffix), - fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - token, err := serviceAccountToken(kbc) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - ExpectWithOffset(2, token).NotTo(BeEmpty()) - - var metricsOutput string - By("validating that the controller-manager service is available") - _, err = kbc.Kubectl.Get( - true, - "service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), - ) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist") - - By("ensuring the service endpoint is ready") - eventuallyCheckServiceEndpoint := func() error { - output, err := kbc.Kubectl.Get( - true, - "endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), - "-o", "jsonpath={.subsets[*].addresses[*].ip}", - ) - if err != nil { - return err - } - if output == "" { - return fmt.Errorf("no endpoints found") - } - return nil - } - EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(), - "Service endpoint should be ready") - - By("creating a curl pod to access the metrics endpoint") - cmdOpts := cmdOptsToCreateCurlPod(kbc, token) - _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - - By("validating that the curl pod is running as expected") - verifyCurlUp := func() error { - status, err := kbc.Kubectl.Get( - true, - "pods", "curl", "-o", "jsonpath={.status.phase}") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - if status != "Succeeded" { - return fmt.Errorf("curl pod in %s status", status) - } - return nil - } - EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) - - By("validating that the metrics endpoint is serving as expected") - getCurlLogs := func() string { - metricsOutput, err = kbc.Kubectl.Logs("curl") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - return metricsOutput - } - EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/1.1 200 OK")) - removeCurlPod(kbc) - return metricsOutput -} - func metricsShouldBeUnavailable(kbc *utils.TestContext) { _, err := kbc.Kubectl.Command( "create", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix), @@ -449,12 +368,12 @@ func metricsShouldBeUnavailable(kbc *utils.TestContext) { fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - token, err := serviceAccountToken(kbc) + token, err := utils.ServiceAccountToken(kbc) ExpectWithOffset(2, err).NotTo(HaveOccurred()) ExpectWithOffset(2, token).NotTo(BeEmpty()) By("creating a curl pod to access the metrics endpoint") - cmdOpts := cmdOptsToCreateCurlPod(kbc, token) + cmdOpts := utils.CmdOptsToCreateCurlPod(kbc, token) _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) ExpectWithOffset(2, err).NotTo(HaveOccurred()) @@ -479,64 +398,7 @@ func metricsShouldBeUnavailable(kbc *utils.TestContext) { return metricsOutput } EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("Could not resolve host")) - removeCurlPod(kbc) -} - -func cmdOptsToCreateCurlPod(kbc *utils.TestContext, token string) []string { - // nolint:lll - cmdOpts := []string{ - "run", "curl", - "--restart=Never", - "--namespace", kbc.Kubectl.Namespace, - "--image=curlimages/curl:7.78.0", - "--", - "/bin/sh", "-c", fmt.Sprintf("curl -v -k -H 'Authorization: Bearer %s' https://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8443/metrics", - token, kbc.TestSuffix, kbc.Kubectl.Namespace), - } - return cmdOpts -} - -func removeCurlPod(kbc *utils.TestContext) { - By("cleaning up the curl pod") - _, err := kbc.Kubectl.Delete(true, "pods/curl") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) -} - -// serviceAccountToken provides a helper function that can provide you with a service account -// token that you can use to interact with the service. This function leverages the k8s' -// TokenRequest API in raw format in order to make it generic for all version of the k8s that -// is currently being supported in kubebuilder test infra. -// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. -func serviceAccountToken(kbc *utils.TestContext) (out string, err error) { - secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount) - tokenRequestFile := filepath.Join(kbc.Dir, secretName) - err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) - if err != nil { - return out, err - } - var rawJson string - Eventually(func() error { - // Output of this is already a valid JWT token. No need to covert this from base64 to string format - rawJson, err = kbc.Kubectl.Command( - "create", - "--raw", fmt.Sprintf( - "/api/v1/namespaces/%s/serviceaccounts/%s/token", - kbc.Kubectl.Namespace, - kbc.Kubectl.ServiceAccount, - ), - "-f", tokenRequestFile, - ) - if err != nil { - return err - } - var token tokenRequest - err = json.Unmarshal([]byte(rawJson), &token) - if err != nil { - return err - } - out = token.Status.Token - return nil - }, time.Minute, time.Second).Should(Succeed()) - return out, err + By("removing Curl Pod") + utils.RemoveCurlPod(kbc) }