diff --git a/Makefile b/Makefile index 05e38c173d26..2d05bdd59616 100644 --- a/Makefile +++ b/Makefile @@ -963,7 +963,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra .PHONY: test-e2e test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ - -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ + -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter=Foo \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 8e45da3e83d1..d2c8afd68af5 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -115,6 +115,14 @@ docker events > "${ARTIFACTS_LOCAL}/docker-events.txt" 2>&1 & ctr -n moby events > "${ARTIFACTS_LOCAL}/containerd-events.txt" 2>&1 & # Run e2e tests -mkdir -p "$ARTIFACTS" -echo "+ run tests!" -make test-e2e + +ORIGINAL_ARTIFACTS="${ARTIFACTS}" + +for i in {1..9}; do + ARTIFACTS="${ORIGINAL_ARTIFACTS}/${i}" + echo "Using artifacts location: $ARTIFACTS" + mkdir -p "$ARTIFACTS" + echo "+ run tests!" + + make test-e2e +done diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 4a262bdfdf3d..6161fefd5ebe 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -110,7 +110,7 @@ var _ = Describe("When testing clusterctl upgrades (v0.3=>v1.5=>current)", func( }) }) -var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", func() { +var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", Label("Foo"), func() { // We are testing v0.4=>v1.6=>current to ensure that old entries with v1alpha4 in managed files do not cause issues // as described in https://github.com/kubernetes-sigs/cluster-api/issues/10051. // NOTE: The combination of v0.4=>v1.6=>current allows us to verify this without being forced to upgrade diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index ed19ac6d6b37..bbb72170787e 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -43,6 +43,42 @@ import ( . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" ) +// GetGlobalCAPIResourcesInput is the input for GetCAPIResources. +type GetGlobalCAPIResourcesInput struct { + Lister Lister + Namespace string +} + +// GetGlobalCAPIResources reads all the CAPI resources in a namespace. +// This list includes all the types belonging to CAPI providers. +func GetGlobalCAPIResources(ctx context.Context, input GetGlobalCAPIResourcesInput) []*unstructured.Unstructured { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetGlobalCAPIResources") + Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetGlobalCAPIResources") + + types := getGlobalClusterAPITypes(ctx, input.Lister) + + objList := []*unstructured.Unstructured{} + for i := range types { + typeMeta := types[i] + typeList := new(unstructured.UnstructuredList) + typeList.SetAPIVersion(typeMeta.APIVersion) + typeList.SetKind(typeMeta.Kind) + + if err := input.Lister.List(ctx, typeList); err != nil { + if apierrors.IsNotFound(err) { + continue + } + Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err)) + } + for i := range typeList.Items { + obj := typeList.Items[i] + objList = append(objList, &obj) + } + } + + return objList +} + // GetCAPIResourcesInput is the input for GetCAPIResources. type GetCAPIResourcesInput struct { Lister Lister @@ -80,6 +116,35 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr return objList } +// getGlobalClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase. +// This list includes all the types belonging to CAPI providers. +func getGlobalClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { + discoveredTypes := []metav1.TypeMeta{} + + crdList := &apiextensionsv1.CustomResourceDefinitionList{} + Eventually(func() error { + return lister.List(ctx, crdList, client.HasLabels{"clusterctl.cluster.x-k8s.io/core"}) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers") + + for _, crd := range crdList.Items { + for _, version := range crd.Spec.Versions { + if !version.Storage { + continue + } + + discoveredTypes = append(discoveredTypes, metav1.TypeMeta{ + Kind: crd.Spec.Names.Kind, + APIVersion: metav1.GroupVersion{ + Group: crd.Spec.Group, + Version: version.Name, + }.String(), + }) + } + } + + return discoveredTypes +} + // getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase. // This list includes all the types belonging to CAPI providers. func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { @@ -105,6 +170,7 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { }) } } + return discoveredTypes } @@ -122,7 +188,16 @@ func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) { Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for DumpAllResources") Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for DumpAllResources") - resources := GetCAPIResources(ctx, GetCAPIResourcesInput{ + resources := GetGlobalCAPIResources(ctx, GetGlobalCAPIResourcesInput{ + Lister: input.Lister, + }) + + for i := range resources { + r := resources[i] + dumpObject(r, input.LogPath) + } + + resources = GetCAPIResources(ctx, GetCAPIResourcesInput{ Lister: input.Lister, Namespace: input.Namespace, }) diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index abe1b82b3cb2..bd6236d874c5 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -18,6 +18,7 @@ package clusterctl import ( "context" + "fmt" "os" "path/filepath" "time" @@ -128,6 +129,35 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init }) } } + + certManagerDeployments := framework.GetCertManagerDeployments(ctx, framework.GetControllerDeploymentsInput{ + Lister: client, + }) + for _, deployment := range certManagerDeployments { + fmt.Printf("Pod status of %s/%s:\n%++v\n", deployment.Namespace, deployment.Name, deployment.Status) + // framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + // Getter: client, + // Deployment: deployment, + // }, intervals...) + + // Start streaming logs from all controller providers + framework.WatchDeploymentLogsByName(ctx, framework.WatchDeploymentLogsByNameInput{ + GetLister: client, + Cache: input.ClusterProxy.GetCache(ctx), + ClientSet: input.ClusterProxy.GetClientSet(), + Deployment: deployment, + LogPath: filepath.Join(input.LogFolder, "logs", deployment.GetNamespace()), + }) + + // if !input.DisableMetricsCollection { + // framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{ + // GetLister: client, + // ClientSet: input.ClusterProxy.GetClientSet(), + // Deployment: deployment, + // MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), + // }) + // } + } } // UpgradeManagementClusterAndWaitInput is the input type for UpgradeManagementClusterAndWait. @@ -147,6 +177,29 @@ type UpgradeManagementClusterAndWaitInput struct { ClusterctlBinaryPath string } +func (i *UpgradeManagementClusterAndWaitInput) allProviders() []string { + providers := []string{i.CoreProvider} + for _, p := range i.BootstrapProviders { + providers = append(providers, "bootstrap"+p) + } + for _, p := range i.ControlPlaneProviders { + providers = append(providers, "control-plane-"+p) + } + for _, p := range i.InfrastructureProviders { + providers = append(providers, "infrastructure-"+p) + } + for _, p := range i.IPAMProviders { + providers = append(providers, "ipam-"+p) + } + for _, p := range i.RuntimeExtensionProviders { + providers = append(providers, "runtime-extension-"+p) + } + for _, p := range i.AddonProviders { + providers = append(providers, "addon-"+p) + } + return providers +} + // UpgradeManagementClusterAndWait upgrades provider a management cluster using clusterctl, and waits for the cluster to be ready. func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagementClusterAndWaitInput, intervals ...interface{}) { Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeManagementClusterAndWait") @@ -213,6 +266,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), }) } + + // log.Logf("Waiting for cert-manager to inject the new certificates to webhook relevant objects") + // framework.WaitForProviderCAInjection(ctx, client, filepath.Join(input.LogFolder, "foo", "crds")) } // ApplyClusterTemplateAndWaitInput is the input type for ApplyClusterTemplateAndWait. diff --git a/test/framework/controller_helpers.go b/test/framework/controller_helpers.go index a4327a37264d..6e88510c81ff 100644 --- a/test/framework/controller_helpers.go +++ b/test/framework/controller_helpers.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) // GetControllerDeploymentsInput is the input for GetControllerDeployments. @@ -46,6 +47,23 @@ func GetControllerDeployments(ctx context.Context, input GetControllerDeployment return deployments } +// GetCertManagerDeployments returns all the deployment for the cluster API controllers existing in a management cluster. +func GetCertManagerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment { + deploymentList := &appsv1.DeploymentList{} + Eventually(func() error { + return input.Lister.List(ctx, deploymentList, client.InNamespace("cert-manager")) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers") + + deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items)) + for i := range deploymentList.Items { + d := &deploymentList.Items[i] + if !skipDeployment(d, input.ExcludeNamespaces) { + deployments = append(deployments, d) + } + } + return deployments +} + func skipDeployment(d *appsv1.Deployment, excludeNamespaces []string) bool { if !d.DeletionTimestamp.IsZero() { return true diff --git a/test/framework/management_cluster_helpers.go b/test/framework/management_cluster_helpers.go new file mode 100644 index 000000000000..5466d1c76ad8 --- /dev/null +++ b/test/framework/management_cluster_helpers.go @@ -0,0 +1,73 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "os" + "path/filepath" + "time" + + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework/internal/log" +) + +// WaitForProviderCAInjection tries to query a list of all CRD objects for a provider +// to ensure the webhooks are correctly setup, especially that cert-manager did inject +// the up-to-date CAs into the relevant objects. +func WaitForProviderCAInjection(ctx context.Context, lister Lister, outpath string) { + crdList := &apiextensionsv1.CustomResourceDefinitionList{} + Eventually(func() error { + return lister.List(ctx, crdList, client.HasLabels{clusterv1.ProviderNameLabel}) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get crds of providers") + + outDir := outpath + Expect(os.MkdirAll(outDir, 0750)).To(Succeed()) + + for i := range crdList.Items { + crd := crdList.Items[i] + data, err := yaml.Marshal(crd) + Expect(err).ToNot(HaveOccurred()) + outFile := filepath.Join(outDir, crd.Name+".yaml") + log.Logf("Writing crd to %s", outpath) + Expect(os.WriteFile(outFile, data, 0600)).To(Succeed()) + // Use all versions so we also test conversion webhooks + for _, version := range crd.Spec.Versions { + if !version.Served { + continue + } + gvk := schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: version.Name, + Kind: crd.Spec.Names.Kind, + } + log.Logf("Checking crd %s - %s", gvk, time.Now()) + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk) + Eventually(func() error { + return lister.List(ctx, list) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get objects for crd") + } + } +} diff --git a/test/framework/ownerreference_helpers.go b/test/framework/ownerreference_helpers.go index 910952fbc77a..a755ce6a9a0e 100644 --- a/test/framework/ownerreference_helpers.go +++ b/test/framework/ownerreference_helpers.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "sort" + "strings" "time" . "github.com/onsi/gomega" @@ -112,6 +113,12 @@ func AssertOwnerReferences(namespace, kubeconfigPath string, assertFuncs ...map[ ctx := context.Background() graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, kubeconfigPath) + // Sometimes the conversion-webhooks are not ready yet / cert-managers ca-injector + // may not yet have injected the new ca bundle after the upgrade. + // If this is the case we return an error to retry. + if err != nil && strings.Contains(err.Error(), "x509: certificate signed by unknown authority") { + return err + } Expect(err).ToNot(HaveOccurred()) for _, v := range graph { if _, ok := allAssertFuncs[v.Object.Kind]; !ok {