🌱 [DoNotReview] [WIP] debug cert-manager

Makefile

@@ -963,7 +963,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra
 .PHONY: test-e2e
 test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests
 	$(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
-		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \
+		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter=Foo \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \
 	    -e2e.artifacts-folder="$(ARTIFACTS)" \

scripts/ci-e2e.sh

@@ -115,6 +115,14 @@ docker events > "${ARTIFACTS_LOCAL}/docker-events.txt" 2>&1 &
 ctr -n moby events > "${ARTIFACTS_LOCAL}/containerd-events.txt" 2>&1 &
 
 # Run e2e tests
-mkdir -p "$ARTIFACTS"
-echo "+ run tests!"
-make test-e2e
+
+ORIGINAL_ARTIFACTS="${ARTIFACTS}"
+
+for i in {1..9}; do
+  ARTIFACTS="${ORIGINAL_ARTIFACTS}/${i}"
+  echo "Using artifacts location: $ARTIFACTS"
+  mkdir -p "$ARTIFACTS"
+  echo "+ run tests!"
+
+  make test-e2e
+done

test/e2e/clusterctl_upgrade_test.go

@@ -110,7 +110,7 @@ var _ = Describe("When testing clusterctl upgrades (v0.3=>v1.5=>current)", func(
 	})
 })
 
-var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", func() {
+var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", Label("Foo"), func() {
 	// We are testing v0.4=>v1.6=>current to ensure that old entries with v1alpha4 in managed files do not cause issues
 	// as described in https://github.com/kubernetes-sigs/cluster-api/issues/10051.
 	// NOTE: The combination of v0.4=>v1.6=>current allows us to verify this without being forced to upgrade

test/framework/alltypes_helpers.go

@@ -43,6 +43,42 @@ import (
 	. "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
 )
 
+// GetGlobalCAPIResourcesInput is the input for GetCAPIResources.
+type GetGlobalCAPIResourcesInput struct {
+	Lister    Lister
+	Namespace string
+}
+
+// GetGlobalCAPIResources reads all the CAPI resources in a namespace.
+// This list includes all the types belonging to CAPI providers.
+func GetGlobalCAPIResources(ctx context.Context, input GetGlobalCAPIResourcesInput) []*unstructured.Unstructured {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for GetGlobalCAPIResources")
+	Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetGlobalCAPIResources")
+
+	types := getGlobalClusterAPITypes(ctx, input.Lister)
+
+	objList := []*unstructured.Unstructured{}
+	for i := range types {
+		typeMeta := types[i]
+		typeList := new(unstructured.UnstructuredList)
+		typeList.SetAPIVersion(typeMeta.APIVersion)
+		typeList.SetKind(typeMeta.Kind)
+
+		if err := input.Lister.List(ctx, typeList); err != nil {
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+			Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err))
+		}
+		for i := range typeList.Items {
+			obj := typeList.Items[i]
+			objList = append(objList, &obj)
+		}
+	}
+
+	return objList
+}
+
 // GetCAPIResourcesInput is the input for GetCAPIResources.
 type GetCAPIResourcesInput struct {
 	Lister    Lister
@@ -80,6 +116,35 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr
 	return objList
 }
 
+// getGlobalClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase.
+// This list includes all the types belonging to CAPI providers.
+func getGlobalClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta {
+	discoveredTypes := []metav1.TypeMeta{}
+
+	crdList := &apiextensionsv1.CustomResourceDefinitionList{}
+	Eventually(func() error {
+		return lister.List(ctx, crdList, client.HasLabels{"clusterctl.cluster.x-k8s.io/core"})
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers")
+
+	for _, crd := range crdList.Items {
+		for _, version := range crd.Spec.Versions {
+			if !version.Storage {
+				continue
+			}
+
+			discoveredTypes = append(discoveredTypes, metav1.TypeMeta{
+				Kind: crd.Spec.Names.Kind,
+				APIVersion: metav1.GroupVersion{
+					Group:   crd.Spec.Group,
+					Version: version.Name,
+				}.String(),
+			})
+		}
+	}
+
+	return discoveredTypes
+}
+
 // getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase.
 // This list includes all the types belonging to CAPI providers.
 func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta {
@@ -105,6 +170,7 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta {
 			})
 		}
 	}
+
 	return discoveredTypes
 }
 
@@ -122,7 +188,16 @@ func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) {
 	Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for DumpAllResources")
 	Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for DumpAllResources")
 
-	resources := GetCAPIResources(ctx, GetCAPIResourcesInput{
+	resources := GetGlobalCAPIResources(ctx, GetGlobalCAPIResourcesInput{
+		Lister: input.Lister,
+	})
+
+	for i := range resources {
+		r := resources[i]
+		dumpObject(r, input.LogPath)
+	}
+
+	resources = GetCAPIResources(ctx, GetCAPIResourcesInput{
 		Lister:    input.Lister,
 		Namespace: input.Namespace,
 	})

test/framework/clusterctl/clusterctl_helpers.go

@@ -18,6 +18,7 @@ package clusterctl
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -128,6 +129,35 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init
 			})
 		}
 	}
+
+	certManagerDeployments := framework.GetCertManagerDeployments(ctx, framework.GetControllerDeploymentsInput{
+		Lister: client,
+	})
+	for _, deployment := range certManagerDeployments {
+		fmt.Printf("Pod status of %s/%s:\n%++v\n", deployment.Namespace, deployment.Name, deployment.Status)
+		// framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
+		// 	Getter:     client,
+		// 	Deployment: deployment,
+		// }, intervals...)
+
+		// Start streaming logs from all controller providers
+		framework.WatchDeploymentLogsByName(ctx, framework.WatchDeploymentLogsByNameInput{
+			GetLister:  client,
+			Cache:      input.ClusterProxy.GetCache(ctx),
+			ClientSet:  input.ClusterProxy.GetClientSet(),
+			Deployment: deployment,
+			LogPath:    filepath.Join(input.LogFolder, "logs", deployment.GetNamespace()),
+		})
+
+		// if !input.DisableMetricsCollection {
+		// 	framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{
+		// 		GetLister:   client,
+		// 		ClientSet:   input.ClusterProxy.GetClientSet(),
+		// 		Deployment:  deployment,
+		// 		MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()),
+		// 	})
+		// }
+	}
 }
 
 // UpgradeManagementClusterAndWaitInput is the input type for UpgradeManagementClusterAndWait.
@@ -147,6 +177,29 @@ type UpgradeManagementClusterAndWaitInput struct {
 	ClusterctlBinaryPath      string
 }
 
+func (i *UpgradeManagementClusterAndWaitInput) allProviders() []string {
+	providers := []string{i.CoreProvider}
+	for _, p := range i.BootstrapProviders {
+		providers = append(providers, "bootstrap"+p)
+	}
+	for _, p := range i.ControlPlaneProviders {
+		providers = append(providers, "control-plane-"+p)
+	}
+	for _, p := range i.InfrastructureProviders {
+		providers = append(providers, "infrastructure-"+p)
+	}
+	for _, p := range i.IPAMProviders {
+		providers = append(providers, "ipam-"+p)
+	}
+	for _, p := range i.RuntimeExtensionProviders {
+		providers = append(providers, "runtime-extension-"+p)
+	}
+	for _, p := range i.AddonProviders {
+		providers = append(providers, "addon-"+p)
+	}
+	return providers
+}
+
 // UpgradeManagementClusterAndWait upgrades provider a management cluster using clusterctl, and waits for the cluster to be ready.
 func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagementClusterAndWaitInput, intervals ...interface{}) {
 	Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeManagementClusterAndWait")
@@ -213,6 +266,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 			MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()),
 		})
 	}
+
+	// log.Logf("Waiting for cert-manager to inject the new certificates to webhook relevant objects")
+	// framework.WaitForProviderCAInjection(ctx, client, filepath.Join(input.LogFolder, "foo", "crds"))
 }
 
 // ApplyClusterTemplateAndWaitInput is the input type for ApplyClusterTemplateAndWait.

test/framework/controller_helpers.go

@@ -21,6 +21,7 @@ import (
 
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // GetControllerDeploymentsInput is the input for GetControllerDeployments.
@@ -46,6 +47,23 @@ func GetControllerDeployments(ctx context.Context, input GetControllerDeployment
 	return deployments
 }
 
+// GetCertManagerDeployments returns all the deployment for the cluster API controllers existing in a management cluster.
+func GetCertManagerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment {
+	deploymentList := &appsv1.DeploymentList{}
+	Eventually(func() error {
+		return input.Lister.List(ctx, deploymentList, client.InNamespace("cert-manager"))
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers")
+
+	deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items))
+	for i := range deploymentList.Items {
+		d := &deploymentList.Items[i]
+		if !skipDeployment(d, input.ExcludeNamespaces) {
+			deployments = append(deployments, d)
+		}
+	}
+	return deployments
+}
+
 func skipDeployment(d *appsv1.Deployment, excludeNamespaces []string) bool {
 	if !d.DeletionTimestamp.IsZero() {
 		return true

test/framework/management_cluster_helpers.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	. "github.com/onsi/gomega"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/framework/internal/log"
+)
+
+// WaitForProviderCAInjection tries to query a list of all CRD objects for a provider
+// to ensure the webhooks are correctly setup, especially that cert-manager did inject
+// the up-to-date CAs into the relevant objects.
+func WaitForProviderCAInjection(ctx context.Context, lister Lister, outpath string) {
+	crdList := &apiextensionsv1.CustomResourceDefinitionList{}
+	Eventually(func() error {
+		return lister.List(ctx, crdList, client.HasLabels{clusterv1.ProviderNameLabel})
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get crds of providers")
+
+	outDir := outpath
+	Expect(os.MkdirAll(outDir, 0750)).To(Succeed())
+
+	for i := range crdList.Items {
+		crd := crdList.Items[i]
+		data, err := yaml.Marshal(crd)
+		Expect(err).ToNot(HaveOccurred())
+		outFile := filepath.Join(outDir, crd.Name+".yaml")
+		log.Logf("Writing crd to %s", outpath)
+		Expect(os.WriteFile(outFile, data, 0600)).To(Succeed())
+		// Use all versions so we also test conversion webhooks
+		for _, version := range crd.Spec.Versions {
+			if !version.Served {
+				continue
+			}
+			gvk := schema.GroupVersionKind{
+				Group:   crd.Spec.Group,
+				Version: version.Name,
+				Kind:    crd.Spec.Names.Kind,
+			}
+			log.Logf("Checking crd %s - %s", gvk, time.Now())
+			list := &unstructured.UnstructuredList{}
+			list.SetGroupVersionKind(gvk)
+			Eventually(func() error {
+				return lister.List(ctx, list)
+			}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get objects for crd")
+		}
+	}
+}

test/framework/ownerreference_helpers.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strings"
 	"time"
 
 	. "github.com/onsi/gomega"
@@ -112,6 +113,12 @@ func AssertOwnerReferences(namespace, kubeconfigPath string, assertFuncs ...map[
 		ctx := context.Background()
 
 		graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, kubeconfigPath)
+		// Sometimes the conversion-webhooks are not ready yet / cert-managers ca-injector
+		// may not yet have injected the new ca bundle after the upgrade.
+		// If this is the case we return an error to retry.
+		if err != nil && strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+			return err
+		}
 		Expect(err).ToNot(HaveOccurred())
 		for _, v := range graph {
 			if _, ok := allAssertFuncs[v.Object.Kind]; !ok {