update e2e test for apiserver ilb

test/e2e/azure_apiserver_ilb.go

@@ -0,0 +1,115 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/utils/ptr"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure"
+)
+
+// AzureAPIServerILBSpecInput is the input for AzureAPIServerILBSpec.
+type AzureAPIServerILBSpecInput struct {
+	BootstrapClusterProxy framework.ClusterProxy
+	Cluster               *clusterv1.Cluster
+	Namespace             *corev1.Namespace
+	ClusterName           string
+}
+
+// AzureAPIServerILBSpec implements a test that verifies the Azure API server ILB is created.
+func AzureAPIServerILBSpec(ctx context.Context, inputGetter func() AzureAPIServerILBSpecInput) {
+	var (
+		specName    = "azure-apiserver-ilb"
+		input       AzureAPIServerILBSpecInput
+		clusterName = os.Getenv("AZURE_CLUSTER_NAME")
+	)
+
+	input = inputGetter()
+	Expect(input.Namespace).NotTo(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) //nolint:typecheck
+	Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName)
+
+	By("Creating a Kubernetes client to the workload cluster %s", clusterName)
+	workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName)
+	Expect(workloadClusterProxy).NotTo(BeNil())
+
+	By("Creating a Kubernetes client to the management cluster")
+	managementClusterProxy := input.BootstrapClusterProxy.GetManagementCluster(ctx)
+	Expect(managementClusterProxy).NotTo(BeNil())
+
+	// TODO: check if the internal load balancer is created
+	By("1. Fetching new Azure Credentials")
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("2. Getting azureLoabBalancerClient")
+	azureLoadBalancerClient, err := armnetwork.NewLoadBalancersClient(getSubscriptionID(Default), cred, nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("3. Checking if the Azure API Server Internal Load Balancer is created")
+	groupName := os.Getenv(AzureResourceGroup)
+	internalLoadbalancerName := fmt.Sprintf("%s-%s", clusterName, "public-lb-internal")
+
+	backoff := wait.Backoff{
+		Duration: retryBackoffInitialDuration,
+		Factor:   retryBackoffFactor,
+		Jitter:   retryBackoffJitter,
+		Steps:    retryBackoffSteps,
+	}
+	retryFn := func() (bool, error) {
+		resp, err := azureLoadBalancerClient.Get(ctx, groupName, internalLoadbalancerName, "")
+		if err != nil {
+			return false, err
+		}
+
+		internalLoadbalancer := resp.LoadBalancer
+
+		switch ptr.Deref(internalLoadbalancer.Properties.ProvisioningState, "") {
+		case armnetwork.ProvisioningStateSucceeded:
+			return true, nil
+		case armnetwork.ProvisioningStateUpdating:
+			// Wait for operation to complete.
+			return false, nil
+		default:
+			return false, fmt.Errorf("azure internal loadbalancer provisioning failed with state: %q", ptr.Deref(internalLoadbalancer.Properties.ProvisioningState, "(nil)"))
+		}
+	}
+	err = wait.ExponentialBackoff(backoff, retryFn)
+
+	// TODO: deploy a sample deployment on the worker nodes
+	// TODO: verify the worker node's /etc/hosts has the updated DNS entry for the Internal LB for the API Server
+}

test/e2e/azure_test.go

@@ -1155,7 +1155,7 @@ var _ = Describe("Workload cluster creation", func() {
 	})
 
 	Context("Creating a self-managed VM based cluster using API Server ILB feature gate [OPTIONAL][API-Server-ILB]", func() {
-		It("with one controlplane node and three worker nodes", func() {
+		It("with three controlplane node and three worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "apiserver-ilb")
 
 			// Set the environment variables required for the API Server ILB feature gate
@@ -1187,8 +1187,19 @@ var _ = Describe("Workload cluster creation", func() {
 					})
 				}),
 			), result)
-			// TODO: deploy a sample deployment on the worker nodes
-			// TODO: verify the worker node's /etc/hosts has the updated DNS entry for the Internal LB for the API Server
+
+			By("Probing workload cluster with APIServerILB feature gate", func() {
+				AzureAPIServerILBSpec(ctx, func() AzureAPIServerILBSpecInput {
+					return AzureAPIServerILBSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+						SkipCleanup:           skipCleanup,
+					}
+				})
+			})
+
+			By("PASSED!")
 		})
 	})