test: LoadTest Setup

hack/loadtest/main.go

@@ -22,42 +22,41 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
-	workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1"
 
-	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
+	fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
 	"go.goms.io/fleet/hack/loadtest/util"
 )
 
 var (
-	scheme = runtime.NewScheme()
+	scheme  = runtime.NewScheme()
+	crpFile = "test-crp.yaml"
 )
 
 var (
 	placementDeadline   = flag.Int("placement-deadline-second", 300, "The deadline for a placement to be applied (in seconds)")
 	pollInterval        = flag.Int("poll-interval-millisecond", 250, "The poll interval for verification (in milli-second)")
 	maxCurrentPlacement = flag.Int("max-current-placement", 10, "The number of current placement load.")
-	loadTestLength      = flag.Int("load-test-length-minute", 15, "The length of the load test in miniutes.")
-	clusterNames        util.ClusterNames
+	loadTestLength      = flag.Int("load-test-length-minute", 20, "The length of the load test in minutes.")
+	clusterNames        util.ClusterNames //will be used for PickFixed scenario, otherwise will apply to all clusters
 )
 
 func init() {
 	klog.InitFlags(nil)
+	flag.StringVar(&crpFile, "crp-file", "test-crp.yaml", "The CRP yaml file.")
 	utilrand.Seed(time.Now().UnixNano())
 
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-	utilruntime.Must(fleetv1alpha1.AddToScheme(scheme))
-	utilruntime.Must(workv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(fleetv1beta1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 
 func main() {
-	flag.Var(&clusterNames, "cluster", "The name of a member cluster")
 	flag.Parse()
 	defer klog.Flush()
 
-	klog.InfoS("start to run placement load test", "pollInterval", *pollInterval, "placementDeadline", *placementDeadline, "maxCurrentPlacement", *maxCurrentPlacement, "clusterNames", clusterNames)
+	klog.InfoS("start to run placement load test", "crpFile", crpFile, "pollInterval", *pollInterval, "placementDeadline", *placementDeadline, "maxCurrentPlacement", *maxCurrentPlacement, "clusterNames", clusterNames)
 	config := config.GetConfigOrDie()
-	config.QPS, config.Burst = float32(100), 500
+	config.QPS, config.Burst = float32(100), 500 //queries per second, max # of queries queued at once
 	hubClient, err := client.New(config, client.Options{
 		Scheme: scheme,
 	})
@@ -75,7 +74,7 @@ func main() {
 	// setup prometheus server
 	http.Handle("/metrics", promhttp.Handler())
 	/* #nosec */
-	if err = http.ListenAndServe(":4848", http.TimeoutHandler(nil, 10*time.Second, "request time out")); err != nil {
+	if err = http.ListenAndServe(":4848", nil); err != nil {
 		panic(err)
 	}
 }
@@ -101,8 +100,8 @@ func runLoadTest(ctx context.Context, config *rest.Config) {
 				case <-ctx.Done():
 					return
 				default:
-					if err = util.MeasureOnePlacement(ctx, hubClient, time.Duration(*placementDeadline)*time.Second, time.Duration(*pollInterval)*time.Millisecond, *maxCurrentPlacement, clusterNames); err != nil {
-						klog.ErrorS(err, "placement load test failed")
+					if err = util.MeasureOnePlacement(ctx, hubClient, time.Duration(*placementDeadline)*time.Second, time.Duration(*pollInterval)*time.Millisecond, *maxCurrentPlacement, clusterNames, crpFile); err != nil {
+						klog.ErrorS(err, "placement load test finished")
 					}
 				}
 			}

hack/loadtest/test-crp.yaml

@@ -0,0 +1,12 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: crp
+spec:
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 100%
+      maxSurge: 25%
+      unavailablePeriodSeconds: 60
+    revisionHistoryLimit: 15

hack/loadtest/test-crp1.yaml

@@ -0,0 +1,52 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: crp ## updated in code to be load-test-placement-utilrand.String(10)
+spec:
+  resourceSelectors:
+    - group:    apiextensions.k8s.io
+      kind:     CustomResourceDefinition
+      name:     clonesets.apps.kruise.io
+      version:  v1
+  policy:
+    placementType: PickFixed
+    clusterNames:
+      - scale-test-eastus-21-mc-fm
+      - scale-test-eastus-53-mc-fm
+      - scale-test-eastus-67-mc-fm
+      - scale-test-eastus-79-mc-fm
+      - scale-test-westus-10-mc-fm
+      - scale-test-westus-16-mc-fm
+      - scale-test-westus-2-mc-fm
+      - scale-test-westus-22-mc-fm
+      - scale-test-westus-26-mc-fm
+      - scale-test-westus-28-mc-fm
+      - scale-test-westus-30-mc-fm
+      - scale-test-westus-36-mc-fm
+      - scale-test-westus-4-mc-fm
+      - scale-test-westus-40-mc-fm
+      - scale-test-westus-44-mc-fm
+      - scale-test-westus-48-mc-fm
+      - scale-test-westus-52-mc-fm
+      - scale-test-westus-54-mc-fm
+      - scale-test-westus-56-mc-fm
+      - scale-test-westus-58-mc-fm
+      - scale-test-westus-6-mc-fm
+      - scale-test-westus-64-mc-fm
+      - scale-test-westus-70-mc-fm
+      - scale-test-westus-74-mc-fm
+      - scale-test-westus-78-mc-fm
+      - scale-test-westus-80-mc-fm
+      - scale-test-westus-82-mc-fm
+      - scale-test-westus-86-mc-fm
+      - scale-test-westus-90-mc-fm
+      - scale-test-westus-92-mc-fm
+      - scale-test-westus-94-mc-fm
+      - scale-test-westus-96-mc-fm
+  strategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxUnavailable: 100%
+        maxSurge: 25%
+        unavailablePeriodSeconds: 60
+  revisionHistoryLimit: 15

hack/loadtest/test-crp2.yaml

@@ -0,0 +1,37 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: crp
+spec:
+  resourceSelectors:
+    - group: apiextensions.k8s.io
+      kind: CustomResourceDefinition
+      name: clonesets.apps.kruise.io
+      version: v1
+  policy:
+    placementType: PickN
+    numberOfClusters: 20
+    affinity:
+      clusterAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 20
+          preference:
+            labelSelector:
+              matchExpressions:
+                - key: system
+                  operator: DoesNotExist
+        - weight: -20
+          preference:
+            labelSelector:
+              matchExpressions:
+                - key: fleet.azure.com/location
+                  operator: In
+                  values:
+                    - eastus
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 100%
+      maxSurge: 25%
+      unavailablePeriodSeconds: 60
+  revisionHistoryLimit: 15

hack/loadtest/test-crp3.yaml

@@ -0,0 +1,26 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: crp
+spec:
+  resourceSelectors:
+    - group: apiextensions.k8s.io
+      kind: CustomResourceDefinition
+      name: clonesets.apps.kruise.io
+      version: v1
+  policy:
+    placementType: PickAll
+    affinity:
+      clusterAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          clusterSelectorTerms:
+            - labelSelector:
+                matchLabels:
+                  fleet.azure.com/location: westus
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 100%
+      maxSurge: 25%
+      unavailablePeriodSeconds: 60
+  revisionHistoryLimit: 15

hack/loadtest/test-crp4.yaml

@@ -0,0 +1,24 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: crp
+spec:
+  resourceSelectors:
+    - group: apiextensions.k8s.io
+      kind: CustomResourceDefinition
+      name: clonesets.apps.kruise.io
+      version: v1
+  policy:
+    placementType: PickN
+    numberOfClusters: 3
+    topologySpreadConstraints:
+      - maxSkew: 2
+        topologyKey: env
+        whenUnsatisfiable: DoNotSchedule
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 100%
+      maxSurge: 25%
+      unavailablePeriodSeconds: 60
+  revisionHistoryLimit: 15

hack/loadtest/util/help.go

@@ -2,18 +2,22 @@ package util
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	"go.goms.io/fleet/apis/v1alpha1"
+	"go.goms.io/fleet/apis/placement/v1beta1"
 )
 
 type ClusterNames []string
@@ -155,27 +159,78 @@ func deleteNamespace(ctx context.Context, hubClient client.Client, namespaceName
 }
 
 func CleanupAll(hubClient client.Client) error {
+	var crps v1beta1.ClusterResourcePlacementList
+	if err := hubClient.List(context.Background(), &crps); err != nil {
+		klog.ErrorS(err, "failed to list namespace")
+		return err
+	}
+	for i := 0; i < len(crps.Items); i++ {
+		deleteTimeoutCount.Inc()
+		updateTimeoutCount.Inc()
+	}
+
+	diffCount := crpCount.Load() - applySuccessCount.Load()
+	for j := 0; j < int(diffCount); j++ {
+		applyTimeoutCount.Inc()
+	}
+
+	for index := range crps.Items {
+		if err := hubClient.Delete(context.Background(), &crps.Items[index]); err != nil {
+			klog.ErrorS(err, "failed to delete crp", "crp", crps.Items[index].Name)
+		}
+	}
+
 	var namespaces corev1.NamespaceList
 	if err := hubClient.List(context.Background(), &namespaces); err != nil {
 		klog.ErrorS(err, "failed to list namespace")
 		return err
 	}
-	for index, ns := range namespaces.Items {
-		if strings.HasPrefix(ns.Name, nsPrefix) {
+	for index := range namespaces.Items {
+		if strings.HasPrefix(namespaces.Items[index].Name, nsPrefix) {
 			if err := hubClient.Delete(context.Background(), &namespaces.Items[index]); err != nil {
-				klog.ErrorS(err, "failed to delete namespace", "namespace", ns.Name)
+				klog.ErrorS(err, "failed to delete namespace", "namespace", namespaces.Items[index].Name)
 			}
 		}
 	}
-	var crps v1alpha1.ClusterResourcePlacementList
-	if err := hubClient.List(context.Background(), &crps); err != nil {
-		klog.ErrorS(err, "failed to list namespace")
+	return nil
+}
+func getFleetSize(crp v1beta1.ClusterResourcePlacement, clusterNames ClusterNames) (string, ClusterNames, error) {
+	for _, status := range crp.Status.PlacementStatuses {
+		if err := clusterNames.Set(status.ClusterName); err != nil {
+			klog.ErrorS(err, "Failed to set clusterNames.")
+			return "", nil, err
+		}
+	}
+	return strconv.Itoa(len(clusterNames)), clusterNames, nil
+}
+func createCRP(crp *v1beta1.ClusterResourcePlacement, crpFile string, crpName string, nsName string) error {
+	obj, err := readObjFromFile(fmt.Sprintf("hack/loadtest/%s", crpFile), nsName)
+	if err != nil {
+		klog.ErrorS(err, "Failed to read object from file.")
 		return err
 	}
-	for index, crp := range crps.Items {
-		if err := hubClient.Delete(context.Background(), &crps.Items[index]); err != nil {
-			klog.ErrorS(err, "failed to delete crp", "crp", crp.Name)
-		}
+
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, crp)
+	if err != nil {
+		return err
+	}
+
+	crp.Name = crpName
+	crp.Spec.ResourceSelectors = []v1beta1.ClusterResourceSelector{
+		{
+			Group:   "",
+			Version: "v1",
+			Kind:    "Namespace",
+			LabelSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{labelKey: nsName},
+			},
+		},
+		{
+			Group:   apiextensionsv1.GroupName,
+			Version: "v1",
+			Kind:    "CustomResourceDefinition",
+			Name:    "clonesets.apps.kruise.io",
+		},
 	}
 	return nil
 }