[WIP]e2e placement policy with other scheduler plugins

.github/workflows/workflow.yml

@@ -50,7 +50,7 @@ jobs:
   e2e-test:
     name: "E2E Test"
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
     steps:
     - name: Set up Go 1.17
       uses: actions/setup-go@v2

test/e2e/main_test.go

@@ -44,6 +44,7 @@ func TestMain(m *testing.M) {
 	os.Exit(testenv.Run(m))
 }
 
+//deploy placement policy manifest
 func deploySchedulerManifest() env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 		wd, err := os.Getwd()

test/e2e/plugin_cosched_combinations_test.go

@@ -0,0 +1,104 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestMustStrictCoscheduling(t *testing.T) {
+	deploymentFeat := features.New("Test Must Strict Placement policy with Coscheduling plugins").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			// deploy placement policy config
+			if err := deploySchedulerConfig(cfg.KubeconfigFile(), cfg.Namespace(), "examples", "v1alpha1_placementpolicy_strict_must.yml"); err != nil {
+				t.Error("Failed to deploy placement policy config", err)
+			}
+
+			lables := map[string]string{
+				"app":                              "nginx",
+				"pod-group.scheduling.sigs.k8s.io": "nginx",
+			}
+
+			wd, err := os.Getwd()
+			if err != nil {
+				t.Error(err)
+			}
+			pluginsResourceAbsolutePath, err := filepath.Abs(filepath.Join(wd, "plugins/coscheduling"))
+			if err != nil {
+				t.Error(err)
+			}
+
+			// deploy Coscheduling config
+			if err := KubectlApply(cfg.KubeconfigFile(), "scheduler-plugins", []string{"-f", fmt.Sprintf("%s/%s", pluginsResourceAbsolutePath, "")}); err != nil {
+				t.Error("Failed to deploy coscheduling config", err)
+			}
+
+			// deploy a sample replicaset
+			statefulset := newStatefulSet(cfg.Namespace(), "statefulset-test", 6, lables)
+			if err := cfg.Client().Resources().Create(ctx, statefulset); err != nil {
+				t.Error("Failed to create statefulset", err)
+			}
+
+			return ctx
+		}).
+		Assess("Pods successfully assigned to the right nodes with Must Strict and Coscheduling plugins option", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client, err := cfg.NewClient()
+			if err != nil {
+				t.Error("Failed to create new client", err)
+			}
+			resultStatefulset := appsv1.StatefulSet{
+				ObjectMeta: metav1.ObjectMeta{Name: "statefulset-test", Namespace: cfg.Namespace()},
+			}
+
+			if err := wait.For(conditions.New(client.Resources()).ResourceMatch(&resultStatefulset, func(object k8s.Object) bool {
+				s := object.(*appsv1.StatefulSet)
+				return s.Status.ReadyReplicas == 3
+			}), wait.WithTimeout(time.Minute*2)); err != nil {
+				t.Error("Failed to deploy a statefulset", err)
+			}
+
+			var pods corev1.PodList
+			if err := client.Resources().List(ctx, &pods, resources.WithLabelSelector(labels.FormatLabels(map[string]string{"app": "nginx", "pod-group.scheduling.sigs.k8s.io": "nginx"}))); err != nil {
+				t.Error("cannot get list of pods", err)
+			}
+
+			for i := range pods.Items {
+				if pods.Items[i].Spec.NodeName != "placement-policy-worker3" {
+					continue
+				} else {
+					t.Error("pods assigned to the wrong node", err)
+				}
+			}
+
+			return context.WithValue(ctx, "statefulset-test", &resultStatefulset)
+		}).
+		Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client, err := cfg.NewClient()
+			if err != nil {
+				t.Error("failed to create new Client", err)
+			}
+			dep := ctx.Value("statefulset-test").(*appsv1.StatefulSet)
+			if err := client.Resources().Delete(ctx, dep); err != nil {
+				t.Error("failed to delete Statefulset", err)
+			}
+			return ctx
+		}).Feature()
+	testenv.Test(t, deploymentFeat)
+}

test/e2e/plugins/coscheduling/configmap.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scheduler-config
+  namespace: scheduler-plugins
+data:
+  scheduler-config.yaml: |
+    apiVersion: kubescheduler.config.k8s.io/v1beta1
+    kind: KubeSchedulerConfiguration
+    leaderElection:
+      leaderElect: false
+    profiles:
+    - schedulerName: scheduler-plugins-scheduler
+      plugins:
+        queueSort:
+          enabled:
+          - name: Coscheduling
+          disabled:
+          - name: "*"
+        preFilter:
+          enabled:
+          - name: Coscheduling
+        permit:
+          enabled:
+          - name: Coscheduling
+        reserve:
+          enabled:
+          - name: Coscheduling
+        postBind:
+          enabled:
+          - name: Coscheduling
+      pluginConfig:
+      - name: Coscheduling
+        args:
+          permitWaitingTimeSeconds: 10
+          deniedPGExpirationTimeSeconds: 3

test/e2e/plugins/coscheduling/cosched.yaml

@@ -0,0 +1,7 @@
+apiVersion: scheduling.sigs.k8s.io/v1alpha1
+kind: PodGroup
+metadata:
+  name: nginx
+spec:
+  scheduleTimeoutSeconds: 10
+  minMember: 3

test/e2e/plugins/coscheduling/deployment.yaml

@@ -0,0 +1,74 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: scheduler-plugins-controller
+  namespace: scheduler-plugins
+  labels:
+    app: scheduler-plugins-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: scheduler-plugins-controller
+  template:
+    metadata:
+      labels:
+        app: scheduler-plugins-controller
+    spec:
+      serviceAccount: scheduler-plugins-controller
+      containers:
+      - name: scheduler-plugins-controller
+        image: k8s.gcr.io/scheduler-plugins/controller:v0.20.10
+        imagePullPolicy: IfNotPresent
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    component: scheduler
+  name: scheduler-plugins-scheduler
+  namespace: scheduler-plugins
+spec:
+  selector:
+    matchLabels:
+      component: scheduler
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        component: scheduler
+    spec:
+      serviceAccountName: scheduler-plugins-scheduler
+      containers:
+      - command:
+        - /bin/kube-scheduler
+        - --address=0.0.0.0
+        - --leader-elect=false
+        - --config=/etc/kubernetes/scheduler-config.yaml
+        - --scheduler-name=scheduler-plugins-scheduler
+        image: k8s.gcr.io/scheduler-plugins/kube-scheduler:v0.20.10
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10251
+          initialDelaySeconds: 15
+        name: scheduler-plugins-scheduler
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10251
+        resources:
+          requests:
+            cpu: '0.1'
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: scheduler-config
+          mountPath: /etc/kubernetes
+          readOnly: true
+      hostNetwork: false
+      hostPID: false
+      volumes:
+      - name: scheduler-config
+        configMap:
+          name: scheduler-config

test/e2e/plugins/coscheduling/rbac.yaml

@@ -0,0 +1,100 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: scheduler-plugins-scheduler
+rules:
+- apiGroups: ["", "events.k8s.io"]
+  resources: ["events"]
+  verbs: ["create", "patch", "update"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create"]
+- apiGroups: ["coordination.k8s.io"]
+  resourceNames: ["kube-scheduler"]
+  resources: ["leases"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resourceNames: ["kube-scheduler"]
+  resources: ["endpoints"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["delete", "get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["bindings", "pods/binding"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["patch", "update"]
+- apiGroups: [""]
+  resources: ["replicationcontrollers", "services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["persistentvolumeclaims", "persistentvolumes"]
+  verbs: ["get", "list", "watch", "patch", "update"]
+- apiGroups: ["authentication.k8s.io"]
+  resources: ["tokenreviews"]
+  verbs: ["create"]
+- apiGroups: ["authorization.k8s.io"]
+  resources: ["subjectaccessreviews"]
+  verbs: ["create"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["csinodes", "storageclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["scheduling.sigs.k8s.io"]
+  resources: ["podgroups", "elasticquotas"]
+  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins-scheduler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: scheduler-plugins-scheduler
+subjects:
+- kind: ServiceAccount
+  name: scheduler-plugins-scheduler
+  namespace: scheduler-plugins
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins-controller
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["scheduling.sigs.k8s.io"]
+  resources: ["podgroups", "elasticquotas"]
+  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins-controller
+subjects:
+- kind: ServiceAccount
+  name: scheduler-plugins-controller
+  namespace: scheduler-plugins
+roleRef:
+  kind: ClusterRole
+  name: scheduler-plugins-controller
+  apiGroup: rbac.authorization.k8s.io
+