fix: use limited permissions for member clusters in charts and E2E en…

…vironment (#611)

Merged to unblock progress; approval acquired previously. If there's any concern, please let me know.

Makefile

@@ -161,6 +161,7 @@ install-hub-agent-helm:
 .PHONY: e2e-v1alpha1-hub-kubeconfig-secret
 e2e-v1alpha1-hub-kubeconfig-secret:
 	kind export kubeconfig --name $(HUB_KIND_CLUSTER_NAME)
+	kubectl apply -f test/e2e/v1alpha1/hub-agent-sa-secret.yaml
 	TOKEN=$$(kubectl get secret hub-kubeconfig-secret -n fleet-system -o jsonpath='{.data.token}' | base64 -d) ;\
 	kind export kubeconfig --name $(MEMBER_KIND_CLUSTER_NAME) ;\
 	kubectl delete secret hub-kubeconfig-secret --ignore-not-found ;\

charts/hub-agent/values.yaml

@@ -31,8 +31,5 @@ tolerations: []
 
 affinity: {}
 
-secret:
-  name: hub-kubeconfig-secret
-
 enableV1Alpha1APIs: true
 enableV1Beta1APIs: false

test/e2e/framework/cluster.go

@@ -24,19 +24,21 @@ var (
 
 // Cluster object defines the required clients based on the kubeconfig of the test cluster.
 type Cluster struct {
-	Scheme                *runtime.Scheme
-	KubeClient            client.Client
-	ImpersonateKubeClient client.Client
-	DynamicClient         dynamic.Interface
-	ClusterName           string
-	HubURL                string
-	RestMapper            meta.RESTMapper
+	Scheme                                   *runtime.Scheme
+	KubeClient                               client.Client
+	ImpersonateKubeClient                    client.Client
+	DynamicClient                            dynamic.Interface
+	ClusterName                              string
+	PresentingServiceAccountInHubClusterName string
+	HubURL                                   string
+	RestMapper                               meta.RESTMapper
 }
 
-func NewCluster(name string, scheme *runtime.Scheme) *Cluster {
+func NewCluster(name, svcAccountName string, scheme *runtime.Scheme) *Cluster {
 	return &Cluster{
-		Scheme:      scheme,
-		ClusterName: name,
+		Scheme:                                   scheme,
+		ClusterName:                              name,
+		PresentingServiceAccountInHubClusterName: svcAccountName,
 	}
 }
 

test/e2e/setup.sh

@@ -73,10 +73,29 @@ helm install hub-agent ../../charts/hub-agent/ \
 
 # Instal the member agent and related components to the member clusters
 
-# Retrieve an access token from the hub cluster
-TOKEN=$(kubectl get secret hub-kubeconfig-secret -n fleet-system -o jsonpath='{.data.token}' | base64 -d)
+# Set up a service account for each member in the hub cluster.
+#
+# Note that these service account has no permission set up at all; the authorization will be
+# configured by the hub agent.
 for i in "${MEMBER_CLUSTERS[@]}"
 do
+    kubectl create serviceaccount fleet-member-agent-$i -n fleet-system
+    cat <<EOF | kubectl apply -f -
+    apiVersion: v1
+    kind: Secret
+    metadata:
+        name: fleet-member-agent-$i-sa
+        namespace: fleet-system
+        annotations:
+            kubernetes.io/service-account.name: fleet-member-agent-$i
+    type: kubernetes.io/service-account-token
+EOF
+done
+
+for i in "${MEMBER_CLUSTERS[@]}"
+do
+    kind export kubeconfig --name $HUB_CLUSTER
+    TOKEN=$(kubectl get secret fleet-member-agent-$i-sa -n fleet-system -o jsonpath='{.data.token}' | base64 -d)
     kind export kubeconfig --name "$i"
     kubectl delete secret hub-kubeconfig-secret --ignore-not-found
     kubectl create secret generic hub-kubeconfig-secret --from-literal=token=$TOKEN

test/e2e/setup_test.go

@@ -44,7 +44,14 @@ const (
 	memberCluster5LeftName        = "kind-left-cluster"
 	memberCluster6NonExistentName = "kind-non-existent-cluster"
 
-	hubClusterSAName = "hub-agent-sa"
+	// The names of the service accounts used by specific member clusters.
+	//
+	// Note that these names must also match those in `setup.sh`.
+	memberCluster1EastProdSAName   = "fleet-member-agent-cluster-1"
+	memberCluster2EastCanarySAName = "fleet-member-agent-cluster-2"
+	memberCluster3WestProdSAName   = "fleet-member-agent-cluster-3"
+
+	hubClusterSAName = "fleet-hub-agent"
 	fleetSystemNS    = "fleet-system"
 
 	kubeConfigPathEnvVarName = "KUBECONFIG"
@@ -170,27 +177,27 @@ func beforeSuiteForAllProcesses() {
 	Expect(os.Getenv(kubeConfigPathEnvVarName)).NotTo(BeEmpty(), "Required environment variable KUBECONFIG is not set")
 
 	// Initialize the cluster objects and their clients.
-	hubCluster = framework.NewCluster(hubClusterName, scheme)
+	hubCluster = framework.NewCluster(hubClusterName, "", scheme)
 	Expect(hubCluster).NotTo(BeNil(), "Failed to initialize cluster object")
 	framework.GetClusterClient(hubCluster)
 	hubClient = hubCluster.KubeClient
 	Expect(hubClient).NotTo(BeNil(), "Failed to initialize client for accessing Kubernetes cluster")
 	impersonateHubClient = hubCluster.ImpersonateKubeClient
 	Expect(impersonateHubClient).NotTo(BeNil(), "Failed to initialize impersonate client for accessing Kubernetes cluster")
 
-	memberCluster1EastProd = framework.NewCluster(memberCluster1EastProdName, scheme)
+	memberCluster1EastProd = framework.NewCluster(memberCluster1EastProdName, memberCluster1EastProdSAName, scheme)
 	Expect(memberCluster1EastProd).NotTo(BeNil(), "Failed to initialize cluster object")
 	framework.GetClusterClient(memberCluster1EastProd)
 	memberCluster1EastProdClient = memberCluster1EastProd.KubeClient
 	Expect(memberCluster1EastProdClient).NotTo(BeNil(), "Failed to initialize client for accessing Kubernetes cluster")
 
-	memberCluster2EastCanary = framework.NewCluster(memberCluster2EastCanaryName, scheme)
+	memberCluster2EastCanary = framework.NewCluster(memberCluster2EastCanaryName, memberCluster2EastCanarySAName, scheme)
 	Expect(memberCluster2EastCanary).NotTo(BeNil(), "Failed to initialize cluster object")
 	framework.GetClusterClient(memberCluster2EastCanary)
 	memberCluster2EastCanaryClient = memberCluster2EastCanary.KubeClient
 	Expect(memberCluster2EastCanaryClient).NotTo(BeNil(), "Failed to initialize client for accessing Kubernetes cluster")
 
-	memberCluster3WestProd = framework.NewCluster(memberCluster3WestProdName, scheme)
+	memberCluster3WestProd = framework.NewCluster(memberCluster3WestProdName, memberCluster3WestProdSAName, scheme)
 	Expect(memberCluster3WestProd).NotTo(BeNil(), "Failed to initialize cluster object")
 	framework.GetClusterClient(memberCluster3WestProd)
 	memberCluster3WestProdClient = memberCluster3WestProd.KubeClient

test/e2e/utils_test.go

@@ -150,7 +150,7 @@ func markMemberClusterAsLeft(name string) {
 func setAllMemberClustersToJoin() {
 	for idx := range allMemberClusters {
 		memberCluster := allMemberClusters[idx]
-		createMemberCluster(memberCluster.ClusterName, hubClusterSAName, labelsByClusterName[memberCluster.ClusterName])
+		createMemberCluster(memberCluster.ClusterName, memberCluster.PresentingServiceAccountInHubClusterName, labelsByClusterName[memberCluster.ClusterName])
 	}
 }
 

test/e2e/v1alpha1/e2e_test.go

@@ -36,8 +36,8 @@ import (
 var (
 	hubClusterName    = "kind-hub-testing"
 	memberClusterName = "kind-member-testing"
-	HubCluster        = framework.NewCluster(hubClusterName, scheme)
-	MemberCluster     = framework.NewCluster(memberClusterName, scheme)
+	HubCluster        = framework.NewCluster(hubClusterName, "", scheme)
+	MemberCluster     = framework.NewCluster(memberClusterName, "", scheme)
 	hubURL            string
 	scheme            = runtime.NewScheme()
 	mc                *v1alpha1.MemberCluster

test/e2e/v1alpha1/hub-agent-sa-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: hub-kubeconfig-secret
+    namespace: fleet-system
+    annotations:
+        kubernetes.io/service-account.name: hub-agent-sa
+type: kubernetes.io/service-account-token