test: setup e2e tests env for atm feature (#227)

.github/workflows/e2e-tests.yml

@@ -37,6 +37,13 @@ jobs:
       fail-fast: false
       matrix:
         network-setting: [shared-vnet, peered-vnet,  dynamic-ip-allocation]
+        include:
+          - network-setting: shared-vnet
+            enable-traffic-manager: true
+          - network-setting: peered-vnet
+            enable-traffic-manager: false
+          - network-setting: dynamic-ip-allocation
+            enable-traffic-manager: false
     runs-on: ubuntu-latest
     needs: [
       detect-noop,
@@ -92,6 +99,7 @@ jobs:
           AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }}
           AZURE_NETWORK_SETTING: ${{ matrix.network-setting }}
           AZURE_RESOURCE_GROUP: ${{ env.AZURE_RESOURCE_GROUP }}
+          ENABLE_TRAFFIC_MANAGER: ${{ matrix.enable-traffic-manager }}
       - name: Run e2e tests
         run: |
           make e2e-tests

test/scripts/aks-shared-vnet.sh

@@ -37,7 +37,8 @@ az network vnet subnet create \
     --address-prefixes 10.2.0.0/16
 
 # Create aks member cluster1.
-az aks create \
+if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then
+  az aks create \
     --location $MEMBER_1_LOCATION \
     --resource-group $RESOURCE_GROUP \
     --name $MEMBER_CLUSTER_1 \
@@ -46,9 +47,22 @@ az aks create \
     --network-plugin azure \
     --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_1_SUBNET" \
     --no-wait
+else
+  az aks create \
+     --location $MEMBER_1_LOCATION \
+     --resource-group $RESOURCE_GROUP \
+     --name $MEMBER_CLUSTER_1 \
+     --node-count $NODE_COUNT \
+     --generate-ssh-keys \
+     --network-plugin azure \
+     --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_1_SUBNET" \
+     --enable-managed-identity --assign-identity ${MEMBER_CLUSTER_1_AKS_IDENTITY_ID} --assign-kubelet-identity ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY_ID} \
+     --no-wait
+fi
 
 # Create aks member cluster2.
-az aks create \
+if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then
+  az aks create \
     --location $MEMBER_2_LOCATION \
     --resource-group $RESOURCE_GROUP \
     --name $MEMBER_CLUSTER_2 \
@@ -57,3 +71,15 @@ az aks create \
     --network-plugin azure \
     --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_2_SUBNET" \
     --no-wait
+else
+  az aks create \
+      --location $MEMBER_2_LOCATION \
+      --resource-group $RESOURCE_GROUP \
+      --name $MEMBER_CLUSTER_2 \
+      --node-count $NODE_COUNT \
+      --generate-ssh-keys \
+      --network-plugin azure \
+      --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_2_SUBNET" \
+      --enable-managed-identity --assign-identity ${MEMBER_CLUSTER_2_AKS_IDENTITY_ID} --assign-kubelet-identity ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY_ID} \
+      --no-wait
+fi

test/scripts/bootstrap.sh

@@ -7,6 +7,13 @@ set -x
 # Check required variables.
 [[ -z "${AZURE_SUBSCRIPTION_ID}" ]] && echo "AZURE_SUBSCRIPTION_ID is not set" && exit 1
 
+export ENABLE_TRAFFIC_MANAGER=${ENABLE_TRAFFIC_MANAGER:-"false"}
+
+if [ "$ENABLE_TRAFFIC_MANAGER" == "true" ] && [ "$AZURE_NETWORK_SETTING" != "shared-vnet" ]; then
+  echo "error: setting up traffic manager testing env is not supported when AZURE_NETWORK_SETTING is not shared-vnet"
+  exit 1
+fi
+
 az account set -s ${AZURE_SUBSCRIPTION_ID}
 
 # Create resource group to host hub and member clusters.
@@ -33,26 +40,84 @@ az acr create -g $RESOURCE_GROUP -n $REGISTRY_NAME --sku standard --tags "source
 az acr update --name $REGISTRY_NAME --anonymous-pull-enabled -g $RESOURCE_GROUP
 az acr login -n $REGISTRY_NAME
 export REGISTRY=$REGISTRY_NAME.azurecr.io
-export TAG=`git rev-parse --short=7 HEAD`
+export TAG=$(git rev-parse --short=7 HEAD)
 make push
 
 # Create hub and member clusters and wait until all clusters are ready.
-export HUB_CLUSTER=hub
-export MEMBER_CLUSTER_1=member-1
-export MEMBER_CLUSTER_2=member-2
+export HUB_CLUSTER=${HUB_CLUSTER:-"hub"}
+export MEMBER_CLUSTER_1=${MEMBER_CLUSTER_1:-"member-1"}
+export MEMBER_CLUSTER_2=${MEMBER_CLUSTER_2:-"member-2"}
 export NODE_COUNT=2
 
-# Create aks hub cluster
-az aks create \
-    --location $LOCATION \
-    --resource-group $RESOURCE_GROUP \
-    --name $HUB_CLUSTER \
-    --node-count $NODE_COUNT \
-    --generate-ssh-keys \
-    --enable-aad \
-    --enable-azure-rbac \
-    --network-plugin azure \
-    --no-wait
+export HUB_CLUSTER_AKS_ID_NAME=${HUB_CLUSTER_AKS_ID_NAME:-"fleet-net-hub-aks-id"}
+export HUB_CLUSTER_AKS_KUBELET_ID_NAME=${HUB_CLUSTER_AKS_KUBELET_ID_NAME:-"fleet-net-hub-aks-kubelet-id"}
+export MEMBER_CLUSTER_1_AKS_ID_NAME=${MEMBER_CLUSTER_1_AKS_ID_NAME:-"fleet-net-member-1-aks-id"}
+export MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME=${MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME:-"fleet-net-member-1-aks-kubelet-id"}
+export MEMBER_CLUSTER_2_AKS_ID_NAME=${MEMBER_CLUSTER_2_AKS_ID_NAME:-"fleet-net-member-2-aks-id"}
+export MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME=${MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME:-"fleet-net-member-2-aks-kubelet-id"}
+
+
+
+if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then
+  # Create aks hub cluster
+  echo "Creating aks cluster: ${HUB_CLUSTER}"
+  az aks create \
+       --location $LOCATION \
+       --resource-group $RESOURCE_GROUP \
+       --name $HUB_CLUSTER \
+       --node-count $NODE_COUNT \
+       --generate-ssh-keys \
+       --enable-aad \
+       --enable-azure-rbac \
+       --network-plugin azure \
+       --no-wait
+else
+  # Create aks identity
+  echo "Creating hub control-plane identity: ${HUB_CLUSTER_AKS_ID_NAME}"
+  HUB_CLUSTER_AKS_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${HUB_CLUSTER_AKS_ID_NAME})
+  echo "Creating hub kubelet identity: ${HUB_CLUSTER_AKS_KUBELET_ID_NAME}"
+  HUB_CLUSTER_AKS_KUBELET_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${HUB_CLUSTER_AKS_KUBELET_ID_NAME})
+
+  HUB_CLUSTER_AKS_IDENTITY_ID=$(echo ${HUB_CLUSTER_AKS_IDENTITY} | jq -r '. | .id')
+  HUB_CLUSTER_AKS_KUBELET_IDENTITY_ID=$(echo ${HUB_CLUSTER_AKS_KUBELET_IDENTITY} | jq -r '. | .id')
+  HUB_CLUSTER_KUBELET_PRINCIPAL_ID=$(echo ${HUB_CLUSTER_AKS_KUBELET_IDENTITY} | jq -r '. | .principalId')
+  HUB_CLUSTER_KUBELET_CLIENT_ID=$(echo ${HUB_CLUSTER_AKS_KUBELET_IDENTITY} | jq -r '. | .clientId')
+
+  echo "Creating aks cluster: ${HUB_CLUSTER}"
+  az aks create \
+         --location $LOCATION \
+         --resource-group $RESOURCE_GROUP \
+         --name $HUB_CLUSTER \
+         --node-count $NODE_COUNT \
+         --generate-ssh-keys \
+         --enable-aad \
+         --enable-azure-rbac \
+         --network-plugin azure \
+         --enable-managed-identity --assign-identity ${HUB_CLUSTER_AKS_IDENTITY_ID} --assign-kubelet-identity ${HUB_CLUSTER_AKS_KUBELET_IDENTITY_ID} \
+         --no-wait
+
+  # Create aks identity for member-1
+  echo "Creating member-1 control-plane identity: ${MEMBER_CLUSTER_1_AKS_ID_NAME}"
+  MEMBER_CLUSTER_1_AKS_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_1_AKS_ID_NAME})
+  echo "Creating member-1 kubelet identity: ${MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME}"
+  MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME})
+
+  export MEMBER_CLUSTER_1_AKS_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_1_AKS_IDENTITY} | jq -r '. | .id')
+  export MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY} | jq -r '. | .id')
+  export MEMBER_CLUSTER_1_KUBELET_PRINCIPAL_ID=$(echo ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY} | jq -r '. | .principalId')
+  export MEMBER_CLUSTER_1_KUBELET_CLIENT_ID=$(echo ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY} | jq -r '. | .clientId')
+
+  # Create aks identity for member-2
+  echo "Creating member-2 control-plane identity: ${MEMBER_CLUSTER_2_AKS_ID_NAME}"
+  MEMBER_CLUSTER_2_AKS_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_2_AKS_ID_NAME})
+  echo "Creating member-2 kubelet identity: ${MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME}"
+  MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME})
+
+  export MEMBER_CLUSTER_2_AKS_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_2_AKS_IDENTITY} | jq -r '. | .id')
+  export MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY} | jq -r '. | .id')
+  export MEMBER_CLUSTER_2_KUBELET_PRINCIPAL_ID=$(echo ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY} | jq -r '. | .principalId')
+  export MEMBER_CLUSTER_2_KUBELET_CLIENT_ID=$(echo ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY} | jq -r '. | .clientId')
+fi
 
 AZURE_NETWORK_SETTING="${AZURE_NETWORK_SETTING:-shared-vnet}"
 case $AZURE_NETWORK_SETTING in
@@ -95,6 +160,61 @@ then
     az aks wait --created --interval 10 --name $MEMBER_CLUSTER_4 --resource-group $RESOURCE_GROUP --timeout 1800
 fi
 
+# hack: add role assignments to kubelet identity
+if [ "$ENABLE_TRAFFIC_MANAGER" == "true" ]; then
+  echo "Assigning roles to hub kubelet identity on the resourceGroup of the hub cluster"
+  az role assignment create --role "Network Contributor" --assignee ${HUB_CLUSTER_KUBELET_PRINCIPAL_ID} --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP}" > /dev/null
+
+  # Creating azure configuration file for the controllers
+  echo "Generating azure configuration file:" $(pwd)/hub_azure_config.yaml
+  cat << EOF > $(pwd)/hub_azure_config.yaml
+  azureCloudConfig:
+    cloud: "AzurePublicCloud"
+    subscriptionId: "${AZURE_SUBSCRIPTION_ID}"
+    useManagedIdentityExtension: true
+    userAssignedIdentityID: "${HUB_CLUSTER_KUBELET_CLIENT_ID}"
+    resourceGroup: "${RESOURCE_GROUP}"
+    location: "${LOCATION}"
+EOF
+
+  AKS_MEMBER_1=$(az aks show -g $RESOURCE_GROUP -n $MEMBER_CLUSTER_1)
+  AKS_MEMBER_1_NODE_RESOURCE_GROUP=$(echo ${AKS_MEMBER_1} | jq -r '. | .nodeResourceGroup')
+
+  echo "Assigning roles to member-1 kubelet identity on the MC_resourceGroup of the member cluster"
+  az role assignment create --role "Network Contributor" --assignee ${MEMBER_CLUSTER_1_KUBELET_PRINCIPAL_ID} --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AKS_MEMBER_1_NODE_RESOURCE_GROUP}" > /dev/null
+
+  # Creating azure configuration file for the controllers
+  echo "Generating azure configuration file:" $(pwd)/member_1_azure_config.yaml
+  cat << EOF > $(pwd)/member_1_azure_config.yaml
+  azureCloudConfig:
+    cloud: "AzurePublicCloud"
+    subscriptionId: "${AZURE_SUBSCRIPTION_ID}"
+    useManagedIdentityExtension: true
+    userAssignedIdentityID: "${MEMBER_CLUSTER_1_KUBELET_CLIENT_ID}"
+    resourceGroup: "${AKS_MEMBER_1_NODE_RESOURCE_GROUP}"
+    location: "${LOCATION}"
+
+EOF
+
+  AKS_MEMBER_2=$(az aks show -g $RESOURCE_GROUP -n $MEMBER_CLUSTER_2)
+  AKS_MEMBER_2_NODE_RESOURCE_GROUP=$(echo ${AKS_MEMBER_2} | jq -r '. | .nodeResourceGroup')
+
+  echo "Assigning roles to member-2 kubelet identity on the MC_resourceGroup of the member cluster"
+  az role assignment create --role "Network Contributor" --assignee ${MEMBER_CLUSTER_2_KUBELET_PRINCIPAL_ID} --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AKS_MEMBER_2_NODE_RESOURCE_GROUP}" > /dev/null
+
+  # Creating azure configuration file for the controllers
+  echo "Generating azure configuration file:" $(pwd)/member_2_azure_config.yaml
+  cat << EOF > $(pwd)/member_2_azure_config.yaml
+  azureCloudConfig:
+    cloud: "AzurePublicCloud"
+    subscriptionId: "${AZURE_SUBSCRIPTION_ID}"
+    useManagedIdentityExtension: true
+    userAssignedIdentityID: "${MEMBER_CLUSTER_2_KUBELET_CLIENT_ID}"
+    resourceGroup: "${AKS_MEMBER_2_NODE_RESOURCE_GROUP}"
+    location: "${LOCATION}"
+EOF
+fi
+
 # Export kubeconfig.
 az aks get-credentials --name $HUB_CLUSTER -g $RESOURCE_GROUP --admin --overwrite-existing
 az aks get-credentials --name $MEMBER_CLUSTER_1 -g $RESOURCE_GROUP --admin --overwrite-existing
@@ -107,16 +227,23 @@ fi
 export HUB_URL=$(cat ~/.kube/config | yq eval ".clusters | .[] | select(.name=="\"$HUB_CLUSTER\"") | .cluster.server")
 
 # Setup hub cluster credentials.
-export CLIENT_ID_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId')
-export PRINCIPAL_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId')
-export CLIENT_ID_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId')
-export PRINCIPAL_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId')
-if [ "${AZURE_NETWORK_SETTING}" == "perf-test" ]
-then
+if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then
+  export CLIENT_ID_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId')
+  export PRINCIPAL_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId')
+  export CLIENT_ID_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId')
+  export PRINCIPAL_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId')
+  if [ "${AZURE_NETWORK_SETTING}" == "perf-test" ]
+  then
     export CLIENT_ID_FOR_MEMBER_3=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_3"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_3-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId')
     export PRINCIPAL_FOR_MEMBER_3=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_3"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_3-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId')
     export CLIENT_ID_FOR_MEMBER_4=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_4"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_4-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId')
     export PRINCIPAL_FOR_MEMBER_4=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_4"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_4-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId')
+    fi
+else
+  export CLIENT_ID_FOR_MEMBER_1=$MEMBER_CLUSTER_1_KUBELET_CLIENT_ID
+  export PRINCIPAL_FOR_MEMBER_1=$MEMBER_CLUSTER_1_KUBELET_PRINCIPAL_ID
+  export CLIENT_ID_FOR_MEMBER_2=$MEMBER_CLUSTER_2_KUBELET_CLIENT_ID
+  export PRINCIPAL_FOR_MEMBER_2=$MEMBER_CLUSTER_2_KUBELET_PRINCIPAL_ID
 fi
 
 kubectl config use-context $HUB_CLUSTER-admin
@@ -173,7 +300,8 @@ kubectl apply -f config/crd/*
 helm install hub-net-controller-manager \
     ./charts/hub-net-controller-manager/ \
     --set image.repository=$REGISTRY/hub-net-controller-manager \
-    --set image.tag=$TAG
+    --set image.tag=$TAG \
+        $( [ "$ENABLE_TRAFFIC_MANAGER" = "true" ] && echo "--set enableTrafficManagerFeature=true -f hub_azure_config.yaml" )
 
 # Helm install charts for member clusters.
 kubectl config use-context $MEMBER_CLUSTER_1-admin
@@ -192,7 +320,8 @@ helm install member-net-controller-manager ./charts/member-net-controller-manage
     --set config.hubURL=$HUB_URL \
     --set config.provider=azure \
     --set config.memberClusterName=$MEMBER_CLUSTER_1 \
-    --set azure.clientid=$CLIENT_ID_FOR_MEMBER_1
+    --set azure.clientid=$CLIENT_ID_FOR_MEMBER_1 \
+    $( [ "$ENABLE_TRAFFIC_MANAGER" = "true" ] && echo "--set enableTrafficManagerFeature=true -f member_1_azure_config.yaml" )
 
 kubectl config use-context $MEMBER_CLUSTER_2-admin
 kubectl apply -f config/crd/*
@@ -210,7 +339,8 @@ helm install member-net-controller-manager ./charts/member-net-controller-manage
     --set config.hubURL=$HUB_URL \
     --set config.provider=azure \
     --set config.memberClusterName=$MEMBER_CLUSTER_2 \
-    --set azure.clientid=$CLIENT_ID_FOR_MEMBER_2
+    --set azure.clientid=$CLIENT_ID_FOR_MEMBER_2 \
+    $( [ "$ENABLE_TRAFFIC_MANAGER" = "true" ] && echo "--set enableTrafficManagerFeature=true -f member_2_azure_config.yaml" )
 
 # TODO(mainred): Before the app image is publicly available in MCR, we build and publish the image to the test registry.
 # Build and publish the app image dedicated for fleet networking test.