diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 81467f18..5bcb9159 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -37,6 +37,13 @@ jobs: fail-fast: false matrix: network-setting: [shared-vnet, peered-vnet, dynamic-ip-allocation] + include: + - network-setting: shared-vnet + enable-traffic-manager: true + - network-setting: peered-vnet + enable-traffic-manager: false + - network-setting: dynamic-ip-allocation + enable-traffic-manager: false runs-on: ubuntu-latest needs: [ detect-noop, @@ -92,6 +99,7 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }} AZURE_NETWORK_SETTING: ${{ matrix.network-setting }} AZURE_RESOURCE_GROUP: ${{ env.AZURE_RESOURCE_GROUP }} + ENABLE_TRAFFIC_MANAGER: ${{ matrix.enable-traffic-manager }} - name: Run e2e tests run: | make e2e-tests diff --git a/test/scripts/aks-shared-vnet.sh b/test/scripts/aks-shared-vnet.sh index 6600a0a4..5dec1684 100644 --- a/test/scripts/aks-shared-vnet.sh +++ b/test/scripts/aks-shared-vnet.sh @@ -37,7 +37,8 @@ az network vnet subnet create \ --address-prefixes 10.2.0.0/16 # Create aks member cluster1. -az aks create \ +if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then + az aks create \ --location $MEMBER_1_LOCATION \ --resource-group $RESOURCE_GROUP \ --name $MEMBER_CLUSTER_1 \ @@ -46,9 +47,22 @@ az aks create \ --network-plugin azure \ --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_1_SUBNET" \ --no-wait +else + az aks create \ + --location $MEMBER_1_LOCATION \ + --resource-group $RESOURCE_GROUP \ + --name $MEMBER_CLUSTER_1 \ + --node-count $NODE_COUNT \ + --generate-ssh-keys \ + --network-plugin azure \ + --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_1_SUBNET" \ + --enable-managed-identity --assign-identity ${MEMBER_CLUSTER_1_AKS_IDENTITY_ID} --assign-kubelet-identity ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY_ID} \ + --no-wait +fi # Create aks member cluster2. -az aks create \ +if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then + az aks create \ --location $MEMBER_2_LOCATION \ --resource-group $RESOURCE_GROUP \ --name $MEMBER_CLUSTER_2 \ @@ -57,3 +71,15 @@ az aks create \ --network-plugin azure \ --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_2_SUBNET" \ --no-wait +else + az aks create \ + --location $MEMBER_2_LOCATION \ + --resource-group $RESOURCE_GROUP \ + --name $MEMBER_CLUSTER_2 \ + --node-count $NODE_COUNT \ + --generate-ssh-keys \ + --network-plugin azure \ + --vnet-subnet-id "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET/subnets/$MEMBER_2_SUBNET" \ + --enable-managed-identity --assign-identity ${MEMBER_CLUSTER_2_AKS_IDENTITY_ID} --assign-kubelet-identity ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY_ID} \ + --no-wait +fi diff --git a/test/scripts/bootstrap.sh b/test/scripts/bootstrap.sh index 4a1cf76a..3fb6462a 100644 --- a/test/scripts/bootstrap.sh +++ b/test/scripts/bootstrap.sh @@ -7,6 +7,13 @@ set -x # Check required variables. [[ -z "${AZURE_SUBSCRIPTION_ID}" ]] && echo "AZURE_SUBSCRIPTION_ID is not set" && exit 1 +export ENABLE_TRAFFIC_MANAGER=${ENABLE_TRAFFIC_MANAGER:-"false"} + +if [ "$ENABLE_TRAFFIC_MANAGER" == "true" ] && [ "$AZURE_NETWORK_SETTING" != "shared-vnet" ]; then + echo "error: setting up traffic manager testing env is not supported when AZURE_NETWORK_SETTING is not shared-vnet" + exit 1 +fi + az account set -s ${AZURE_SUBSCRIPTION_ID} # Create resource group to host hub and member clusters. @@ -33,26 +40,84 @@ az acr create -g $RESOURCE_GROUP -n $REGISTRY_NAME --sku standard --tags "source az acr update --name $REGISTRY_NAME --anonymous-pull-enabled -g $RESOURCE_GROUP az acr login -n $REGISTRY_NAME export REGISTRY=$REGISTRY_NAME.azurecr.io -export TAG=`git rev-parse --short=7 HEAD` +export TAG=$(git rev-parse --short=7 HEAD) make push # Create hub and member clusters and wait until all clusters are ready. -export HUB_CLUSTER=hub -export MEMBER_CLUSTER_1=member-1 -export MEMBER_CLUSTER_2=member-2 +export HUB_CLUSTER=${HUB_CLUSTER:-"hub"} +export MEMBER_CLUSTER_1=${MEMBER_CLUSTER_1:-"member-1"} +export MEMBER_CLUSTER_2=${MEMBER_CLUSTER_2:-"member-2"} export NODE_COUNT=2 -# Create aks hub cluster -az aks create \ - --location $LOCATION \ - --resource-group $RESOURCE_GROUP \ - --name $HUB_CLUSTER \ - --node-count $NODE_COUNT \ - --generate-ssh-keys \ - --enable-aad \ - --enable-azure-rbac \ - --network-plugin azure \ - --no-wait +export HUB_CLUSTER_AKS_ID_NAME=${HUB_CLUSTER_AKS_ID_NAME:-"fleet-net-hub-aks-id"} +export HUB_CLUSTER_AKS_KUBELET_ID_NAME=${HUB_CLUSTER_AKS_KUBELET_ID_NAME:-"fleet-net-hub-aks-kubelet-id"} +export MEMBER_CLUSTER_1_AKS_ID_NAME=${MEMBER_CLUSTER_1_AKS_ID_NAME:-"fleet-net-member-1-aks-id"} +export MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME=${MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME:-"fleet-net-member-1-aks-kubelet-id"} +export MEMBER_CLUSTER_2_AKS_ID_NAME=${MEMBER_CLUSTER_2_AKS_ID_NAME:-"fleet-net-member-2-aks-id"} +export MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME=${MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME:-"fleet-net-member-2-aks-kubelet-id"} + + + +if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then + # Create aks hub cluster + echo "Creating aks cluster: ${HUB_CLUSTER}" + az aks create \ + --location $LOCATION \ + --resource-group $RESOURCE_GROUP \ + --name $HUB_CLUSTER \ + --node-count $NODE_COUNT \ + --generate-ssh-keys \ + --enable-aad \ + --enable-azure-rbac \ + --network-plugin azure \ + --no-wait +else + # Create aks identity + echo "Creating hub control-plane identity: ${HUB_CLUSTER_AKS_ID_NAME}" + HUB_CLUSTER_AKS_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${HUB_CLUSTER_AKS_ID_NAME}) + echo "Creating hub kubelet identity: ${HUB_CLUSTER_AKS_KUBELET_ID_NAME}" + HUB_CLUSTER_AKS_KUBELET_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${HUB_CLUSTER_AKS_KUBELET_ID_NAME}) + + HUB_CLUSTER_AKS_IDENTITY_ID=$(echo ${HUB_CLUSTER_AKS_IDENTITY} | jq -r '. | .id') + HUB_CLUSTER_AKS_KUBELET_IDENTITY_ID=$(echo ${HUB_CLUSTER_AKS_KUBELET_IDENTITY} | jq -r '. | .id') + HUB_CLUSTER_KUBELET_PRINCIPAL_ID=$(echo ${HUB_CLUSTER_AKS_KUBELET_IDENTITY} | jq -r '. | .principalId') + HUB_CLUSTER_KUBELET_CLIENT_ID=$(echo ${HUB_CLUSTER_AKS_KUBELET_IDENTITY} | jq -r '. | .clientId') + + echo "Creating aks cluster: ${HUB_CLUSTER}" + az aks create \ + --location $LOCATION \ + --resource-group $RESOURCE_GROUP \ + --name $HUB_CLUSTER \ + --node-count $NODE_COUNT \ + --generate-ssh-keys \ + --enable-aad \ + --enable-azure-rbac \ + --network-plugin azure \ + --enable-managed-identity --assign-identity ${HUB_CLUSTER_AKS_IDENTITY_ID} --assign-kubelet-identity ${HUB_CLUSTER_AKS_KUBELET_IDENTITY_ID} \ + --no-wait + + # Create aks identity for member-1 + echo "Creating member-1 control-plane identity: ${MEMBER_CLUSTER_1_AKS_ID_NAME}" + MEMBER_CLUSTER_1_AKS_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_1_AKS_ID_NAME}) + echo "Creating member-1 kubelet identity: ${MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME}" + MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_1_AKS_KUBELET_ID_NAME}) + + export MEMBER_CLUSTER_1_AKS_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_1_AKS_IDENTITY} | jq -r '. | .id') + export MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY} | jq -r '. | .id') + export MEMBER_CLUSTER_1_KUBELET_PRINCIPAL_ID=$(echo ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY} | jq -r '. | .principalId') + export MEMBER_CLUSTER_1_KUBELET_CLIENT_ID=$(echo ${MEMBER_CLUSTER_1_AKS_KUBELET_IDENTITY} | jq -r '. | .clientId') + + # Create aks identity for member-2 + echo "Creating member-2 control-plane identity: ${MEMBER_CLUSTER_2_AKS_ID_NAME}" + MEMBER_CLUSTER_2_AKS_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_2_AKS_ID_NAME}) + echo "Creating member-2 kubelet identity: ${MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME}" + MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY=$(az identity create -g ${RESOURCE_GROUP} -n ${MEMBER_CLUSTER_2_AKS_KUBELET_ID_NAME}) + + export MEMBER_CLUSTER_2_AKS_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_2_AKS_IDENTITY} | jq -r '. | .id') + export MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY_ID=$(echo ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY} | jq -r '. | .id') + export MEMBER_CLUSTER_2_KUBELET_PRINCIPAL_ID=$(echo ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY} | jq -r '. | .principalId') + export MEMBER_CLUSTER_2_KUBELET_CLIENT_ID=$(echo ${MEMBER_CLUSTER_2_AKS_KUBELET_IDENTITY} | jq -r '. | .clientId') +fi AZURE_NETWORK_SETTING="${AZURE_NETWORK_SETTING:-shared-vnet}" case $AZURE_NETWORK_SETTING in @@ -95,6 +160,61 @@ then az aks wait --created --interval 10 --name $MEMBER_CLUSTER_4 --resource-group $RESOURCE_GROUP --timeout 1800 fi +# hack: add role assignments to kubelet identity +if [ "$ENABLE_TRAFFIC_MANAGER" == "true" ]; then + echo "Assigning roles to hub kubelet identity on the resourceGroup of the hub cluster" + az role assignment create --role "Network Contributor" --assignee ${HUB_CLUSTER_KUBELET_PRINCIPAL_ID} --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP}" > /dev/null + + # Creating azure configuration file for the controllers + echo "Generating azure configuration file:" $(pwd)/hub_azure_config.yaml + cat << EOF > $(pwd)/hub_azure_config.yaml + azureCloudConfig: + cloud: "AzurePublicCloud" + subscriptionId: "${AZURE_SUBSCRIPTION_ID}" + useManagedIdentityExtension: true + userAssignedIdentityID: "${HUB_CLUSTER_KUBELET_CLIENT_ID}" + resourceGroup: "${RESOURCE_GROUP}" + location: "${LOCATION}" +EOF + + AKS_MEMBER_1=$(az aks show -g $RESOURCE_GROUP -n $MEMBER_CLUSTER_1) + AKS_MEMBER_1_NODE_RESOURCE_GROUP=$(echo ${AKS_MEMBER_1} | jq -r '. | .nodeResourceGroup') + + echo "Assigning roles to member-1 kubelet identity on the MC_resourceGroup of the member cluster" + az role assignment create --role "Network Contributor" --assignee ${MEMBER_CLUSTER_1_KUBELET_PRINCIPAL_ID} --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AKS_MEMBER_1_NODE_RESOURCE_GROUP}" > /dev/null + + # Creating azure configuration file for the controllers + echo "Generating azure configuration file:" $(pwd)/member_1_azure_config.yaml + cat << EOF > $(pwd)/member_1_azure_config.yaml + azureCloudConfig: + cloud: "AzurePublicCloud" + subscriptionId: "${AZURE_SUBSCRIPTION_ID}" + useManagedIdentityExtension: true + userAssignedIdentityID: "${MEMBER_CLUSTER_1_KUBELET_CLIENT_ID}" + resourceGroup: "${AKS_MEMBER_1_NODE_RESOURCE_GROUP}" + location: "${LOCATION}" + +EOF + + AKS_MEMBER_2=$(az aks show -g $RESOURCE_GROUP -n $MEMBER_CLUSTER_2) + AKS_MEMBER_2_NODE_RESOURCE_GROUP=$(echo ${AKS_MEMBER_2} | jq -r '. | .nodeResourceGroup') + + echo "Assigning roles to member-2 kubelet identity on the MC_resourceGroup of the member cluster" + az role assignment create --role "Network Contributor" --assignee ${MEMBER_CLUSTER_2_KUBELET_PRINCIPAL_ID} --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AKS_MEMBER_2_NODE_RESOURCE_GROUP}" > /dev/null + + # Creating azure configuration file for the controllers + echo "Generating azure configuration file:" $(pwd)/member_2_azure_config.yaml + cat << EOF > $(pwd)/member_2_azure_config.yaml + azureCloudConfig: + cloud: "AzurePublicCloud" + subscriptionId: "${AZURE_SUBSCRIPTION_ID}" + useManagedIdentityExtension: true + userAssignedIdentityID: "${MEMBER_CLUSTER_2_KUBELET_CLIENT_ID}" + resourceGroup: "${AKS_MEMBER_2_NODE_RESOURCE_GROUP}" + location: "${LOCATION}" +EOF +fi + # Export kubeconfig. az aks get-credentials --name $HUB_CLUSTER -g $RESOURCE_GROUP --admin --overwrite-existing az aks get-credentials --name $MEMBER_CLUSTER_1 -g $RESOURCE_GROUP --admin --overwrite-existing @@ -107,16 +227,23 @@ fi export HUB_URL=$(cat ~/.kube/config | yq eval ".clusters | .[] | select(.name=="\"$HUB_CLUSTER\"") | .cluster.server") # Setup hub cluster credentials. -export CLIENT_ID_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId') -export PRINCIPAL_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId') -export CLIENT_ID_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId') -export PRINCIPAL_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId') -if [ "${AZURE_NETWORK_SETTING}" == "perf-test" ] -then +if [ "$ENABLE_TRAFFIC_MANAGER" == "false" ]; then + export CLIENT_ID_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId') + export PRINCIPAL_FOR_MEMBER_1=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_1"_"$MEMBER_1_LOCATION" | jq --arg identity $MEMBER_CLUSTER_1-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId') + export CLIENT_ID_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId') + export PRINCIPAL_FOR_MEMBER_2=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_2"_"$MEMBER_2_LOCATION" | jq --arg identity $MEMBER_CLUSTER_2-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId') + if [ "${AZURE_NETWORK_SETTING}" == "perf-test" ] + then export CLIENT_ID_FOR_MEMBER_3=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_3"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_3-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId') export PRINCIPAL_FOR_MEMBER_3=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_3"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_3-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId') export CLIENT_ID_FOR_MEMBER_4=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_4"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_4-agentpool -r -c 'map(select(.name | contains($identity)))[].clientId') export PRINCIPAL_FOR_MEMBER_4=$(az identity list -g MC_"$RESOURCE_GROUP"_"$MEMBER_CLUSTER_4"_"$MEMBER_LOCATION_2" | jq --arg identity $MEMBER_CLUSTER_4-agentpool -r -c 'map(select(.name | contains($identity)))[].principalId') + fi +else + export CLIENT_ID_FOR_MEMBER_1=$MEMBER_CLUSTER_1_KUBELET_CLIENT_ID + export PRINCIPAL_FOR_MEMBER_1=$MEMBER_CLUSTER_1_KUBELET_PRINCIPAL_ID + export CLIENT_ID_FOR_MEMBER_2=$MEMBER_CLUSTER_2_KUBELET_CLIENT_ID + export PRINCIPAL_FOR_MEMBER_2=$MEMBER_CLUSTER_2_KUBELET_PRINCIPAL_ID fi kubectl config use-context $HUB_CLUSTER-admin @@ -173,7 +300,8 @@ kubectl apply -f config/crd/* helm install hub-net-controller-manager \ ./charts/hub-net-controller-manager/ \ --set image.repository=$REGISTRY/hub-net-controller-manager \ - --set image.tag=$TAG + --set image.tag=$TAG \ + $( [ "$ENABLE_TRAFFIC_MANAGER" = "true" ] && echo "--set enableTrafficManagerFeature=true -f hub_azure_config.yaml" ) # Helm install charts for member clusters. kubectl config use-context $MEMBER_CLUSTER_1-admin @@ -192,7 +320,8 @@ helm install member-net-controller-manager ./charts/member-net-controller-manage --set config.hubURL=$HUB_URL \ --set config.provider=azure \ --set config.memberClusterName=$MEMBER_CLUSTER_1 \ - --set azure.clientid=$CLIENT_ID_FOR_MEMBER_1 + --set azure.clientid=$CLIENT_ID_FOR_MEMBER_1 \ + $( [ "$ENABLE_TRAFFIC_MANAGER" = "true" ] && echo "--set enableTrafficManagerFeature=true -f member_1_azure_config.yaml" ) kubectl config use-context $MEMBER_CLUSTER_2-admin kubectl apply -f config/crd/* @@ -210,7 +339,8 @@ helm install member-net-controller-manager ./charts/member-net-controller-manage --set config.hubURL=$HUB_URL \ --set config.provider=azure \ --set config.memberClusterName=$MEMBER_CLUSTER_2 \ - --set azure.clientid=$CLIENT_ID_FOR_MEMBER_2 + --set azure.clientid=$CLIENT_ID_FOR_MEMBER_2 \ + $( [ "$ENABLE_TRAFFIC_MANAGER" = "true" ] && echo "--set enableTrafficManagerFeature=true -f member_2_azure_config.yaml" ) # TODO(mainred): Before the app image is publicly available in MCR, we build and publish the image to the test registry. # Build and publish the app image dedicated for fleet networking test.