chore: use msi in az copy command (#4033)

Azure · Feb 5, 2024 · 21568e9 · 21568e9
1 parent c71c3d8
commit 21568e9
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 37 deletions.
diff --git a/e2e/windows/e2e-helper.sh b/e2e/windows/e2e-helper.sh
@@ -68,19 +68,15 @@ upload_linux_file_to_storage_account() {
     E2E_MC_RESOURCE_GROUP_NAME="MC_${E2E_RESOURCE_GROUP_NAME}_${AZURE_E2E_CLUSTER_NAME}_$AZURE_BUILD_LOCATION"
     MC_VMSS_NAME=$(az vmss list -g $E2E_MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
     VMSS_INSTANCE_ID="$(az vmss list-instances --name $MC_VMSS_NAME -g $E2E_MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
-
-    set +x
-    expiryTime=$(date --date="2 day" +%Y-%m-%d)
-    token=$(az storage container generate-sas --account-name $AZURE_E2E_STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'w' --expiry $expiryTime --name $WINDOWS_E2E_STORAGE_CONTAINER)
-    linuxFileURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip?${token}"
+
+    linuxFileURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip"
 
     az vmss run-command invoke --command-id RunShellScript \
         --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
         --name $MC_VMSS_NAME \
         --instance-id $VMSS_INSTANCE_ID \
-        --scripts "cat /etc/kubernetes/azure.json > /home/fields.json; cat /etc/kubernetes/certs/apiserver.crt | base64 -w 0 > /home/apiserver.crt; cat /etc/kubernetes/certs/ca.crt | base64 -w 0 > /home/ca.crt; cat /etc/kubernetes/certs/client.key | base64 -w 0 > /home/client.key; cat /var/lib/kubelet/bootstrap-kubeconfig > /home/bootstrap-kubeconfig; cd /home; zip file.zip fields.json apiserver.crt ca.crt client.key bootstrap-kubeconfig; wget https://aka.ms/downloadazcopy-v10-linux; tar -xvf downloadazcopy-v10-linux; cd ./azcopy_*; ./azcopy copy /home/file.zip $linuxFileURL" || retval=$?
+        --scripts "cat /etc/kubernetes/azure.json > /home/fields.json; cat /etc/kubernetes/certs/apiserver.crt | base64 -w 0 > /home/apiserver.crt; cat /etc/kubernetes/certs/ca.crt | base64 -w 0 > /home/ca.crt; cat /etc/kubernetes/certs/client.key | base64 -w 0 > /home/client.key; cat /var/lib/kubelet/bootstrap-kubeconfig > /home/bootstrap-kubeconfig; cd /home; zip file.zip fields.json apiserver.crt ca.crt client.key bootstrap-kubeconfig; wget https://aka.ms/downloadazcopy-v10-linux; tar -xvf downloadazcopy-v10-linux; cd ./azcopy_*; export AZCOPY_AUTO_LOGIN_TYPE=\"MSI\"; export AZCOPY_MSI_RESOURCE_STRING=\"${AZURE_MSI_RESOURCE_STRING}\"; ./azcopy copy /home/file.zip $linuxFileURL" || retval=$?
 
-    set -x
     if [ "$retval" -eq 0 ]; then
         log "Upload linux file successfully"
     else
@@ -92,13 +88,10 @@ download_linux_file_from_storage_account() {
     wget https://aka.ms/downloadazcopy-v10-linux
     tar -xvf downloadazcopy-v10-linux
 
-    expiryTime=$(date --date="2 day" +%Y-%m-%d)
+    linuxFileURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip"
 
-    set +x
-
-    token=$(az storage container generate-sas --account-name $AZURE_E2E_STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'rl' --expiry $expiryTime --name $WINDOWS_E2E_STORAGE_CONTAINER)
-    tokenWithoutQuote=${token//\"}
-    linuxFileURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip?${tokenWithoutQuote}"
+    export AZCOPY_AUTO_LOGIN_TYPE="MSI"
+    export AZCOPY_MSI_RESOURCE_STRING="${AZURE_MSI_RESOURCE_STRING}"
 
     array=(azcopy_*)
     noExistStr="File count: 0"
@@ -113,16 +106,13 @@ download_linux_file_from_storage_account() {
         fileExist="true"
         break;
     done
-    set -x
 
     if [ "$fileExist" == "false" ]; then
         err "File does not exist in storage account."
         exit 1
     fi
 
-    set +x
     ${array[0]}/azcopy copy $linuxFileURL file.zip
-    set -x
 
     unzip file.zip
 }

diff --git a/e2e/windows/e2e-scenario.sh b/e2e/windows/e2e-scenario.sh
@@ -13,34 +13,33 @@ collect-logs() {
     retval=0
     mkdir -p $SCENARIO_NAME-logs
     VMSS_INSTANCE_ID="$(az vmss list-instances --name $DEPLOYMENT_VMSS_NAME -g $E2E_MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
-    set +x
-    expiryTime=$(date --date="2 day" +%Y-%m-%d)
-    token=$(az storage container generate-sas --account-name $AZURE_E2E_STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'rwld' --expiry $expiryTime --name $AZURE_E2E_STORAGE_LOG_CONTAINER --https-only)
+
     # Use .ps1 file to run scripts since single quotes of parameters for --scripts would fail in check-shell
     az vmss run-command invoke --command-id RunPowerShellScript \
         --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
         --name $DEPLOYMENT_VMSS_NAME \
         --instance-id $VMSS_INSTANCE_ID \
         --scripts @upload-cse-logs.ps1 \
-        --parameters arg1=$AZURE_E2E_STORAGE_ACCOUNT_NAME arg2=$AZURE_E2E_STORAGE_LOG_CONTAINER arg3=$DEPLOYMENT_VMSS_NAME arg4=$token || retval=$?
+        --parameters arg1=$AZURE_E2E_STORAGE_ACCOUNT_NAME arg2=$AZURE_E2E_STORAGE_LOG_CONTAINER arg3=$DEPLOYMENT_VMSS_NAME arg4=$AZURE_MSI_RESOURCE_STRING || retval=$?
     if [ "$retval" -ne 0 ]; then
         err "Failed in uploading cse logs. Error code is $retval."
     fi
 
-    tokenWithoutQuote=${token//\"}
     # use array to pass shellcheck
+    export AZCOPY_AUTO_LOGIN_TYPE="MSI"
+    export AZCOPY_MSI_RESOURCE_STRING="${AZURE_MSI_RESOURCE_STRING}"
+
     array=(azcopy_*)
-    ${array[0]}/azcopy copy "https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_LOG_CONTAINER}/${DEPLOYMENT_VMSS_NAME}-cse.log?${tokenWithoutQuote}" $SCENARIO_NAME-logs/$WINDOWS_E2E_IMAGE$WINDOWS_GPU_DRIVER_SUFFIX-CustomDataSetupScript.log || retval=$?
+    ${array[0]}/azcopy copy "https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_LOG_CONTAINER}/${DEPLOYMENT_VMSS_NAME}-cse.log}" $SCENARIO_NAME-logs/$WINDOWS_E2E_IMAGE$WINDOWS_GPU_DRIVER_SUFFIX-CustomDataSetupScript.log || retval=$?
     if [ "$retval" -ne 0 ]; then
         err "Failed in downloading cse logs. Error code is $retval."
     else
         log "Collect cse logs done"
-        ${array[0]}/azcopy rm "https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_LOG_CONTAINER}/${DEPLOYMENT_VMSS_NAME}-cse.log?${tokenWithoutQuote}" || retval=$?
+        ${array[0]}/azcopy rm "https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_LOG_CONTAINER}/${DEPLOYMENT_VMSS_NAME}-cse.log" || retval=$?
         if [ "$retval" -ne 0 ]; then
             err "Failed in deleting cse logs in remote storage. Error code is $retval."
         fi
     fi
-    set -x
 }
 
 E2E_RESOURCE_GROUP_NAME="$AZURE_E2E_RESOURCE_GROUP_NAME-$WINDOWS_E2E_IMAGE$WINDOWS_GPU_DRIVER_SUFFIX-$K8S_VERSION"
@@ -58,16 +57,14 @@ cd ../staging/cse/windows
 zip -r ../../../$WINDOWS_E2E_IMAGE/$WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip ./* -x ./*.tests.ps1 -x "*azurecnifunc.tests.suites*" -x README -x provisioningscripts/*.md -x debug/update-scripts.ps1
 log "Zip cse packages done"
 
-set +x
-expiryTime=$(date --date="2 day" +%Y-%m-%d)
-token=$(az storage container generate-sas --account-name $AZURE_E2E_STORAGE_ACCOUNT_NAME --account-key $MAPPED_ACCOUNT_KEY --permissions 'rwld' --expiry $expiryTime --name $AZURE_E2E_STORAGE_PACKAGE_CONTAINER)
-tokenWithoutQuote=${token//\"}
-
-csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip?${tokenWithoutQuote}"
+csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
 export csePackageURL
 
 cd ../../../$WINDOWS_E2E_IMAGE
 
+export AZCOPY_AUTO_LOGIN_TYPE="MSI"
+export AZCOPY_MSI_RESOURCE_STRING="${AZURE_MSI_RESOURCE_STRING}"
+
 array=(azcopy_*)
 noExistStr="File count: 0"
 listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
@@ -76,24 +73,21 @@ for i in $(seq 1 10); do
     if [[ "$listResult" != *"$noExistStr"* ]]; then
         log "Cse package with the same exists, retry $i to use new name..."
         timeStamp=$(date +%s)
-        csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip?${tokenWithoutQuote}"
+        csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
         listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
         continue
     fi
     ${array[0]}/azcopy copy $WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip $csePackageURL
     break;
 done
 
-set -x
-
 listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
 if [[ "$listResult" == *"$noExistStr"* ]]; then
     err "Failed to upload cse package"
     exit 1
 fi
 
 log "Upload cse packages done"
-csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
 
 log "Scenario is $SCENARIO_NAME"
 log "Windows package version is $WINDOWS_PACKAGE_VERSION"
@@ -178,7 +172,7 @@ if [ "$retval" -ne 0 ]; then
 fi
 
 # delete cse package in storage account
-csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip?${tokenWithoutQuote}"
+csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
 ${array[0]}/azcopy rm $csePackageURL || retval=$?
 
 if [ "$retval" -ne 0 ]; then

diff --git a/e2e/windows/e2e-starter.sh b/e2e/windows/e2e-starter.sh
@@ -76,7 +76,7 @@ if [ "$create_cluster" == "true" ]; then
     clusterCreateStartTime=$(date +%s)
     retval=0
 
-    az aks create -g $E2E_RESOURCE_GROUP_NAME -n $AZURE_E2E_CLUSTER_NAME --node-count 1 --generate-ssh-keys --network-plugin azure -ojson || retval=$?
+    az aks create -g $E2E_RESOURCE_GROUP_NAME -n $AZURE_E2E_CLUSTER_NAME --enable-managed-identity --assign-identity $AZURE_MSI_RESOURCE_STRING --assign-kubelet-identity $AZURE_MSI_RESOURCE_STRING --node-count 1 --generate-ssh-keys --network-plugin azure -ojson || retval=$?
 
     if [ "$retval" -ne 0  ]; then
         log "Other pipelines may be creating cluster $AZURE_E2E_CLUSTER_NAME, waiting for ready"

diff --git a/e2e/windows/upload-cse-logs.ps1 b/e2e/windows/upload-cse-logs.ps1
@@ -5,4 +5,4 @@ param(
     [string]$arg4
 )
 
-Invoke-WebRequest -UseBasicParsing https://aka.ms/downloadazcopy-v10-windows -OutFile azcopy.zip;expand-archive azcopy.zip;cd .\azcopy\*;.\azcopy.exe copy "C:\azuredata\CustomDataSetupScript.log" "https://$arg1.blob.core.windows.net/$arg2/$arg3-cse.log?$arg4"
+Invoke-WebRequest -UseBasicParsing https://aka.ms/downloadazcopy-v10-windows -OutFile azcopy.zip;expand-archive azcopy.zip;cd .\azcopy\*; $env:AZCOPY_AUTO_LOGIN_TYPE="MSI"; $env:AZCOPY_MSI_RESOURCE_STRING=$arg4; .\azcopy.exe copy "C:\azuredata\CustomDataSetupScript.log" "https://$arg1.blob.core.windows.net/$arg2/$arg3-cse.log"