Merge branch 'lily/add-go-binary' of https://github.com/Azure/AgentBaker

into lily/add-go-binary
Azure · May 22, 2024 · fb7178d · fb7178d
2 parents c77f5f9 + 582d0ef
commit fb7178d
Show file tree

Hide file tree

Showing 504 changed files with 49,286 additions and 4,054 deletions.
diff --git a/.pipelines/.vsts-vhd-automation.yaml b/.pipelines/.vsts-vhd-automation.yaml
@@ -17,6 +17,9 @@ parameters:
   type: boolean
   default: true
 
+variables:
+  - group: "AKS Dev Assistant (KV)"
+
 steps:
 - bash: |
         set -x
@@ -38,7 +41,7 @@ steps:
         echo $MAPPED_ADO_PAT | az devops login --organization=https://dev.azure.com/msazure
         az devops configure --defaults organization=https://dev.azure.com/msazure project=CloudNativeCompute
   env:
-        MAPPED_ADO_PAT: $(ADO_PAT)
+        MAPPED_ADO_PAT: $(PAT-aksdevassistant)
   displayName: 'az devops login'
 - bash: |
         echo "PR for Image Bumping, Official Branch Cutting"

diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml
@@ -110,6 +110,10 @@ parameters:
   displayName: Build 2204 ARM64 Gen2 containerd
   type: boolean
   default: true
+- name: build2404arm64gen2containerd
+  displayName: Build 2404 ARM64 Gen2 containerd
+  type: boolean
+  default: false
 - name: build2204containerd
   displayName: Build 2204 Gen1 Containerd
   type: boolean
@@ -118,6 +122,14 @@ parameters:
   displayName: Build 2204 Gen2 Containerd
   type: boolean
   default: true
+- name: build2404containerd
+  displayName: Build 2404 Gen1 Containerd
+  type: boolean
+  default: false
+- name: build2404gen2containerd
+  displayName: Build 2404 Gen2 Containerd
+  type: boolean
+  default: false
 - name: build2004cvmgen2containerd
   displayName: Build 2004 CVM Gen2 Containerd
   type: boolean
@@ -890,6 +902,34 @@ stages:
         - template: ./templates/.builder-release-template.yaml
           parameters:
             artifactName: 2204-arm64-gen2-containerd
+  - stage: build_vhd_2404_arm64_gen2_containerd
+    dependsOn: []
+    condition: eq('${{ parameters.build2404arm64gen2containerd }}', true)
+    jobs:
+    - job: build
+      timeoutInMinutes: 180
+      steps:
+        # TODO(artunduman): Remove 'daily' when available
+        - bash: |
+            echo '##vso[task.setvariable variable=DRY_RUN]${{parameters.dryrun}}'
+            echo '##vso[task.setvariable variable=OS_SKU]Ubuntu'
+            echo '##vso[task.setvariable variable=OS_VERSION]24.04'
+            echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical'
+            echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-server-noble-daily'
+            echo '##vso[task.setvariable variable=IMG_SKU]24_04-daily-lts-arm64'
+            echo '##vso[task.setvariable variable=IMG_VERSION]latest'
+            echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2'
+            echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v5'
+            echo '##vso[task.setvariable variable=FEATURE_FLAGS]None'
+            echo '##vso[task.setvariable variable=CONTAINER_RUNTIME]containerd'
+            echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64'
+            echo '##vso[task.setvariable variable=ENABLE_FIPS]False'
+            echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False'
+            echo '##vso[task.setvariable variable=SGX_INSTALL]False'
+          displayName: Setup Build Variables
+        - template: ./templates/.builder-release-template.yaml
+          parameters:
+            artifactName: 2404-arm64-gen2-containerd
   - stage: build_vhd_2204_containerd
     dependsOn: []
     condition: eq('${{ parameters.build2204containerd }}', true)
@@ -942,6 +982,58 @@ stages:
         - template: ./templates/.builder-release-template.yaml
           parameters:
             artifactName: 2204-gen2-containerd
+  - stage: build_vhd_2404_containerd
+    dependsOn: []
+    condition: eq('${{ parameters.build2404containerd }}', true)
+    jobs:
+      - job: build
+        timeoutInMinutes: 180
+        steps:
+          - bash: |
+              echo '##vso[task.setvariable variable=DRY_RUN]${{parameters.dryrun}}'
+              echo '##vso[task.setvariable variable=OS_SKU]Ubuntu'
+              echo '##vso[task.setvariable variable=OS_VERSION]24.04'
+              echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical'
+              echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-server-noble-daily'
+              echo '##vso[task.setvariable variable=IMG_SKU]24_04-daily-lts'
+              echo '##vso[task.setvariable variable=IMG_VERSION]latest'
+              echo '##vso[task.setvariable variable=HYPERV_GENERATION]V1'
+              echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5'
+              echo '##vso[task.setvariable variable=FEATURE_FLAGS]None'
+              echo '##vso[task.setvariable variable=CONTAINER_RUNTIME]containerd'
+              echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64'
+              echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False'
+              echo '##vso[task.setvariable variable=SGX_INSTALL]False'
+            displayName: Setup Build Variables
+          - template: ./templates/.builder-release-template.yaml
+            parameters:
+              artifactName: 2404-containerd
+  - stage: build_vhd_2404_gen2_containerd
+    dependsOn: []
+    condition: eq('${{ parameters.build2404gen2containerd }}', true)
+    jobs:
+      - job: build
+        timeoutInMinutes: 180
+        steps:
+          - bash: |
+              echo '##vso[task.setvariable variable=DRY_RUN]${{parameters.dryrun}}'
+              echo '##vso[task.setvariable variable=OS_SKU]Ubuntu'
+              echo '##vso[task.setvariable variable=OS_VERSION]24.04'
+              echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical'
+              echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-server-noble-daily'
+              echo '##vso[task.setvariable variable=IMG_SKU]24_04-daily-lts-gen2'
+              echo '##vso[task.setvariable variable=IMG_VERSION]latest'
+              echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2'
+              echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5'
+              echo '##vso[task.setvariable variable=FEATURE_FLAGS]None'
+              echo '##vso[task.setvariable variable=CONTAINER_RUNTIME]containerd'
+              echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64'
+              echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False'
+              echo '##vso[task.setvariable variable=SGX_INSTALL]True'
+            displayName: Setup Build Variables
+          - template: ./templates/.builder-release-template.yaml
+            parameters:
+              artifactName: 2404-gen2-containerd
   - stage: build_vhd_2004_cvm_gen2_containerd
     dependsOn: []
     condition: eq('${{ parameters.build2004cvmgen2containerd }}', true)

diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml
@@ -18,6 +18,7 @@ pr:
     - pkg/agent/testdata/AKSWindows* # Windows test data
 
 variables:
+- group: "AKS Dev Assistant (KV)"
 - group: ab-e2e
 - name: defaultTimeout
   value: 90
@@ -46,7 +47,7 @@ jobs:
     displayName: Run AgentBaker E2E
     env:
       VHD_BUILD_ID: $(VHD_BUILD_ID)
-      ADO_PAT: $(ADO_PAT)
+      ADO_PAT: $(PAT-aksdevassistant)
   - publish: $(System.DefaultWorkingDirectory)/e2e/scenario-logs
     artifact: scenario-logs
     condition: always()

diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml
@@ -54,8 +54,7 @@ steps:
       SKU_NAME=${OS_VERSION} && \
       if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \
       if [[ ${OS_VERSION} == "V2" && ${ARCHITECTURE,,} == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \
-      if [[ ${OS_VERSION} == "18.04" && ${ARCHITECTURE,,} == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \
-      if [[ ${OS_VERSION} == "22.04" && ${ARCHITECTURE,,} == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \
+      if [[ (${OS_VERSION} == "18.04" || ${OS_VERSION} == "22.04" || ${OS_VERSION} == "24.04") && ${ARCHITECTURE,,} == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \
       if [[ (${OS_VERSION} == "18.04" || ${OS_VERSION} == "20.04" || ${OS_VERSION} == "22.04") && ${ENABLE_FIPS,,} == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \
       if [[ ${OS_VERSION} == "V2" && ${ENABLE_FIPS,,} == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \
       if [[ "$(FEATURE_FLAGS)" == *"fullgpu"* ]]; then SKU_NAME="${SKU_NAME}gpu"; fi && \

diff --git a/e2e/windows/e2e-scenario.sh b/e2e/windows/e2e-scenario.sh
@@ -82,8 +82,8 @@ cd ../staging/cse/windows
 zip -r ../../../$WINDOWS_E2E_IMAGE/$WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip ./* -x ./*.tests.ps1 -x "*azurecnifunc.tests.suites*" -x README -x provisioningscripts/*.md -x debug/update-scripts.ps1
 log "Zip cse packages done"
 
-csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
-export csePackageURL
+csePackageName="${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
+cseBlobUrlForUploading="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/\$web/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${csePackageName}"
 
 cd ../../../$WINDOWS_E2E_IMAGE
 
@@ -92,26 +92,31 @@ export AZCOPY_MSI_RESOURCE_STRING="${AZURE_MSI_RESOURCE_STRING}"
 
 array=(azcopy_*)
 noExistStr="File count: 0"
-listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
+listResult=$(${array[0]}/azcopy list $cseBlobUrlForUploading --running-tally)
 
 for i in $(seq 1 10); do
     if [[ "$listResult" != *"$noExistStr"* ]]; then
         log "Cse package with the same exists, retry $i to use new name..."
         timeStamp=$(date +%s)
-        csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
-        listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
+        csePackageName="${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
+        cseBlobUrlForUploading="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/\$web/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${csePackageName}"
+        listResult=$(${array[0]}/azcopy list $cseBlobUrlForUploading --running-tally)
         continue
     fi
-    ${array[0]}/azcopy copy $WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip $csePackageURL
+    ${array[0]}/azcopy copy $WINDOWS_E2E_IMAGE-aks-windows-cse-scripts.zip $cseBlobUrlForUploading
     break;
 done
 
-listResult=$(${array[0]}/azcopy list $csePackageURL --running-tally)
+listResult=$(${array[0]}/azcopy list $cseBlobUrlForUploading --running-tally)
 if [[ "$listResult" == *"$noExistStr"* ]]; then
     err "Failed to upload cse package"
     exit 1
 fi
 
+# Use website url to allow anonymous download
+csePackageURL="${AZURE_E2E_STORAGE_CSE_PACKAGE_ENDPOINT}/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${csePackageName}"
+export csePackageURL
+
 log "Upload cse packages done"
 
 log "Scenario is $SCENARIO_NAME"
@@ -191,15 +196,14 @@ az deployment group create --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
 set -e
 log "Deployment of windows vmss succeeded."
 
+# delete cse package in storage account
+${array[0]}/azcopy rm $cseBlobUrlForUploading || retval=$?
+
 if [ "$retval" -ne 0 ]; then
     err "Failed to deploy windows vmss. Error code is $retval."
     exit 1
 fi
 
-# delete cse package in storage account
-csePackageURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AZURE_E2E_STORAGE_PACKAGE_CONTAINER}/${timeStamp}-${DEPLOYMENT_VMSS_NAME}-aks-windows-cse-scripts.zip"
-${array[0]}/azcopy rm $csePackageURL || retval=$?
-
 if [ "$retval" -ne 0 ]; then
     err "Failed to delete cse package in storage account. Error code is $retval."
 else

diff --git a/hack/tools/go.mod b/hack/tools/go.mod
@@ -14,6 +14,6 @@ require (
 	github.com/mitchellh/iochan v1.0.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/gomega v1.17.0 // indirect
-	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
+	golang.org/x/sys v0.1.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 )
diff --git a/hack/tools/go.sum b/hack/tools/go.sum
@@ -66,8 +66,9 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh
@@ -9,7 +9,7 @@ assignRootPW() {
         SALT=$(openssl rand -base64 5)
         SECRET=$(openssl rand -base64 37)
         CMD="import crypt, getpass, pwd; print(crypt.crypt('$SECRET', '\$6\$$SALT\$'))"
-        if [[ "${VERSION}" == "22.04" ]]; then
+        if [[ "${VERSION}" == "22.04" || "${VERSION}" == "24.04" ]]; then
             HASH=$(python3 -c "$CMD")
         else
             HASH=$(python -c "$CMD")