Merge branch 'master' into renovate/oss-kubernetes-azure-cloud-node-m…

…anager-1.29.x
Azure · Sep 30, 2024 · 7a2c35c · 7a2c35c
2 parents 36d9464 + 022487d
commit 7a2c35c
Show file tree

Hide file tree

Showing 186 changed files with 341 additions and 289 deletions.
diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml
@@ -20,5 +20,6 @@ pr:
       - pkg/agent/datamodel/sig_config*.go # SIG config changes
       - pkg/agent/datamodel/*.json # SIG version changes
       - pkg/agent/testdata/AKSWindows* # Windows test data
+      - parts/linux/cloud-init/artifacts/components.json # centralized components management file
 jobs:
   - template: ./templates/e2e-template.yaml
diff --git a/.pipelines/templates/e2e-windows-template.yaml b/.pipelines/templates/e2e-windows-template.yaml
@@ -21,6 +21,10 @@ parameters:
 jobs:
 - job: Setup_Test_Cluster
   steps:
+    - bash: |
+        echo "Cleaning up artifacts..."
+        rm -rf $(Build.ArtifactStagingDirectory)/${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-clusterConfig
+      displayName: "Clean up artifacts"
     - bash: |
         az login --identity
         az account set -s $(AZURE_E2E_SUBSCRIPTION_ID)
@@ -82,6 +86,10 @@ jobs:
     matrix: $[ dependencies.Generate_Matrix.outputs['matrix.scenarioMatrix'] ]
     maxParallel: 3
   steps:
+    - bash: |
+        echo "Cleaning up artifacts..."
+        rm -rf $(Build.ArtifactStagingDirectory)/${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-$(SCENARIO_NAME)-logs
+      displayName: "Clean up artifacts"
     - download: current
       artifact: ${{ parameters.windowsImage }}${{ parameters.windowsGpuDriverSuffix }}-clusterConfig
     - bash: |

diff --git a/README.md b/README.md
@@ -136,3 +136,4 @@ Reference: https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html
 Package:
 
 - Calico Windows: https://docs.projectcalico.org/release-notes/
+
diff --git a/e2e/windows/e2e-helper.sh b/e2e/windows/e2e-helper.sh
@@ -85,6 +85,40 @@ upload_linux_file_to_storage_account() {
     fi
 }
 
+check_linux_file_outdated() {
+    declare -l E2E_RESOURCE_GROUP_NAME="$AZURE_E2E_RESOURCE_GROUP_NAME-$WINDOWS_E2E_IMAGE$WINDOWS_GPU_DRIVER_SUFFIX-$K8S_VERSION"
+    E2E_MC_RESOURCE_GROUP_NAME="MC_${E2E_RESOURCE_GROUP_NAME}_${AZURE_E2E_CLUSTER_NAME}_$AZURE_BUILD_LOCATION"
+    MC_VMSS_NAME=$(az vmss list -g $E2E_MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
+    VMSS_INSTANCE_ID="$(az vmss list-instances --name $MC_VMSS_NAME -g $E2E_MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
+
+    linuxFileURL="https://${AZURE_E2E_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${WINDOWS_E2E_STORAGE_CONTAINER}/${MC_VMSS_NAME}-linux-file.zip"
+
+    # download uploaded linux files to old.zip
+    # compress current linux files to new.zip
+    az vmss run-command invoke --command-id RunShellScript \
+        --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
+        --name $MC_VMSS_NAME \
+        --instance-id $VMSS_INSTANCE_ID \
+        --scripts "cat /etc/kubernetes/azure.json > /home/fields.json; cat /etc/kubernetes/certs/apiserver.crt | base64 -w 0 > /home/apiserver.crt; cat /etc/kubernetes/certs/ca.crt | base64 -w 0 > /home/ca.crt; cat /etc/kubernetes/certs/client.key | base64 -w 0 > /home/client.key; cat /var/lib/kubelet/bootstrap-kubeconfig > /home/bootstrap-kubeconfig; cd /home; zip new.zip fields.json apiserver.crt ca.crt client.key bootstrap-kubeconfig; wget https://aka.ms/downloadazcopy-v10-linux; tar -xvf downloadazcopy-v10-linux; cd ./azcopy_*; export AZCOPY_AUTO_LOGIN_TYPE=\"MSI\"; export AZCOPY_MSI_RESOURCE_STRING=\"${AZURE_MSI_RESOURCE_STRING}\"; ./azcopy copy $linuxFileURL /home/old.zip"
+
+    # Use "unzip -d new new.zip" to unzip the compreesed file to a folder
+    # Use "diff -r -q new old" to compare two folder
+    # If files are different in these two folders, the output message will contain "Files xxx and xxx differ"
+    # Example: "Files /home/new/apiserver.crt and /home/old/apiserver.crt differ"
+    compare_message=$(az vmss run-command invoke --command-id RunShellScript \
+        --resource-group $E2E_MC_RESOURCE_GROUP_NAME \
+        --name $MC_VMSS_NAME \
+        --instance-id $VMSS_INSTANCE_ID \
+        --scripts "apt install unzip --yes; unzip -d /home/new /home/new.zip; unzip -d /home/old /home/old.zip; diff -r -q /home/new /home/old")
+
+    if [[ "$compare_message" == *"differ"* ]]; then
+        err "Linux files are outdated."
+        return
+    fi
+
+    log "Linux files are still the latest."
+}
+
 download_linux_file_from_storage_account() {
     local retval
     retval=0

diff --git a/e2e/windows/e2e-starter.sh b/e2e/windows/e2e-starter.sh
@@ -114,10 +114,11 @@ if [ "$create_cluster" == "true" ]; then
         cleanupOutdatedFiles
     fi
 else
-    if [[ "$(check_linux_file_exists_in_storage_account)" == *"Linux file does not exist in storage account."* ]]; then 
+    if [[ "$(check_linux_file_exists_in_storage_account)" == *"Linux file does not exist in storage account."* ]] || [[ "$(check_linux_file_outdated)" == *"Linux files are outdated."* ]]; then 
         upload_linux_file_to_storage_account
     fi
 fi
+
 download_linux_file_from_storage_account
 log "Download of linux file from storage account completed"
 

diff --git a/parts/linux/cloud-init/artifacts/components.json b/parts/linux/cloud-init/artifacts/components.json
@@ -116,8 +116,8 @@
         },
         {
           "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/azure-cns",
-          "latestVersion": "v1.6.7",
-          "previousLatestVersion": "v1.6.5",
+          "latestVersion": "v1.6.10",
+          "previousLatestVersion": "v1.6.7",
           "containerImagePrefetch": {
             "latestVersion": {
               "binaries": [
@@ -616,15 +616,15 @@
             "versionsV2": [
               {
                 "renovateTag": "name=moby-containerd, os=ubuntu, release=22.04",
-                "latestVersion": "1.7.20"
+                "latestVersion": "1.7.22-ubuntu22.04u1"
               }
             ]
           },
           "r2004": {
             "versionsV2": [
               {
                 "renovateTag": "name=moby-containerd, os=ubuntu, release=20.04",
-                "latestVersion": "1.7.20"
+                "latestVersion": "1.7.22-ubuntu20.04u1"
               }
             ]
           },
@@ -714,8 +714,8 @@
               },
               {
                 "renovateTag": "<DO_NOT_UPDATE>",
-                "latestVersion": "1.6.7",
-                "previousLatestVersion": "1.6.3"
+                "latestVersion": "1.6.10",
+                "previousLatestVersion": "1.6.7"
               }
             ],
             "downloadURL": "https://acs-mirror.azureedge.net/azure-cni/v${version}/binaries/azure-vnet-cni-linux-${CPU_ARCH}-v${version}.tgz"

diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh
@@ -223,7 +223,7 @@ wasmFilesExist() {
 installContainerdWasmShims(){
     local download_location=${1}
     PACKAGE_DOWNLOAD_URL=${2} # global URL that is set from the components.json
-    local package_versions=("${@:2}") # Capture all arguments starting from the second indx
+    local package_versions=("${@:3}") # Capture all arguments starting from the third indx
 
     for version in "${package_versions[@]}"; do
         local shims_to_download=("spin" "slight")
@@ -291,7 +291,7 @@ updateContainerdWasmShimsPermissions() {
 installSpinKube(){
     local download_location=${1}
     PACKAGE_DOWNLOAD_URL=${2}
-    local package_versions=("${@:2}") # Capture all arguments starting from the second indx
+    local package_versions=("${@:3}") # Capture all arguments starting from the third indx
 
     for version in "${package_versions[@]}"; do
         containerd_spinkube_url=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL})

diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go
@@ -374,6 +374,9 @@ const (
 	//  of support and image builds have stopped.
 	FrozenCBLMarinerV1SIGImageVersionForDeprecation string = "202308.28.0"
 
+	// AzureLinuxV3 pinned to 202409.23.0 for testing.
+	FrozenAzureLinuxV3SIGImageVersion string = "202409.23.0"
+
 	// This is currently for testing only, we do not build 2404 images regularly
 	// Once 2404 is preview in AKS, we will refer to the images using regular LinuxSIGImageVersion and remove this.
 	Ubuntu2404SIGImageVersionForTest string = "202405.20.0"
@@ -630,7 +633,7 @@ var (
 		ResourceGroup: AKSAzureLinuxResourceGroup,
 		Gallery:       AKSAzureLinuxGalleryName,
 		Definition:    "V3",
-		Version:       LinuxSIGImageVersion,
+		Version:       FrozenAzureLinuxV3SIGImageVersion,
 	}
 
 	SIGCBLMarinerV2Gen2ImageConfigTemplate = SigImageConfigTemplate{
@@ -651,7 +654,7 @@ var (
 		ResourceGroup: AKSAzureLinuxResourceGroup,
 		Gallery:       AKSAzureLinuxGalleryName,
 		Definition:    "V3gen2",
-		Version:       LinuxSIGImageVersion,
+		Version:       FrozenAzureLinuxV3SIGImageVersion,
 	}
 
 	SIGCBLMarinerV2Gen1FIPSImageConfigTemplate = SigImageConfigTemplate{
@@ -672,7 +675,7 @@ var (
 		ResourceGroup: AKSAzureLinuxResourceGroup,
 		Gallery:       AKSAzureLinuxGalleryName,
 		Definition:    "V3fips",
-		Version:       LinuxSIGImageVersion,
+		Version:       FrozenAzureLinuxV3SIGImageVersion,
 	}
 
 	SIGCBLMarinerV2Gen2FIPSImageConfigTemplate = SigImageConfigTemplate{
@@ -693,7 +696,7 @@ var (
 		ResourceGroup: AKSAzureLinuxResourceGroup,
 		Gallery:       AKSAzureLinuxGalleryName,
 		Definition:    "V3gen2fips",
-		Version:       LinuxSIGImageVersion,
+		Version:       FrozenAzureLinuxV3SIGImageVersion,
 	}
 
 	SIGCBLMarinerV2KataImageConfigTemplate = SigImageConfigTemplate{
@@ -728,7 +731,7 @@ var (
 		ResourceGroup: AKSAzureLinuxResourceGroup,
 		Gallery:       AKSAzureLinuxGalleryName,
 		Definition:    "V3gen2arm64",
-		Version:       LinuxSIGImageVersion,
+		Version:       FrozenAzureLinuxV3SIGImageVersion,
 	}
 
 	SIGCBLMarinerV2TLImageConfigTemplate = SigImageConfigTemplate{

diff --git a/pkg/agent/datamodel/sig_config_test.go b/pkg/agent/datamodel/sig_config_test.go
@@ -75,7 +75,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() {
 		Expect(azurelinuxV3.ResourceGroup).To(Equal("resourcegroup"))
 		Expect(azurelinuxV3.Gallery).To(Equal("aksazurelinux"))
 		Expect(azurelinuxV3.Definition).To(Equal("V3"))
-		Expect(azurelinuxV3.Version).To(Equal(LinuxSIGImageVersion))
+		Expect(azurelinuxV3.Version).To(Equal(FrozenAzureLinuxV3SIGImageVersion))
 
 		azurelinuxV2Gen2 := sigConfig.SigAzureLinuxImageConfig[AKSAzureLinuxV2Gen2]
 		Expect(azurelinuxV2Gen2.ResourceGroup).To(Equal("resourcegroup"))
@@ -87,7 +87,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() {
 		Expect(azurelinuxV3Gen2.ResourceGroup).To(Equal("resourcegroup"))
 		Expect(azurelinuxV3Gen2.Gallery).To(Equal("aksazurelinux"))
 		Expect(azurelinuxV3Gen2.Definition).To(Equal("V3gen2"))
-		Expect(azurelinuxV3Gen2.Version).To(Equal(LinuxSIGImageVersion))
+		Expect(azurelinuxV3Gen2.Version).To(Equal(FrozenAzureLinuxV3SIGImageVersion))
 
 		Expect(len(sigConfig.SigWindowsImageConfig)).To(Equal(6))
 
@@ -173,7 +173,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() {
 		Expect(azurelinuxV3Arm64.ResourceGroup).To(Equal("resourcegroup"))
 		Expect(azurelinuxV3Arm64.Gallery).To(Equal("aksazurelinux"))
 		Expect(azurelinuxV3Arm64.Definition).To(Equal("V3gen2arm64"))
-		Expect(azurelinuxV3Arm64.Version).To(Equal(LinuxSIGImageVersion))
+		Expect(azurelinuxV3Arm64.Version).To(Equal(FrozenAzureLinuxV3SIGImageVersion))
 
 		aksUbuntu2204TLGen2Containerd := sigConfig.SigUbuntuImageConfig[AKSUbuntuContainerd2204TLGen2]
 		Expect(aksUbuntu2204TLGen2Containerd.ResourceGroup).To(Equal("resourcegroup"))
@@ -241,7 +241,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() {
 		Expect(azurelinuxV3FIPS.ResourceGroup).To(Equal("resourcegroup"))
 		Expect(azurelinuxV3FIPS.Gallery).To(Equal("aksazurelinux"))
 		Expect(azurelinuxV3FIPS.Definition).To(Equal("V3fips"))
-		Expect(azurelinuxV3FIPS.Version).To(Equal(LinuxSIGImageVersion))
+		Expect(azurelinuxV3FIPS.Version).To(Equal(FrozenAzureLinuxV3SIGImageVersion))
 
 		marinerV2Gen2FIPS := sigConfig.SigCBLMarinerImageConfig[AKSCBLMarinerV2Gen2FIPS]
 		Expect(marinerV2Gen2FIPS.ResourceGroup).To(Equal("resourcegroup"))
@@ -259,7 +259,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() {
 		Expect(azurelinuxV3Gen2FIPS.ResourceGroup).To(Equal("resourcegroup"))
 		Expect(azurelinuxV3Gen2FIPS.Gallery).To(Equal("aksazurelinux"))
 		Expect(azurelinuxV3Gen2FIPS.Definition).To(Equal("V3gen2fips"))
-		Expect(azurelinuxV3Gen2FIPS.Version).To(Equal(LinuxSIGImageVersion))
+		Expect(azurelinuxV3Gen2FIPS.Version).To(Equal(FrozenAzureLinuxV3SIGImageVersion))
 
 		azurelinuxV2Gen2Kata := sigConfig.SigAzureLinuxImageConfig[AKSAzureLinuxV2Gen2Kata]
 		Expect(azurelinuxV2Gen2Kata.ResourceGroup).To(Equal("resourcegroup"))
Original file line number	Diff line number	Diff line change
Expand Up		@@ -136,3 +136,4 @@ Reference: https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html
		Package:

		- Calico Windows: https://docs.projectcalico.org/release-notes/