feat: cache and use Linux private packages for kubctl and kubelet.

.pipelines/templates/.builder-release-template.yaml

@@ -105,6 +105,8 @@ steps:
         -e SGX_INSTALL=${SGX_INSTALL} \
         -e ENABLE_CGROUPV2=${ENABLE_CGROUPV2} \
         -e IMAGE_VERSION=${IMAGE_VERSION} \
+        -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+        -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
         ${CONTAINER_IMAGE} make -f packer.mk run-packer
     displayName: Building VHD
   - script: |
@@ -138,6 +140,8 @@ steps:
         -e SGX_INSTALL=${SGX_INSTALL} \
         -e ENABLE_CGROUPV2=${ENABLE_CGROUPV2} \
         -e GIT_BRANCH=$(Build.SourceBranch) \
+        -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+        -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
         ${CONTAINER_IMAGE} make -f packer.mk test-building-vhd
     displayName: Run VHD Tests
   - task: PublishPipelineArtifact@0
@@ -160,6 +164,8 @@ steps:
       ARM64_OS_DISK_SNAPSHOT_NAME="$(cat vhdbuilder/packer/settings.json | grep "arm64_os_disk_snapshot_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       IMAGE_NAME="${SIG_IMAGE_NAME}-${captured_sig_version}" && \
+      PRIVATE_PACKAGES_URL="$(cat vhdbuilder/packer/settings.json | grep "private_packages_url" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
+      LINUX_MSI_RESOURCE_ID="$(cat vhdbuilder/packer/settings.json | grep "linux_msi_resource_ids" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       docker run --rm \
       -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
       -w /go/src/github.com/Azure/AgentBaker \
@@ -180,6 +186,8 @@ steps:
       -e SIG_IMAGE_NAME=${SIG_IMAGE_NAME} \
       -e ARCHITECTURE=${ARCHITECTURE} \
       -e ARM64_OS_DISK_SNAPSHOT_NAME=${ARM64_OS_DISK_SNAPSHOT_NAME} \
+      -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+      -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
       ${CONTAINER_IMAGE} make -f packer.mk cleanup
     displayName: Clean Up Packer Generated Resources
     condition: always()
@@ -188,6 +196,8 @@ steps:
         captured_sig_version="$(cat vhdbuilder/packer/settings.json | grep "captured_sig_version" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
         SIG_GALLERY_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_gallery_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
         SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
+        PRIVATE_PACKAGES_URL="$(cat vhdbuilder/packer/settings.json | grep "private_packages_url" | awk -F\" '{print $4}')" && \
+        LINUX_MSI_RESOURCE_ID="$(cat vhdbuilder/packer/settings.json | grep "linux_msi_resource_id" | awk -F\" '{print $4}')" && \
         docker run --rm \
         -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
         -w /go/src/github.com/Azure/AgentBaker \
@@ -206,6 +216,8 @@ steps:
         -e SIG_IMAGE_NAME=${SIG_IMAGE_NAME} \
         -e CAPTURED_SIG_VERSION=${captured_sig_version} \
         -e ENABLE_TRUSTED_LAUNCH=${ENABLE_TRUSTED_LAUNCH} \
+        -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+        -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
         ${CONTAINER_IMAGE} make -f packer.mk convert-sig-to-classic-storage-account-blob
     displayName: Convert Shared Image Gallery To VHD Blob In Classic Storage Account
     condition: eq(variables.DRY_RUN, 'False')
@@ -253,4 +265,4 @@ steps:
       vstsFeedPublish: 'CloudNativeCompute/AKS-Linux-VHD-Artifacts'
       vstsFeedPackagePublish: 'publishing-info-${{ parameters.artifactName }}'
       packagePublishDescription: 'VHD publishing info ${{ parameters.artifactName }}'
-      versionOption: minor
+      versionOption: minor

e2e/scenario/base_vhd_catalog.json

@@ -13,6 +13,9 @@
         "gen2containerd": {
             "artifactName": "2204-gen2-containerd",
             "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1699020621.21398"
+        },
+        "gen2containerdprivatekubepkg": {
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1704411049.2812"
         }
     },
     "azurelinuxv2": {
@@ -35,4 +38,4 @@
             "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/CBLMarinerV2Gen2/versions/1.1699020635.14957"
         }
     }
-}
+}

e2e/scenario/scenario.go

@@ -69,6 +69,7 @@ func (t *Template) scenarios() []*Scenario {
 		t.ubuntu2204gpuNoDriver(),
 		t.ubuntu2204CustomCATrust(),
 		t.ubuntu2204ArtifactStreaming(),
+		t.ubuntu2204privatekubepkg(),
 	}
 }
 

e2e/scenario/scenario_ubuntu2204-private-kube-pkg.go

@@ -0,0 +1,23 @@
+package scenario
+
+import (
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
+)
+
+func (t *Template) ubuntu2204privatekubepkg() *Scenario {
+	return &Scenario{
+		Name:        "ubuntu2204privatekubepkg",
+		Description: "Tests that a node using the Ubuntu 2204 VHD that was built with private kube packages can be properly bootstrapped with the specified kube version",
+		Config: Config{
+			ClusterSelector: NetworkPluginKubenetSelector,
+			ClusterMutator:  NetworkPluginKubenetMutator,
+			VHDSelector:     t.Ubuntu2204Gen2ContainerdPrivateKubePkg,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-ubuntu-containerd-22.04-gen2"
+				nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = "1.25.6"
+				nbc.AgentPoolProfile.Distro = "aks-ubuntu-containerd-22.04-gen2"
+				nbc.K8sComponents.LinuxPrivatePackageURL = "https://privatekube.blob.core.windows.net/kubernetes/v1.25.6-hotfix.20230612/binaries/v1.25.6-hotfix.20230612.tar.gz"
+			},
+		},
+	}
+}

e2e/scenario/vhd.go

@@ -133,8 +133,9 @@ type Ubuntu1804 struct {
 
 // Ubuntu2204 contains all the Ubuntu2204-based VHD catalog entries.
 type Ubuntu2204 struct {
-	Gen2Arm64Containerd VHD `json:"gen2arm64containerd,omitempty"`
-	Gen2Containerd      VHD `json:"gen2containerd,omitempty"`
+	Gen2Arm64Containerd          VHD `json:"gen2arm64containerd,omitempty"`
+	Gen2Containerd               VHD `json:"gen2containerd,omitempty"`
+	Gen2ContainerdPrivateKubePkg VHD `json:"gen2containerdprivatekubepkg,omitempty"`
 }
 
 // AzureLinuxV2 contains all the AzureLinuxV2-based VHD catalog entries.
@@ -164,6 +165,11 @@ func (c *VHDCatalog) Ubuntu2204Gen2Containerd() VHD {
 	return c.Ubuntu2204.Gen2Containerd
 }
 
+// Returns the gen2containerdprivatekubepkg catalog entry.
+func (c *VHDCatalog) Ubuntu2204Gen2ContainerdPrivateKubePkg() VHD {
+	return c.Ubuntu2204.Gen2ContainerdPrivateKubePkg
+}
+
 // Returns the AzureLinuxV/gen2arm64 catalog entry.
 func (c *VHDCatalog) AzureLinuxV2Gen2ARM64() VHD {
 	return c.AzureLinuxV2.Gen2Arm64

e2e/template.go

@@ -257,6 +257,7 @@ func baseTemplate(location string) *datamodel.NodeBootstrappingConfiguration {
 			PodInfraContainerImageURL: "mcr.microsoft.com/oss/kubernetes/pause:3.6",
 			HyperkubeImageURL:         "mcr.microsoft.com/oss/kubernetes/",
 			WindowsPackageURL:         "windowspackage",
+			LinuxPrivatePackageURL:    "",
 		},
 		AgentPoolProfile: &datamodel.AgentPoolProfile{
 			Name:                "nodepool2",

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -16,6 +16,7 @@ KUBERNETES_VERSION={{GetParameter "kubernetesVersion"}}
 HYPERKUBE_URL={{GetParameter "kubernetesHyperkubeSpec"}}
 KUBE_BINARY_URL={{GetParameter "kubeBinaryURL"}}
 CUSTOM_KUBE_BINARY_URL={{GetParameter "customKubeBinaryURL"}}
+PRIVATE_KUBE_BINARY_URL="{{GetLinuxPrivatePackageURL}}"
 KUBEPROXY_URL={{GetParameter "kubeProxySpec"}}
 APISERVER_PUBLIC_KEY={{GetParameter "apiServerCertificate"}}
 SUBSCRIPTION_ID={{GetVariable "subscriptionId"}}

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# ERR_SYSTEMCTL_ENABLE_FAIL=3 Service could not be enabled by systemctl -- DEPRECATED 
+# ERR_SYSTEMCTL_ENABLE_FAIL=3 Service could not be enabled by systemctl -- DEPRECATED
 ERR_SYSTEMCTL_START_FAIL=4 # Service could not be started or enabled by systemctl
 ERR_CLOUD_INIT_TIMEOUT=5 # Timeout waiting for cloud-init runcmd to complete
 ERR_FILE_WATCH_TIMEOUT=6 # Timeout waiting for a file
@@ -93,6 +93,9 @@ ERR_VHD_REBOOT_REQUIRED=200 # Reserved for VHD reboot required exit condition
 ERR_NO_PACKAGES_FOUND=201 # Reserved for no security packages found exit condition
 ERR_SNAPSHOT_UPDATE_START_FAIL=202 # snapshot-update could not be started by systemctl
 
+ERR_PRIVATE_K8S_PKG_ERR=203 # Error downloading (at build-time) or extracting (at run-time) private kubernetes packages
+ERR_PRIVATE_K8S_INSTALL_ERR=204 # Error installing kubernetes binaries on disk
+
 ERR_SYSTEMCTL_MASK_FAIL=2 # Service could not be masked by systemctl
 
 OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
@@ -265,7 +268,7 @@ systemctlDisableAndStop() {
     fi
 }
 
-# return true if a >= b 
+# return true if a >= b
 semverCompare() {
     VERSION_A=$(echo $1 | cut -d "+" -f 1)
     VERSION_B=$(echo $2 | cut -d "+" -f 1)
@@ -334,7 +337,7 @@ logs_to_events() {
         --arg Version     "1.23" \
         --arg TaskName    "${task}" \
         --arg EventLevel  "Informational" \
-        --arg Message     "Completed: ${@}" \
+        --arg Message     "Completed: $*" \
         --arg EventPid    "0" \
         --arg EventTid    "0" \
         '{Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid}'

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -10,6 +10,7 @@ CRICTL_BIN_DIR="/usr/local/bin"
 CONTAINERD_DOWNLOADS_DIR="/opt/containerd/downloads"
 RUNC_DOWNLOADS_DIR="/opt/runc/downloads"
 K8S_DOWNLOADS_DIR="/opt/kubernetes/downloads"
+K8S_CACHE_DIR="/opt/kubernetes/downloads/private-packages"
 UBUNTU_RELEASE=$(lsb_release -r -s)
 SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR="/opt/azure/tlsbootstrap"
 SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION="v0.1.0-alpha.2"
@@ -229,34 +230,71 @@ installAzureCNI() {
 }
 
 extractKubeBinaries() {
-    K8S_VERSION=$1
-    KUBE_BINARY_URL=$2
+    local k8s_version="$1"
+    local kube_binary_url="$2"
+    local is_private_url="$3"
 
     mkdir -p ${K8S_DOWNLOADS_DIR}
-    K8S_TGZ_TMP=${KUBE_BINARY_URL##*/}
-    retrycmd_get_tarball 120 5 "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" ${KUBE_BINARY_URL} || exit $ERR_K8S_DOWNLOAD_TIMEOUT
-    tar --transform="s|.*|&-${K8S_VERSION}|" --show-transformed-names -xzvf "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" \
+    local k8s_tgz_tmp_fn=${kube_binary_url##*/}
+    k8s_tgz_tmp="${K8S_DOWNLOADS_DIR}/${k8s_tgz_tmp_fn}"
+
+    local err=$ERR_K8S_DOWNLOAD_TIMEOUT
+    if [[ $is_private_url == true ]]; then
+        k8s_tgz_tmp="${K8S_CACHE_DIR}/${k8s_tgz_tmp_fn}"
+        if [[ -f "${k8s_tgz_tmp}" ]]; then
+            echo "cached package ${k8s_tgz_tmp} is found, will use that"
+        else
+            echo "cached package ${k8s_tgz_tmp} not found"
+            return 1
+        fi
+
+        # remove the current kubelet and kubectl binaries before extracting new binaries from the cached package
+        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        err=$ERR_PRIVATE_K8S_PKG_ERR
+    fi
+
+    retrycmd_get_tarball 120 5 "${k8s_tgz_tmp}" ${kube_binary_url} || exit "$err"
+    if [ ! -f ${k8s_tgz_tmp} ]; then
+        exit "$err"
+    fi
+    tar --transform="s|.*|&-${k8s_version}|" --show-transformed-names -xzvf "${k8s_tgz_tmp}" \
         --strip-components=3 -C /usr/local/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
-    rm -f "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}"
+    if [ ! -f /usr/local/bin/kubectl-${k8s_version} ] || [ ! -f /usr/local/bin/kubelet-${k8s_version} ]; then
+        exit $ERR_PRIVATE_K8S_INSTALL_ERR
+    fi
+
+    if [[ $is_private_url == false ]]; then
+        rm -f "${k8s_tgz_tmp}"
+    fi
 }
 
 installKubeletKubectlAndKubeProxy() {
-
+    # when both, custom and private urls for kubernetes packages are set, custom url will be used and private url will be ignored
     CUSTOM_KUBE_BINARY_DOWNLOAD_URL="${CUSTOM_KUBE_BINARY_URL:=}"
+    PRIVATE_KUBE_BINARY_DOWNLOAD_URL="${PRIVATE_KUBE_BINARY_URL:=}"
+    echo "using private url: ${PRIVATE_KUBE_BINARY_DOWNLOAD_URL}, custom url: ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}"
+    install_default_if_missing=true
+
     if [[ ! -z ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL} ]]; then
-        # remove the kubelet binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL
+        # remove the kubelet and kubectl binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL
         rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
 
         # NOTE(mainred): we expect kubelet binary to be under `kubernetes/node/bin`. This suits the current setting of
         # kube binaries used by AKS and Kubernetes upstream.
         # TODO(mainred): let's see if necessary to auto-detect the path of kubelet
-        logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}
+        logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL} false
+        install_default_if_missing=false
+    elif [[ ! -z ${PRIVATE_KUBE_BINARY_DOWNLOAD_URL} ]]; then
+        # extract new binaries from the cached package if exists (cached at build-time)
+        logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${PRIVATE_KUBE_BINARY_DOWNLOAD_URL} true
+    fi
 
-    else
-        if [[ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ]]; then
+    # if the custom url is not specified and the required kubectl/kubelet-version via private url is not installed, install using the default url/package
+    if [[ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ]] || [[ ! -f "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" ]]; then
+        if [[ "$install_default_if_missing" == true ]]; then
             #TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
             if (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
-                logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${KUBE_BINARY_URL}
+                logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${KUBE_BINARY_URL} false
             fi
         fi
     fi
@@ -284,7 +322,7 @@ retagContainerImage() {
     CLI_TOOL=$1
     CONTAINER_IMAGE_URL=$2
     RETAG_IMAGE_URL=$3
-    echo "retaging from ${CONTAINER_IMAGE_URL} to ${RETAG_IMAGE_URL} using ${CLI_TOOL}"
+    echo "retagging from ${CONTAINER_IMAGE_URL} to ${RETAG_IMAGE_URL} using ${CLI_TOOL}"
     if [[ ${CLI_TOOL} == "ctr" ]]; then
         ctr --namespace k8s.io image tag $CONTAINER_IMAGE_URL $RETAG_IMAGE_URL
     elif [[ ${CLI_TOOL} == "crictl" ]]; then
@@ -415,4 +453,4 @@ datasource:
         apply_network_config: false
 EOF
 }
-#EOF
+#EOF

pkg/agent/baker.go

@@ -707,6 +707,9 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
 		"GetHyperkubeImageReference": func() string {
 			return config.K8sComponents.HyperkubeImageURL
 		},
+		"GetLinuxPrivatePackageURL": func() string {
+			return config.K8sComponents.LinuxPrivatePackageURL
+		},
 		"GetTargetEnvironment": func() string {
 			if cs.IsAKSCustomCloud() {
 				return cs.Properties.CustomCloudEnv.Name

pkg/agent/datamodel/types.go

@@ -1606,6 +1606,10 @@ type K8sComponents struct {
 	// Full path to the Windows package (windowszip) to use.
 	// For example: https://acs-mirror.azureedge.net/kubernetes/v1.17.8/windowszip/v1.17.8-1int.zip.
 	WindowsPackageURL string
+
+	// Full path to the Linux package (tar.gz) to use.
+	// For example: url=https://acs-mirror.azureedge.net/kubernetes/v1.25.6-hotfix.20230612/binaries/v1.25.6-hotfix.20230612.tar.gz
+	LinuxPrivatePackageURL string
 }
 
 // GetLatestSigImageConfigRequest describes the input for a GetLatestSigImageConfig HTTP request.