feat: cache and use Linux private packages for kube-proxy, kubctl and…

… kubelet

.pipelines/.vsts-vhd-builder-release.yaml

@@ -35,15 +35,15 @@ parameters:
   type: boolean
   default: false
 - name: buildMarinerV2gen1fips
-  displayName: Build MarinerV2 Gen1 FIPS 
+  displayName: Build MarinerV2 Gen1 FIPS
   type: boolean
   default: true
 - name: buildAzureLinuxV2gen1fips
   displayName: Build AzureLinuxV2 Gen1 FIPS
   type: boolean
   default: false
 - name: buildMarinerV2gen2fips
-  displayName: Build MarinerV2 Gen2 FIPS 
+  displayName: Build MarinerV2 Gen2 FIPS
   type: boolean
   default: true
 - name: buildAzureLinuxV2gen2fips
@@ -134,7 +134,7 @@ parameters:
 variables:
 - name: CONTAINER_IMAGE
   value: 'mcr.microsoft.com/oss/azcu/go-dev:v1.34.7'
-  
+
 pool:
   name: $(POOL_NAME)
 

.pipelines/.vsts-vhd-builder.yaml

@@ -106,4 +106,3 @@ stages:
         - template: ./templates/.builder-release-template.yaml
           parameters:
             artifactName: marinerv2-gen2
-

.pipelines/templates/.builder-release-template.yaml

@@ -105,6 +105,9 @@ steps:
         -e SGX_INSTALL=${SGX_INSTALL} \
         -e ENABLE_CGROUPV2=${ENABLE_CGROUPV2} \
         -e IMAGE_VERSION=${IMAGE_VERSION} \
+        -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+        -e PRIVATE_KUBE_PROXY_IMAGES="${PRIVATE_KUBE_PROXY_IMAGES}" \
+        -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
         ${CONTAINER_IMAGE} make -f packer.mk run-packer
     displayName: Building VHD
   - script: |
@@ -138,6 +141,9 @@ steps:
         -e SGX_INSTALL=${SGX_INSTALL} \
         -e ENABLE_CGROUPV2=${ENABLE_CGROUPV2} \
         -e GIT_BRANCH=$(Build.SourceBranch) \
+        -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+        -e PRIVATE_KUBE_PROXY_IMAGES="${PRIVATE_KUBE_PROXY_IMAGES}" \
+        -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
         ${CONTAINER_IMAGE} make -f packer.mk test-building-vhd
     displayName: Run VHD Tests
   - task: PublishPipelineArtifact@0
@@ -160,6 +166,9 @@ steps:
       ARM64_OS_DISK_SNAPSHOT_NAME="$(cat vhdbuilder/packer/settings.json | grep "arm64_os_disk_snapshot_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       IMAGE_NAME="${SIG_IMAGE_NAME}-${captured_sig_version}" && \
+      PRIVATE_PACKAGES_URL="$(cat vhdbuilder/packer/settings.json | grep "private_packages_url" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
+      PRIVATE_KUBE_PROXY_IMAGES="$(cat vhdbuilder/packer/settings.json | grep "private_kube_proxy_images" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
+      LINUX_MSI_RESOURCE_ID="$(cat vhdbuilder/packer/settings.json | grep "linux_msi_resource_ids" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       docker run --rm \
       -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
       -w /go/src/github.com/Azure/AgentBaker \
@@ -180,14 +189,24 @@ steps:
       -e SIG_IMAGE_NAME=${SIG_IMAGE_NAME} \
       -e ARCHITECTURE=${ARCHITECTURE} \
       -e ARM64_OS_DISK_SNAPSHOT_NAME=${ARM64_OS_DISK_SNAPSHOT_NAME} \
+      -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+      -e PRIVATE_KUBE_PROXY_IMAGES="${PRIVATE_KUBE_PROXY_IMAGES}" \
+      -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
       ${CONTAINER_IMAGE} make -f packer.mk cleanup
     displayName: Clean Up Packer Generated Resources
     condition: always()
   - bash: |
         echo MODE=$(MODE) && \
+        echo "container image: ${CONTAINER_IMAGE}" && \
         captured_sig_version="$(cat vhdbuilder/packer/settings.json | grep "captured_sig_version" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
         SIG_GALLERY_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_gallery_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
         SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
+        PRIVATE_PACKAGES_URL="$(cat vhdbuilder/packer/settings.json | grep "private_packages_url" | awk -F\" '{print $4}')" && \
+        echo "private_packages_url: $PRIVATE_PACKAGES_URL" && \
+        PRIVATE_KUBE_PROXY_IMAGES="$(cat vhdbuilder/packer/settings.json | grep "private_kube_proxy_images" | awk -F\" '{print $4}')" && \
+        echo "private_kube_proxy_images: $PRIVATE_KUBE_PROXY_IMAGES" && \
+        LINUX_MSI_RESOURCE_ID="$(cat vhdbuilder/packer/settings.json | grep "linux_msi_resource_id" | awk -F\" '{print $4}')" && \
+        echo "linux_msi_resource_id: $LINUX_MSI_RESOURCE_ID" && \
         docker run --rm \
         -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
         -w /go/src/github.com/Azure/AgentBaker \
@@ -206,11 +225,15 @@ steps:
         -e SIG_IMAGE_NAME=${SIG_IMAGE_NAME} \
         -e CAPTURED_SIG_VERSION=${captured_sig_version} \
         -e ENABLE_TRUSTED_LAUNCH=${ENABLE_TRUSTED_LAUNCH} \
+        -e PRIVATE_PACKAGES_URL="${PRIVATE_PACKAGES_URL}" \
+        -e PRIVATE_KUBE_PROXY_IMAGES="${PRIVATE_KUBE_PROXY_IMAGES}" \
+        -e LINUX_MSI_RESOURCE_ID=${AZURE_MSI_RESOURCE_ID} \
         ${CONTAINER_IMAGE} make -f packer.mk convert-sig-to-classic-storage-account-blob
     displayName: Convert Shared Image Gallery To VHD Blob In Classic Storage Account
     condition: eq(variables.DRY_RUN, 'False')
   - bash: |
         echo MODE=$(MODE) && \
+        echo "container image: ${CONTAINER_IMAGE}" && \
         captured_sig_version="$(cat vhdbuilder/packer/settings.json | grep "captured_sig_version" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
         SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
         VHD_NAME="${captured_sig_version}.vhd" && \

e2e/scenario/base_vhd_catalog.json

@@ -13,6 +13,10 @@
         "gen2containerd": {
             "artifactName": "2204-gen2-containerd",
             "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1699020621.21398"
+        },
+        "gen2containerdprivatekubepkg": {
+            "artifactName": "2204-gen2-containerd",
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1703538073.30837"
         }
     },
     "azurelinuxv2": {

e2e/scenario/scenario.go

@@ -69,6 +69,7 @@ func (t *Template) scenarios() []*Scenario {
 		t.ubuntu2204gpuNoDriver(),
 		t.ubuntu2204CustomCATrust(),
 		t.ubuntu2204ArtifactStreaming(),
+		t.ubuntu2204privatekubepkg(),
 	}
 }
 

e2e/scenario/scenario_ubuntu1804.go

@@ -1,5 +1,9 @@
 package scenario
 
+import (
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
+)
+
 // Returns config for the 'base' E2E scenario
 func (t *Template) ubuntu1804() *Scenario {
 	return &Scenario{
@@ -9,6 +13,11 @@ func (t *Template) ubuntu1804() *Scenario {
 			ClusterSelector: NetworkPluginKubenetSelector,
 			ClusterMutator:  NetworkPluginKubenetMutator,
 			VHDSelector:     t.Ubuntu1804Gen2Containerd,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-ubuntu-containerd-18.04"
+				nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = "1.28.3"
+				nbc.AgentPoolProfile.Distro = "aks-ubuntu-containerd-18.04"
+			},
 		},
 	}
 }

e2e/scenario/scenario_ubuntu2204-private-kube-pkg.go

@@ -0,0 +1,23 @@
+package scenario
+
+import (
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
+)
+
+func (t *Template) ubuntu2204privatekubepkg() *Scenario {
+	return &Scenario{
+		Name:        "ubuntu2204privatekubepkg",
+		Description: "Tests that a node using the Ubuntu 2204 VHD that was built with private kube packages can be properly bootstrapped with the specified kube version",
+		Config: Config{
+			ClusterSelector: NetworkPluginKubenetSelector,
+			ClusterMutator:  NetworkPluginKubenetMutator,
+			VHDSelector:     t.Ubuntu2204Gen2ContainerdPrivateKubePkg,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-ubuntu-containerd-22.04-gen2"
+				nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = "1.25.6"
+				nbc.AgentPoolProfile.Distro = "aks-ubuntu-containerd-22.04-gen2"
+				nbc.K8sComponents.LinuxPrivatePackageURL = "https://privatekube.blob.core.windows.net/kubernetes/v1.25.6-hotfix.20230612/binaries/v1.25.6-hotfix.20230612.tar.gz"
+			},
+		},
+	}
+}

e2e/scenario/vhd.go

@@ -133,8 +133,9 @@ type Ubuntu1804 struct {
 
 // Ubuntu2204 contains all the Ubuntu2204-based VHD catalog entries.
 type Ubuntu2204 struct {
-	Gen2Arm64Containerd VHD `json:"gen2arm64containerd,omitempty"`
-	Gen2Containerd      VHD `json:"gen2containerd,omitempty"`
+	Gen2Arm64Containerd               VHD `json:"gen2arm64containerd,omitempty"`
+	Gen2Containerd                    VHD `json:"gen2containerd,omitempty"`
+	Gen2Arm64ContainerdPrivateKubePkg VHD `json:"gen2containerdprivatekubepkg,omitempty"`
 }
 
 // AzureLinuxV2 contains all the AzureLinuxV2-based VHD catalog entries.
@@ -164,6 +165,11 @@ func (c *VHDCatalog) Ubuntu2204Gen2Containerd() VHD {
 	return c.Ubuntu2204.Gen2Containerd
 }
 
+// Returns the gen2containerdprivatekubepkg catalog entry.
+func (c *VHDCatalog) Ubuntu2204Gen2ContainerdPrivateKubePkg() VHD {
+	return c.Ubuntu2204.Gen2Arm64ContainerdPrivateKubePkg
+}
+
 // Returns the AzureLinuxV/gen2arm64 catalog entry.
 func (c *VHDCatalog) AzureLinuxV2Gen2ARM64() VHD {
 	return c.AzureLinuxV2.Gen2Arm64

e2e/template.go

@@ -257,6 +257,7 @@ func baseTemplate(location string) *datamodel.NodeBootstrappingConfiguration {
 			PodInfraContainerImageURL: "mcr.microsoft.com/oss/kubernetes/pause:3.6",
 			HyperkubeImageURL:         "mcr.microsoft.com/oss/kubernetes/",
 			WindowsPackageURL:         "windowspackage",
+			LinuxPrivatePackageURL:    "linuxPrivatepackagesURL",
 		},
 		AgentPoolProfile: &datamodel.AgentPoolProfile{
 			Name:                "nodepool2",

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -16,6 +16,7 @@ KUBERNETES_VERSION={{GetParameter "kubernetesVersion"}}
 HYPERKUBE_URL={{GetParameter "kubernetesHyperkubeSpec"}}
 KUBE_BINARY_URL={{GetParameter "kubeBinaryURL"}}
 CUSTOM_KUBE_BINARY_URL={{GetParameter "customKubeBinaryURL"}}
+PRIVATE_KUBE_BINARY_URL="{{GetLinuxPrivatePackageURL}}"
 KUBEPROXY_URL={{GetParameter "kubeProxySpec"}}
 APISERVER_PUBLIC_KEY={{GetParameter "apiServerCertificate"}}
 SUBSCRIPTION_ID={{GetVariable "subscriptionId"}}

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -10,6 +10,7 @@ CRICTL_BIN_DIR="/usr/local/bin"
 CONTAINERD_DOWNLOADS_DIR="/opt/containerd/downloads"
 RUNC_DOWNLOADS_DIR="/opt/runc/downloads"
 K8S_DOWNLOADS_DIR="/opt/kubernetes/downloads"
+K8S_CACHE_DIR="/opt/kubernetes/downloads/private-packages"
 UBUNTU_RELEASE=$(lsb_release -r -s)
 SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR="/opt/azure/tlsbootstrap"
 SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION="v0.1.0-alpha.2"
@@ -240,9 +241,32 @@ extractKubeBinaries() {
     rm -f "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}"
 }
 
-installKubeletKubectlAndKubeProxy() {
+extractPrivateKubeBinaries() {
+    K8S_VERSION=$1
+    KUBE_BINARY_URL=$2
+
+    K8S_TGZ_TMP=${KUBE_BINARY_URL##*/}
+    CACHED_PKG="${K8S_CACHE_DIR}/${K8S_TGZ_TMP}"
+
+    if [[ -f "${CACHED_PKG}" ]]; then
+        echo "cached package ${CACHED_PKG} is found, will use that"
+    else
+        echo "cached package ${CACHED_PKG} not found"
+        return 1
+    fi
+
+    retrycmd_get_tarball 120 5 "${CACHED_PKG}" ${KUBE_BINARY_URL} || exit $ERR_K8S_DOWNLOAD_TIMEOUT
+    tar --transform="s|.*|&-${K8S_VERSION}|" --show-transformed-names -xzvf "${CACHED_PKG}" \
+        --strip-components=3 -C /usr/local/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
+    # TODO: confirm
+    # rm -f "$K8S_CACHE_DIR/${K8S_TGZ_TMP}" // don't delete the cached package
+}
 
+installKubeletKubectlAndKubeProxy() {
     CUSTOM_KUBE_BINARY_DOWNLOAD_URL="${CUSTOM_KUBE_BINARY_URL:=}"
+    PRIVATE_KUBE_BINARY_DOWNLOAD_URL="${PRIVATE_KUBE_BINARY_URL:=}"
+    echo "using private url: ${PRIVATE_KUBE_BINARY_DOWNLOAD_URL}, custom url: ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}"
+
     if [[ ! -z ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL} ]]; then
         # remove the kubelet binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL
         rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
@@ -251,7 +275,10 @@ installKubeletKubectlAndKubeProxy() {
         # kube binaries used by AKS and Kubernetes upstream.
         # TODO(mainred): let's see if necessary to auto-detect the path of kubelet
         logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} ${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}
-
+    elif [[ ! -z ${PRIVATE_KUBE_BINARY_DOWNLOAD_URL} ]]; then
+        # remove the kubelet binaries to make sure the only binary left is from the PRIVATE_KUBE_BINARY_DOWNLOAD_URL
+        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractPrivateKubeBinaries" extractPrivateKubeBinaries ${KUBERNETES_VERSION} ${PRIVATE_KUBE_BINARY_DOWNLOAD_URL}
     else
         if [[ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ]]; then
             #TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
@@ -284,7 +311,7 @@ retagContainerImage() {
     CLI_TOOL=$1
     CONTAINER_IMAGE_URL=$2
     RETAG_IMAGE_URL=$3
-    echo "retaging from ${CONTAINER_IMAGE_URL} to ${RETAG_IMAGE_URL} using ${CLI_TOOL}"
+    echo "retagging from ${CONTAINER_IMAGE_URL} to ${RETAG_IMAGE_URL} using ${CLI_TOOL}"
     if [[ ${CLI_TOOL} == "ctr" ]]; then
         ctr --namespace k8s.io image tag $CONTAINER_IMAGE_URL $RETAG_IMAGE_URL
     elif [[ ${CLI_TOOL} == "crictl" ]]; then
@@ -324,14 +351,50 @@ removeContainerImage() {
     CONTAINER_IMAGE_URL=$2
     if [[ "${CLI_TOOL}" == "docker" ]]; then
         docker image rm $CONTAINER_IMAGE_URL
+    elif [[ "${CLI_TOOL}" == "ctr" ]]; then
+        ctr -n k8s.io image rm $CONTAINER_IMAGE_URL
     else
         # crictl should always be present
         crictl rmi $CONTAINER_IMAGE_URL
     fi
 }
 
+# retag the given image with mcr.microsoft.* base for all the clouds
+# TODO: current code tags all the images for Mooncake but not for AGC cloud, confirm if still needed
+retagImageForAllClouds() {
+    CLI_TOOL=$1
+    CONTAINER_IMAGE=$2
+
+    echo "retagging image: $CONTAINER_IMAGE to mcr.microsoft.* base for all clouds"
+    base=$(echo $CONTAINER_IMAGE | cut -d "/" -f1)
+
+    if [[ "$base" =~  "mcr.microsoft."* ]]; then # path is required to be <base>/oss/kubernetes/kube-proxy:<version> for kube-proxy images
+      echo "$CONTAINER_IMAGE is already an mcr image, don't need to re-tag"
+      return
+    fi
+
+    clouds=(
+        "mcr.microsoft.com"           # public
+        "mcr.azk8s.cn"                # mooncake
+        "mcr.microsoft.eaglex.ic.gov" # usnat # have to confirm egressing usnat and ussec endpoints
+        "mcr.microsoft.scloud"        # ussec
+    )
+    for cloud in "${clouds[@]}"; do
+        newtag=${CONTAINER_IMAGE/$base/"$cloud"}
+        echo "retagging with tool: \"$CLI_TOOL\", current image: \"$CONTAINER_IMAGE\", new image: \"$newtag\""
+        retagContainerImage "$CLI_TOOL" "$CONTAINER_IMAGE" "$newtag"
+    done
+
+    removeContainerImage "$CLI_TOOL" "$CONTAINER_IMAGE"
+}
+
 cleanUpImages() {
     local targetImage=$1
+    if [[ "$targetImage" == "kube-proxy" ]]; then
+        echo "keeping all the kube-proxy images around to be able to use when needed w/o downloading again"
+        return
+    fi
+
     export targetImage
     function cleanupImagesRun() {
         if [ "${NEEDS_CONTAINERD}" == "true" ]; then
@@ -361,9 +424,7 @@ cleanUpImages() {
 }
 
 cleanUpKubeProxyImages() {
-    echo $(date),$(hostname), startCleanUpKubeProxyImages
-    cleanUpImages "kube-proxy"
-    echo $(date),$(hostname), endCleanUpKubeProxyImages
+    echo "keeping all the kube-proxy images around to be able to use when needed w/o downloading again"
 }
 
 cleanupRetaggedImages() {