feat: support pulling images from sov cloud's MCR for network isolate…

…d cluster feature (#5663)
Azure · Feb 14, 2025 · 5dbf888 · 5dbf888
1 parent 6a486fc
commit 5dbf888
Show file tree

Hide file tree

Showing 274 changed files with 411 additions and 273 deletions.
diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh
@@ -161,6 +161,7 @@ ARTIFACT_STREAMING_ENABLED="{{IsArtifactStreamingEnabled}}"
 SYSCTL_CONTENT="{{GetSysctlContent}}"
 PRIVATE_EGRESS_PROXY_ADDRESS="{{GetPrivateEgressProxyAddress}}"
 BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{GetBootstrapProfileContainerRegistryServer}}"
+MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
 ENABLE_IMDS_RESTRICTION="{{EnableIMDSRestriction}}"
 INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE="{{InsertIMDSRestrictionRuleToMangleTable}}"
 /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh"
diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh
@@ -341,8 +341,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  # TODO(binxi): need to update for sovereign cloud.
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go
@@ -1005,6 +1005,9 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
 		"GetBootstrapProfileContainerRegistryServer": func() string {
 			return config.ContainerService.Properties.SecurityProfile.GetPrivateEgressContainerRegistryServer()
 		},
+		"GetMCRRepositoryBase": func() string {
+			return config.CloudSpecConfig.KubernetesSpecConfig.MCRKubernetesImageBase
+		},
 		"IsArtifactStreamingEnabled": func() bool {
 			return config.EnableArtifactStreaming
 		},

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+Docker/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Docker/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/CustomData b/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S115/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+K8S115/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S115/CustomData b/pkg/agent/testdata/AKSUbuntu1604+K8S115/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S115/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+K8S115/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S117/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+K8S117/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S117/CustomData b/pkg/agent/testdata/AKSUbuntu1604+K8S117/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S117/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+K8S117/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S118/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+K8S118/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S118/CustomData b/pkg/agent/testdata/AKSUbuntu1604+K8S118/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+K8S118/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+K8S118/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+KubeletConfigFile/CSECommand b/pkg/agent/testdata/AKSUbuntu1604+KubeletConfigFile/CSECommand
diff --git a/pkg/agent/testdata/AKSUbuntu1604+KubeletConfigFile/CustomData b/pkg/agent/testdata/AKSUbuntu1604+KubeletConfigFile/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+KubeletConfigFile/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+KubeletConfigFile/line70.sh
@@ -331,7 +331,8 @@ EOF
 }
 
 configureContainerdRegistryHost() {
-  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  MCR_REPOSITORY_BASE="${MCR_REPOSITORY_BASE:=mcr.microsoft.com}"
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/${MCR_REPOSITORY_BASE}/hosts.toml"
   mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
   touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
   chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"