[WIP] Default to Azure Linux images

Tiltfile

@@ -32,6 +32,7 @@ settings = {
     "worker_machine_count": "2",
     "az_node_machine_type": "Standard_B2s",
     "cluster_class_name": "default",
+    "ccm_ca_cert_dir": "/etc/pki/tls",
 }
 
 # Auth keys that need to be loaded from the environment
@@ -359,6 +360,7 @@ def deploy_worker_templates(template, substitutions):
         "AZURE_NODE_MACHINE_TYPE": settings.get("az_node_machine_type"),
         "FLATCAR_VERSION": settings.get("flatcar_version"),
         "CLUSTER_CLASS_NAME": settings.get("cluster_class_name"),
+        "CCM_CA_CERT_DIR": settings.get("ccm_ca_cert_dir"),
     }
 
     if "aks" in flavor:
@@ -400,9 +402,12 @@ def get_addons(flavor_name):
 
     addon_cmd = "; export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}')"
     addon_cmd += "; export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ',')"
+    addon_cmd += "; export CCM_CA_CERT_DIR=" + settings.get("ccm_ca_cert_dir")
     addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}"
     if "flatcar" in flavor_name:  # append caCetDir location to the cloud-provider-azure helm install command for flatcar flavor
         addon_cmd += " --set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates"
+    else:  # append caCetDir location to the cloud-provider-azure helm install command for Azure Linux flavor
+        addon_cmd += " --set-string cloudControllerManager.caCertDir=${CCM_CA_CERT_DIR}"
 
     if "azure-cni-v1" in flavor_name:
         addon_cmd += "; " + kubectl_cmd + " apply -f ./templates/addons/azure-cni-v1.yaml --kubeconfig ./${CLUSTER_NAME}.kubeconfig"

azure/scope/machine.go

@@ -723,7 +723,7 @@ func (m *MachineScope) GetVMImage(ctx context.Context) (*infrav1.Image, error) {
 	}
 
 	log.Info("No image specified for machine, using default Linux Image", "machine", m.AzureMachine.GetName())
-	return svc.GetDefaultUbuntuImage(ctx, m.Location(), ptr.Deref(m.Machine.Spec.Version, ""))
+	return svc.GetDefaultLinuxImage(ctx, m.Location(), ptr.Deref(m.Machine.Spec.Version, ""))
 }
 
 // SetSubnetName defaults the AzureMachine subnet name to the name of one the subnets with the machine role when there is only one of them.

azure/scope/machine_test.go

@@ -1740,7 +1740,7 @@ func TestMachineScope_GetVMImage(t *testing.T) {
 				ClusterScoper: clusterMock,
 			},
 			want: func() *infrav1.Image {
-				image, _ := svc.GetDefaultUbuntuImage(context.TODO(), "", "1.20.1")
+				image, _ := svc.GetDefaultLinuxImage(context.TODO(), "", "1.20.1")
 				return image
 			}(),
 			expectedErr: "",

azure/scope/machinepool.go

@@ -772,7 +772,7 @@ func (m *MachinePoolScope) GetVMImage(ctx context.Context) (*infrav1.Image, erro
 		log.V(4).Info("No image specified for machine, using default Windows Image", "machine", m.MachinePool.GetName(), "runtime", runtime, "windowsServerVersion", windowsServerVersion)
 		defaultImage, err = svc.GetDefaultWindowsImage(ctx, m.Location(), ptr.Deref(m.MachinePool.Spec.Template.Spec.Version, ""), runtime, windowsServerVersion)
 	} else {
-		defaultImage, err = svc.GetDefaultUbuntuImage(ctx, m.Location(), ptr.Deref(m.MachinePool.Spec.Template.Spec.Version, ""))
+		defaultImage, err = svc.GetDefaultLinuxImage(ctx, m.Location(), ptr.Deref(m.MachinePool.Spec.Template.Spec.Version, ""))
 	}
 
 	if err != nil {

azure/scope/machinepool_test.go

@@ -423,7 +423,7 @@ func TestMachinePoolScope_GetVMImage(t *testing.T) {
 						ImagePlan: infrav1.ImagePlan{
 							Publisher: "cncf-upstream",
 							Offer:     "capi",
-							SKU:       "k8s-1dot19dot11-ubuntu-1804",
+							SKU:       "k8s-1dot19dot11-azurelinux-3",
 						},
 						Version:         "latest",
 						ThirdPartyImage: false,

azure/services/virtualmachineimages/images.go

@@ -47,8 +47,59 @@
 	}, nil
 }
 
+// GetDefaultLinuxImage returns the default image spec for a Linux node.
+func (s *Service) GetDefaultLinuxImage(ctx context.Context, location, k8sVersion string) (*infrav1.Image, error) {
+	ctx, log, done := tele.StartSpanWithLogger(ctx, "virtualmachineimages.Service.GetDefaultLinuxImage")
+	defer done()
+
+	// First try Azure Linux, then Ubuntu.
+	defaultImage, err := s.GetDefaultAzureLinuxImage(ctx, location, k8sVersion)
+	if err != nil {
+		log.V(4).Info("Failed to get default Azure Linux image, trying default Ubuntu image", "error", err)
+		return s.GetDefaultUbuntuImage(ctx, location, k8sVersion)
+	}
+
+	return defaultImage, nil
+}
+
+// GetDefaultAzureLinuxImage returns the default image spec for Azure Linux.
+func (s *Service) GetDefaultAzureLinuxImage(ctx context.Context, location, k8sVersion string) (*infrav1.Image, error) {
+	ctx, log, done := tele.StartSpanWithLogger(ctx, "virtualmachineimages.Service.GetDefaultAzureLinuxImage")
+	defer done()
+
+	// First try Azure Linux 3.
+	publisher, offer := azure.DefaultImagePublisherID, azure.DefaultImageOfferID
+	skuID, version, err := s.getSKUAndVersion(
+		ctx, location, publisher, offer, k8sVersion, "azurelinux-3")
+	if err != nil {
+		// If that fails, log the error and try Azure Linux 2.
+		log.V(4).Info("Failed to get default image for Azure Linux 3, trying Azure Linux 2", "error", err)
+		skuID, version, err = s.getSKUAndVersion(
+			ctx, location, publisher, offer, k8sVersion, "mariner-2")
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get default image")
+		}
+	}
+
+	defaultImage := &infrav1.Image{
+		Marketplace: &infrav1.AzureMarketplaceImage{
+			ImagePlan: infrav1.ImagePlan{
+				Publisher: publisher,
+				Offer:     offer,
+				SKU:       skuID,
+			},
+			Version: version,
+		},
+	}
+
+	return defaultImage, nil
+}
+
 // GetDefaultUbuntuImage returns the default image spec for Ubuntu.
 func (s *Service) GetDefaultUbuntuImage(ctx context.Context, location, k8sVersion string) (*infrav1.Image, error) {
+	ctx, _, done := tele.StartSpanWithLogger(ctx, "virtualmachineimages.Service.GetDefaultAzureLinuxImage")
+	defer done()
+
 	v, err := semver.ParseTolerant(k8sVersion)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to parse Kubernetes version \"%s\"", k8sVersion)
@@ -121,7 +172,7 @@
 
 // getSKUAndVersion gets the SKU ID and version of the image to use for the provided version of Kubernetes.
 // note: osAndVersion is expected to be in the format of {os}-{version} (ex: ubuntu-2004 or windows-2022)
-func (s *Service) getSKUAndVersion(ctx context.Context, location, publisher, offer, k8sVersion, osAndVersion string) (skuID string, imageVersion string, err error) {
+func (s *Service) getSKUAndVersion(ctx context.Context, location, publisher, offer, k8sVersion, osAndVersion string) (skuID string, imageVersion string, err error) { //nolint:unparam // Keep "publisher" in the function signature.
 	ctx, log, done := tele.StartSpanWithLogger(ctx, "virtualmachineimages.Service.getSKUAndVersion")
 	defer done()
 

templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml

@@ -22,6 +22,7 @@ spec:
       logVerbosity: ${CCM_LOG_VERBOSITY:-4}
       replicas: ${CCM_COUNT:-1}
       enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
+      caCertDir: ${CCM_CA_CERT_DIR:-/etc/pki/tls}
     cloudNodeManager:
       imageName: "${CNM_IMAGE_NAME:-""}"
       imageRepository: "${IMAGE_REGISTRY:-""}"

templates/addons/cluster-api-helm/cloud-provider-azure.yaml

@@ -15,3 +15,4 @@ spec:
     cloudControllerManager:
       clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
       logVerbosity: 4
+      caCertDir: ${CCM_CA_CERT_DIR:-/etc/pki/tls}