feat: Azure Linux Image Family Support

hack/codegen.sh

@@ -32,7 +32,7 @@ skugen() {
   NO_UPDATE=" pkg/fake/zz_generated.sku.$location.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)"
   SUBJECT="SKUGEN"
 
-  go run hack/code/instancetype_testdata_gen.go -- "${GENERATED_FILE}" "$location" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_F16s_v2,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3,Standard_DC8s_v3"
+  go run hack/code/instancetype_testdata_gen.go -- "${GENERATED_FILE}" "$location" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_F16s_v2,Standard_NC6s,Standard_NC6s_v3,Standard_NC16as_T4_v3,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3,Standard_DC8s_v3"
   go fmt "${GENERATED_FILE}"
 
   GIT_DIFF=$(git diff --stat "${GENERATED_FILE}")

pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml

@@ -46,6 +46,7 @@ spec:
                 description: ImageFamily is the image family that instances use.
                 enum:
                 - Ubuntu2204
+                - AzureLinux
                 type: string
               imageVersion:
                 description: ImageVersion is the image version that instances use.

pkg/apis/v1alpha2/aksnodeclass.go

@@ -36,7 +36,7 @@ type AKSNodeClassSpec struct {
 	ImageID *string `json:"-"`
 	// ImageFamily is the image family that instances use.
 	// +kubebuilder:default=Ubuntu2204
-	// +kubebuilder:validation:Enum:={Ubuntu2204}
+	// +kubebuilder:validation:Enum:={Ubuntu2204,AzureLinux}
 	ImageFamily *string `json:"imageFamily,omitempty"`
 	// ImageVersion is the image version that instances use.
 	// +optional

pkg/apis/v1alpha2/labels.go

@@ -122,3 +122,8 @@ var (
 
 	NodeClaimLinkedAnnotationKey = v1alpha5.MachineLinkedAnnotationKey // still using the one from v1alpha5
 )
+
+const (
+	Ubuntu2204ImageFamily = "Ubuntu2204"
+	AzureLinuxImageFamily = "AzureLinux"
+)

pkg/fake/skus_test.go

@@ -0,0 +1,71 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func TestSKUExistence(t *testing.T) {
+	testSKUExistenceForRegion(t, "eastus", sets.New(
+		"Standard_A0",
+		"Standard_B1s",
+		"Standard_D2s_v3",
+		"Standard_D2_v2",
+		"Standard_D2_v3",
+		"Standard_D2_v5",
+		"Standard_D4s_v3",
+		"Standard_D64s_v3",
+		"Standard_DC8s_v3",
+		"Standard_DS2_v2",
+		"Standard_F16s_v2",
+		"Standard_M8-2ms",
+		"Standard_NC24ads_A100_v4",
+		"Standard_NC6s_v3",
+		"Standard_NC16as_T4_v3",
+	))
+	testSKUExistenceForRegion(t, "westcentralus", sets.New(
+		"Standard_A0",
+		"Standard_B1s",
+		"Standard_D2s_v3",
+		"Standard_D2_v2",
+		"Standard_D2_v3",
+		"Standard_D2_v5",
+		"Standard_D4s_v3",
+		"Standard_D64s_v3",
+		"Standard_DS2_v2",
+		"Standard_F16s_v2",
+	))
+}
+
+func testSKUExistenceForRegion(t *testing.T, region string, expectedSKUs sets.Set[string]) {
+	generatedSKUs := ResourceSkus[region]
+
+	skuSets := make(sets.Set[string])
+	for _, sku := range generatedSKUs {
+		skuName := *sku.Name
+		skuSets.Insert(skuName)
+	}
+
+	for _, expectedSKU := range expectedSKUs.UnsortedList() {
+		if exists := skuSets.Has(expectedSKU); !exists {
+			t.Errorf("SKU not found in %v: %v", region, expectedSKU)
+		}
+	}
+}

pkg/fake/zz_generated.sku.eastus.go

@@ -24,7 +24,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
 )
 
-// generated at 2023-12-12T02:02:44Z
+// generated at 2023-12-18T15:28:53Z
 
 func init() {
 	// ResourceSkus is a list of selected VM SKUs for a given region
@@ -39,6 +39,17 @@ func init() {
 			APIVersions:  &[]string{},
 			Costs:        &[]compute.ResourceSkuCosts{},
 			Restrictions: &[]compute.ResourceSkuRestrictions{
+				{
+					Type:   compute.ResourceSkuRestrictionsType("Location"),
+					Values: &[]string{"eastus"},
+					RestrictionInfo: &compute.ResourceSkuRestrictionInfo{
+						Locations: &[]string{
+							"eastus",
+						},
+						Zones: &[]string{},
+					},
+					ReasonCode: "NotAvailableForSubscription",
+				},
 				{
 					Type:   compute.ResourceSkuRestrictionsType("Zone"),
 					Values: &[]string{"eastus"},
@@ -601,6 +612,53 @@ func init() {
 			},
 			},
 		},
+		{
+			Name:         lo.ToPtr("Standard_NC16as_T4_v3"),
+			Tier:         lo.ToPtr("Standard"),
+			Kind:         lo.ToPtr(""),
+			Size:         lo.ToPtr("NC16as_T4_v3"),
+			Family:       lo.ToPtr("Standard NCASv3_T4 Family"),
+			ResourceType: lo.ToPtr("virtualMachines"),
+			APIVersions:  &[]string{},
+			Costs:        &[]compute.ResourceSkuCosts{},
+			Restrictions: &[]compute.ResourceSkuRestrictions{},
+			Capabilities: &[]compute.ResourceSkuCapabilities{
+				{Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("360448")},
+				{Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")},
+				{Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("16")},
+				{Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("False")},
+				{Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")},
+				{Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("110")},
+				{Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("32")},
+				{Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")},
+				{Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")},
+				{Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("16")},
+				{Name: lo.ToPtr("GPUs"), Value: lo.ToPtr("1")},
+				{Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")},
+				{Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("16320")},
+				{Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("251658240")},
+				{Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("251658240")},
+				{Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("154619000000")},
+				{Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("24480")},
+				{Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("368640000")},
+				{Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")},
+				{Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")},
+				{Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("8")},
+			},
+			Locations: &[]string{"eastus"},
+			LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{
+				"1",
+				"2",
+				"3",
+			},
+			},
+			},
+		},
 		{
 			Name:         lo.ToPtr("Standard_NC24ads_A100_v4"),
 			Tier:         lo.ToPtr("Standard"),
@@ -647,5 +705,62 @@ func init() {
 			},
 			},
 		},
+		{
+			Name:         lo.ToPtr("Standard_NC6s_v3"),
+			Tier:         lo.ToPtr("Standard"),
+			Kind:         lo.ToPtr(""),
+			Size:         lo.ToPtr("NC6s_v3"),
+			Family:       lo.ToPtr("standardNCSv3Family"),
+			ResourceType: lo.ToPtr("virtualMachines"),
+			APIVersions:  &[]string{},
+			Costs:        &[]compute.ResourceSkuCosts{},
+			Restrictions: &[]compute.ResourceSkuRestrictions{
+				{
+					Type:   compute.ResourceSkuRestrictionsType("Zone"),
+					Values: &[]string{"eastus"},
+					RestrictionInfo: &compute.ResourceSkuRestrictionInfo{
+						Locations: &[]string{
+							"eastus",
+						},
+						Zones: &[]string{
+							"1",
+							"2",
+							"3",
+						},
+					},
+					ReasonCode: "NotAvailableForSubscription",
+				},
+			},
+			Capabilities: &[]compute.ResourceSkuCapabilities{
+				{Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("344064")},
+				{Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")},
+				{Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("6")},
+				{Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("False")},
+				{Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")},
+				{Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("112")},
+				{Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("12")},
+				{Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")},
+				{Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")},
+				{Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("6")},
+				{Name: lo.ToPtr("GPUs"), Value: lo.ToPtr("1")},
+				{Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")},
+				{Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")},
+				{Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")},
+				{Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")},
+				{Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("4")},
+			},
+			Locations: &[]string{"eastus"},
+			LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{
+				"1",
+				"2",
+				"3",
+			},
+			},
+			},
+		},
 	}
 }

pkg/fake/zz_generated.sku.westcentralus.go

@@ -24,7 +24,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
 )
 
-// generated at 2023-12-12T02:03:06Z
+// generated at 2023-12-18T15:29:11Z
 
 func init() {
 	// ResourceSkus is a list of selected VM SKUs for a given region
@@ -38,7 +38,19 @@ func init() {
 			ResourceType: lo.ToPtr("virtualMachines"),
 			APIVersions:  &[]string{},
 			Costs:        &[]compute.ResourceSkuCosts{},
-			Restrictions: &[]compute.ResourceSkuRestrictions{},
+			Restrictions: &[]compute.ResourceSkuRestrictions{
+				{
+					Type:   compute.ResourceSkuRestrictionsType("Location"),
+					Values: &[]string{"westcentralus"},
+					RestrictionInfo: &compute.ResourceSkuRestrictionInfo{
+						Locations: &[]string{
+							"westcentralus",
+						},
+						Zones: &[]string{},
+					},
+					ReasonCode: "NotAvailableForSubscription",
+				},
+			},
 			Capabilities: &[]compute.ResourceSkuCapabilities{
 				{Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("20480")},
 				{Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")},

pkg/providers/imagefamily/azlinux.go

@@ -0,0 +1,103 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package imagefamily
+
+import (
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/Azure/karpenter/pkg/apis/v1alpha2"
+	"github.com/Azure/karpenter/pkg/providers/imagefamily/bootstrap"
+	"github.com/Azure/karpenter/pkg/providers/launchtemplate/parameters"
+
+	corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
+	"github.com/aws/karpenter-core/pkg/cloudprovider"
+	"github.com/aws/karpenter-core/pkg/scheduling"
+)
+
+const (
+	AzureLinuxGen2CommunityImage    = "V2gen2"
+	AzureLinuxGen1CommunityImage    = "V2"
+	AzureLinuxGen2ArmCommunityImage = "V2gen2arm64"
+)
+
+type AzureLinux struct {
+	Options *parameters.StaticParameters
+}
+
+func (u AzureLinux) Name() string {
+	return v1alpha2.AzureLinuxImageFamily
+}
+
+func (u AzureLinux) DefaultImages() []DefaultImageOutput {
+	// image provider will select these images in order, first match wins. This is why we chose to put AzureLinuxGen2containerd first in the defaultImages
+	return []DefaultImageOutput{
+		{
+			CommunityImage:   AzureLinuxGen2CommunityImage,
+			PublicGalleryURL: AKSAzureLinuxPublicGalleryURL,
+			Requirements: scheduling.NewRequirements(
+				scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
+				scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2),
+			),
+		},
+		{
+			CommunityImage:   AzureLinuxGen1CommunityImage,
+			PublicGalleryURL: AKSAzureLinuxPublicGalleryURL,
+			Requirements: scheduling.NewRequirements(
+				scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64),
+				scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1),
+			),
+		},
+		{
+			CommunityImage:   AzureLinuxGen2ArmCommunityImage,
+			PublicGalleryURL: AKSAzureLinuxPublicGalleryURL,
+			Requirements: scheduling.NewRequirements(
+				scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64),
+				scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2),
+			),
+		},
+	}
+}
+
+// UserData returns the default userdata script for the image Family
+func (u AzureLinux) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper {
+	return bootstrap.AKS{
+		Options: bootstrap.Options{
+			ClusterName:      u.Options.ClusterName,
+			ClusterEndpoint:  u.Options.ClusterEndpoint,
+			KubeletConfig:    kubeletConfig,
+			Taints:           taints,
+			Labels:           labels,
+			CABundle:         caBundle,
+			GPUNode:          u.Options.GPUNode,
+			GPUDriverVersion: u.Options.GPUDriverVersion,
+			// GPUImageSHA: u.Options.GPUImageSHA, image sha only applies to ubuntu
+			// SEE: https://github.com/Azure/AgentBaker/blob/f393d6e4d689d9204d6000c85623ad9b764e2a29/vhdbuilder/packer/install-dependencies.sh#L201
+		},
+		Arch:                           u.Options.Arch,
+		TenantID:                       u.Options.TenantID,
+		SubscriptionID:                 u.Options.SubscriptionID,
+		Location:                       u.Options.Location,
+		UserAssignedIdentityID:         u.Options.UserAssignedIdentityID,
+		ResourceGroup:                  u.Options.ResourceGroup,
+		ClusterID:                      u.Options.ClusterID,
+		APIServerName:                  u.Options.APIServerName,
+		KubeletClientTLSBootstrapToken: u.Options.KubeletClientTLSBootstrapToken,
+		NetworkPlugin:                  u.Options.NetworkPlugin,
+		NetworkPolicy:                  u.Options.NetworkPolicy,
+		KubernetesVersion:              u.Options.KubernetesVersion,
+	}
+}

pkg/providers/imagefamily/resolver.go

@@ -113,8 +113,10 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass,
 
 func getImageFamily(familyName *string, parameters *template.StaticParameters) ImageFamily {
 	switch lo.FromPtr(familyName) {
-	case Ubuntu2204ImageFamily:
+	case v1alpha2.Ubuntu2204ImageFamily:
 		return &Ubuntu2204{Options: parameters}
+	case v1alpha2.AzureLinuxImageFamily:
+		return &AzureLinux{Options: parameters}
 	default:
 		return &Ubuntu2204{Options: parameters}
 	}