feat: support custom subnet ID (--vnet-subnet-id)

Makefile-az.mk

@@ -16,6 +16,9 @@ AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME ?= karpentermsi
 KARPENTER_FEDERATED_IDENTITY_CREDENTIAL_NAME ?= KARPENTER_FID
 
 az-all:         az-login az-create-workload-msi az-mkaks-cilium az-create-federated-cred az-perm az-perm-acr az-patch-skaffold-azureoverlay az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
+
+az-all-custom-vnet: az-login az-create-workload-msi az-mkaks-custom-vnet az-create-federated-cred az-perm az-perm-subnet az-perm-acr az-patch-skaffold-azureoverlay az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
+
 az-all-savm:    az-login az-mkaks-savm az-perm-savm az-patch-skaffold-azure az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload - StandaloneVirtualMachines
 
 az-login: ## Login into Azure
@@ -44,6 +47,20 @@ az-mkaks-cilium: az-mkacr ## Create test AKS cluster (with --network-dataplane c
 	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
 	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
 
+az-mkvnet:
+	az group create --name $(AZURE_RESOURCE_GROUP)-vnet --location $(AZURE_LOCATION) 
+	az network vnet create --name $(AZURE_CLUSTER_NAME)-vnet --resource-group $(AZURE_RESOURCE_GROUP)-vnet --location $(AZURE_LOCATION) --address-prefixes "10.1.0.0/16"
+
+az-mksubnet: 
+	az network vnet subnet create --name $(AZURE_CLUSTER_NAME)-subnet --resource-group $(AZURE_RESOURCE_GROUP)-vnet --vnet-name $(AZURE_CLUSTER_NAME)-vnet --address-prefixes "10.1.0.0/24"
+
+az-mkaks-custom-vnet: az-mkacr ## Create test AKS cluster with custom VNET
+	az aks create --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \
+		--enable-managed-identity --node-count 3 --generate-ssh-keys -o none --network-dataplane cilium --network-plugin azure --network-plugin-mode overlay \
+		--enable-oidc-issuer --enable-workload-identity --vnet-subnet-id "/subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP)-vnet/providers/Microsoft.Network/virtualNetworks/$(AZURE_CLUSTER_NAME)-vnet/subnets/$(AZURE_CLUSTER_NAME)-subnet"
+	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
+	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
+
 az-create-workload-msi: az-mkrg
 	# create the workload MSI that is the backing for the karpenter pod auth
 	az identity create --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --resource-group "${AZURE_RESOURCE_GROUP}" --location "${AZURE_LOCATION}"
@@ -82,20 +99,20 @@ az-patch-skaffold: 	## Update Azure client env vars and settings in skaffold con
 	yq -i  '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="KUBELET_BOOTSTRAP_TOKEN")).value =                             "$(BOOTSTRAP_TOKEN)"'         skaffold.yaml
 	yq -i  '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="SSH_PUBLIC_KEY")).value =                                               "$(SSH_PUBLIC_KEY)"'          skaffold.yaml
 
-az-patch-skaffold-kubenet: az-patch-skaffold	az-fetch-network-info
-	$(eval AZURE_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq  -r ".[0].subnets[0].id"))
+az-patch-skaffold-kubenet: az-patch-skaffold
+	$(eval AZURE_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId"))
 	yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID"))               .value = "$(AZURE_SUBNET_ID)"'         skaffold.yaml
 	yq -i  '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="NETWORK_PLUGIN").value) =                                              "kubenet"'                    skaffold.yaml
 
-az-patch-skaffold-azure: az-patch-skaffold	az-fetch-network-info
+az-patch-skaffold-azure: az-patch-skaffold
 	$(eval AZURE_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId"))
 	yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID"))               .value = "$(AZURE_SUBNET_ID)"'         skaffold.yaml
 
-az-patch-skaffold-azureoverlay: az-patch-skaffold	az-fetch-network-info
-	$(eval AZURE_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq  -r ".[0].subnets[0].id"))
+az-patch-skaffold-azureoverlay: az-patch-skaffold	
+	$(eval AZURE_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId"))
 	yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml
 	yq -i  '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="NETWORK_PLUGIN").value) =                                              "azure"'                      skaffold.yaml
-
+        
 	# old identity path is still the default, so need to override the values values with new logic.
 	# TODO (chmcbrid): update the new logic path as the default.
 	$(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'clientId' -otsv))
@@ -108,12 +125,6 @@ az-patch-skaffold-azureoverlay: az-patch-skaffold	az-fetch-network-info
 
 	yq -i  '.manifests.helm.releases[0].overrides.podLabels ."azure.workload.identity/use" = "true"' skaffold.yaml
 
-az-fetch-network-info:
-	$(eval AZURE_VNET_NAME=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) |  jq -r ".[0].name"))
-	yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_VNET_NAME"))	.value = "$(AZURE_VNET_NAME)"'         skaffold.yaml
-	$(eval AZURE_SUBNET_NAME=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) |  jq -r ".[0].subnets[0].name"))
-	yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_NAME"))	.value = "$(AZURE_SUBNET_NAME)"'         skaffold.yaml
-
 az-mkvmssflex: ## Create VMSS Flex (optional, only if creating VMs referencing this VMSS)
 	az vmss create --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --location $(AZURE_LOCATION) \
 		--instance-count 0 --orchestration-mode Flexible --platform-fault-domain-count 1 --zones 1 2 3
@@ -128,8 +139,16 @@ az-perm: ## Create role assignments to let Karpenter manage VMs and Network
 	az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Network Contributor"
 	az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Managed Identity Operator"
 	az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP)    --role "Network Contributor" # in some case we create vnet here
+
 	@echo Consider "make az-patch-skaffold"!
 
+az-perm-subnet: 
+	# give Network Contributor permission to the subnet rg for the AKS cluster 
+	$(eval AZURE_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId")) 
+	$(eval SUBNET_RESOURCE_GROUP=$(shell az network vnet subnet show --id $(AZURE_SUBNET_ID) | jq -r ".resourceGroup")) 
+	$(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) 
+	az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(SUBNET_RESOURCE_GROUP) --role "Network Contributor" 
+
 az-perm-savm: ## Create role assignments to let Karpenter manage VMs and Network
 	# Note: savm has not been converted over to use a workload identity
 	$(eval AZURE_OBJECT_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq  -r ".identityProfile.kubeletidentity.objectId"))
@@ -235,6 +254,9 @@ az-rmnodeclaims: ## kubectl delete all nodeclaims; don't wait for finalizers (us
 az-taintsystemnodes: ## Taint all system nodepool nodes
 	kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --selector='kubernetes.azure.com/mode=system' --overwrite
 
+az-taintnodes:
+	kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite
+
 az-e2etests: ## Run e2etests
 	kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite
 	TEST_SUITE=Utilization make e2etests

pkg/auth/config.go

@@ -80,10 +80,6 @@ type Config struct {
 	ClusterName string `json:"clusterName" yaml:"clusterName"`
 	//Config only for AKS
 	NodeResourceGroup string `json:"nodeResourceGroup" yaml:"nodeResourceGroup"`
-	//SubnetId is the resource ID of the subnet that VM network interfaces should use
-	SubnetID   string `json:"subnetId" yaml:"subnetId"`
-	VnetName   string `json:"vnetName" yaml:"vnetName"`
-	SubnetName string `json:"subnetName" yaml:"subnetName"`
 }
 
 func (cfg *Config) PrepareConfig() error {
@@ -108,10 +104,6 @@ func (cfg *Config) BaseVars() {
 	cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD")
 	cfg.ClusterName = os.Getenv("AZURE_CLUSTER_NAME")
 	cfg.NodeResourceGroup = os.Getenv("AZURE_NODE_RESOURCE_GROUP")
-	cfg.SubnetID = os.Getenv("AZURE_SUBNET_ID")
-	cfg.SubnetName = os.Getenv("AZURE_SUBNET_NAME")
-	cfg.VnetName = os.Getenv("AZURE_VNET_NAME")
-	// cfg.VnetGuid = os.Getenv("AZURE_VNET_GUID") // This field needs to be resolved inside of karpenter, so we will get it in the azClient initialization
 }
 
 func (cfg *Config) prepareID() error {
@@ -186,9 +178,6 @@ func (cfg *Config) TrimSpace() {
 	cfg.AADClientCertPassword = strings.TrimSpace(cfg.AADClientCertPassword)
 	cfg.ClusterName = strings.TrimSpace(cfg.ClusterName)
 	cfg.NodeResourceGroup = strings.TrimSpace(cfg.NodeResourceGroup)
-	cfg.SubnetID = strings.TrimSpace(cfg.SubnetID)
-	cfg.SubnetName = strings.TrimSpace(cfg.SubnetName)
-	cfg.VnetName = strings.TrimSpace(cfg.VnetName)
 }
 
 func (cfg *Config) validate() error {
@@ -199,9 +188,6 @@ func (cfg *Config) validate() error {
 		{cfg.VMType, "VM type"},
 		// Even though the config doesnt use some of these,
 		// its good to validate they were set in the environment
-		{cfg.SubnetID, "subnet ID"},
-		{cfg.SubnetName, "subnet name"},
-		{cfg.VnetName, "vnet name"},
 	}
 
 	for _, field := range fields {

pkg/auth/config_test.go

@@ -40,9 +40,6 @@ func TestBuildAzureConfig(t *testing.T) {
 				SubscriptionID:    "12345",
 				ResourceGroup:     "my-rg",
 				NodeResourceGroup: "my-node-rg",
-				SubnetID:          "12345",
-				SubnetName:        "my-subnet",
-				VnetName:          "my-vnet",
 				VMType:            "vmss",
 			},
 			wantErr: false,
@@ -61,9 +58,6 @@ func TestBuildAzureConfig(t *testing.T) {
 				SubscriptionID:    "12345",
 				ResourceGroup:     "my-rg",
 				NodeResourceGroup: "my-node-rg",
-				SubnetID:          "12345",
-				SubnetName:        "my-subnet",
-				VnetName:          "my-vnet",
 				VMType:            "vm",
 			},
 			wantErr: false,
@@ -97,9 +91,6 @@ func TestBuildAzureConfig(t *testing.T) {
 				SubscriptionID:              "12345",
 				ResourceGroup:               "my-rg",
 				NodeResourceGroup:           "my-node-rg",
-				SubnetID:                    "12345",
-				SubnetName:                  "my-subnet",
-				VnetName:                    "my-vnet",
 				VMType:                      "vmss",
 				UseManagedIdentityExtension: true,
 				UserAssignedIdentityID:      "12345",

pkg/operator/operator.go

@@ -30,6 +30,8 @@ import (
 	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/operator/scheme"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
+
 	"github.com/Azure/karpenter-provider-azure/pkg/apis"
 	"github.com/Azure/karpenter-provider-azure/pkg/auth"
 	azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache"
@@ -40,6 +42,8 @@ import (
 	"github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate"
 	"github.com/Azure/karpenter-provider-azure/pkg/providers/loadbalancer"
 	"github.com/Azure/karpenter-provider-azure/pkg/providers/pricing"
+	"github.com/Azure/karpenter-provider-azure/pkg/utils"
+	armopts "github.com/Azure/karpenter-provider-azure/pkg/utils/opts"
 	"sigs.k8s.io/karpenter/pkg/operator"
 )
 
@@ -64,11 +68,14 @@ type Operator struct {
 
 func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) {
 	azConfig, err := GetAZConfig()
-	lo.Must0(err, "creating Azure config") // TODO: I assume we prefer this over the cleaner azConfig := lo.Must(GetAzConfig()), as this has a helpful error message?
+	lo.Must0(err, "creating Azure config") // NOTE: we prefer this over the cleaner azConfig := lo.Must(GetAzConfig()), as when initializing the client there are helpful error messages in initializing clients and the azure config
 
 	azClient, err := instance.CreateAZClient(ctx, azConfig)
 	lo.Must0(err, "creating Azure client")
 
+	vnetGUID, err := getVNETGUID(azConfig, options.FromContext(ctx).SubnetID)
+	lo.Must0(err, "getting VNET GUID")
+
 	unavailableOfferingsCache := azurecache.NewUnavailableOfferings()
 	pricingProvider := pricing.NewProvider(
 		ctx,
@@ -83,7 +90,12 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		azClient.ImageVersionsClient,
 		azConfig.Location,
 	)
-	imageResolver := imagefamily.New(operator.GetClient(), imageProvider)
+	imageResolver := imagefamily.New(
+		operator.GetClient(),
+		imageProvider,
+		options.FromContext(ctx).SubnetID,
+		vnetGUID,
+	)
 	launchTemplateProvider := launchtemplate.NewProvider(
 		ctx,
 		imageResolver,
@@ -116,7 +128,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		unavailableOfferingsCache,
 		azConfig.Location,
 		azConfig.NodeResourceGroup,
-		azConfig.SubnetID,
+		options.FromContext(ctx).SubnetID,
 		azConfig.SubscriptionID,
 	)
 
@@ -156,3 +168,28 @@ func getCABundle(restConfig *rest.Config) (*string, error) {
 	}
 	return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil
 }
+
+func getVNETGUID(cfg *auth.Config, subnetID string) (string, error) {
+	creds, err := auth.NewCredential(cfg)
+	if err != nil {
+		return "", err
+	}
+	opts := armopts.DefaultArmOpts()
+	vnetClient, err := armnetwork.NewVirtualNetworksClient(cfg.SubscriptionID, creds, opts)
+	if err != nil {
+		return "", err
+	}
+
+	subnetParts, err := utils.GetVnetSubnetIDComponents(subnetID)
+	if err != nil {
+		return "", err
+	}
+	vnet, err := vnetClient.Get(context.Background(), subnetParts.ResourceGroupName, subnetParts.VNetName, nil)
+	if err != nil {
+		return "", err
+	}
+	if vnet.Properties == nil || vnet.Properties.ResourceGUID == nil {
+		return "", fmt.Errorf("vnet %s does not have a resource GUID", subnetParts.VNetName)
+	}
+	return *vnet.Properties.ResourceGUID, nil
+}

pkg/operator/options/options.go

@@ -63,11 +63,13 @@ type Options struct {
 	VMMemoryOverheadPercent        float64
 	ClusterID                      string
 	KubeletClientTLSBootstrapToken string   // => TLSBootstrapToken in bootstrap (may need to be per node/nodepool)
-	SSHPublicKey                   string   // ssh.publicKeys.keyData => VM SSH public key // TODO: move to node template?
+	SSHPublicKey                   string   // ssh.publicKeys.keyData => VM SSH public key // TODO: move to v1alpha2.AKSNodeClass?
 	NetworkPlugin                  string   // => NetworkPlugin in bootstrap
 	NetworkPolicy                  string   // => NetworkPolicy in bootstrap
 	NodeIdentities                 []string // => Applied onto each VM
 
+	SubnetID string // => VnetSubnetID set on the systempool that we will fall back to if not specified in the nodeclass
+
 	setFlags map[string]bool
 }
 
@@ -79,6 +81,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.")
 	fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", "azure"), "The network plugin used by the cluster.")
 	fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.")
+	fs.StringVar(&o.SubnetID, "default-subnet-id", env.WithDefaultString("AZURE_SUBNET_ID", ""), "The default subnet ID to use for new nodes.")
 	fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.")
 }
 

pkg/providers/imagefamily/bootstrap/aksbootstrap.go

@@ -21,7 +21,6 @@ import (
 	_ "embed"
 	"encoding/base64"
 	"fmt"
-	"os"
 	"strings"
 	"text/template"
 
@@ -389,17 +388,9 @@ var (
 	}
 )
 
-// Node Labels for Vnet
 const (
-	vnetDataPlaneLabel      = "kubernetes.azure.com/ebpf-dataplane"
-	vnetNetworkNameLabel    = "kubernetes.azure.com/network-name"
-	vnetSubnetNameLabel     = "kubernetes.azure.com/network-subnet"
-	vnetSubscriptionIDLabel = "kubernetes.azure.com/network-subscription"
-	vnetGUIDLabel           = "kubernetes.azure.com/nodenetwork-vnetguid"
-	vnetPodNetworkTypeLabel = "kubernetes.azure.com/podnetwork-type"
-	ciliumDataPlane         = "cilium"
-	overlayNetworkType      = "overlay"
-	globalAKSMirror         = "https://acs-mirror.azureedge.net"
+	vnetSubnetNameLabel = "kubernetes.azure.com/network-subnet"
+	globalAKSMirror     = "https://acs-mirror.azureedge.net"
 )
 
 func (a AKS) aksBootstrapScript() (string, error) {
@@ -464,21 +455,8 @@ func (a AKS) applyOptions(nbv *NodeBootstrapVariables) {
 	kubeletLabels := lo.Assign(kubeletNodeLabelsBase, a.Labels)
 	getAgentbakerGeneratedLabels(a.ResourceGroup, kubeletLabels)
 
-	//Adding vnet-related labels to the nodeLabels.
-	azureVnetGUID := os.Getenv("AZURE_VNET_GUID")
-	azureVnetName := os.Getenv("AZURE_VNET_NAME")
-	azureSubnetName := os.Getenv("AZURE_SUBNET_NAME")
-
-	vnetLabels := map[string]string{
-		vnetDataPlaneLabel:      ciliumDataPlane,
-		vnetNetworkNameLabel:    azureVnetName,
-		vnetSubnetNameLabel:     azureSubnetName,
-		vnetSubscriptionIDLabel: a.SubscriptionID,
-		vnetGUIDLabel:           azureVnetGUID,
-		vnetPodNetworkTypeLabel: overlayNetworkType,
-	}
+	nbv.Subnet = a.Labels[vnetSubnetNameLabel]
 
-	kubeletLabels = lo.Assign(kubeletLabels, vnetLabels)
 	nbv.KubeletNodeLabels = strings.Join(lo.MapToSlice(kubeletLabels, func(k, v string) string {
 		return fmt.Sprintf("%s=%s", k, v)
 	}), ",")