diff --git a/Makefile-az.mk b/Makefile-az.mk index 9533caad5..704281fb7 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -119,8 +119,9 @@ az-configure-values: ## Generate cluster-related values for Karpenter Helm char hack/deploy/configure-values.sh $(AZURE_CLUSTER_NAME) $(AZURE_RESOURCE_GROUP) $(KARPENTER_SERVICE_ACCOUNT_NAME) $(AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME) az-configure-values-custom-vnet: ## Generate cluster-related values for Karpenter Helm chart (take custom subnet ID from first agentpool) - VNET_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId") \ - $(MAKE) az-configure-values + VNET_SUBNET_ID=$$(az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId"); \ + VNET_GUID=$$(bash -c 's=$$(az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId"); vnet_id=$${s%/subnets*}; az network vnet show --ids "$$vnet_id" --query "resourceGuid" -o tsv'); \ + $(MAKE) az-configure-values VNET_SUBNET_ID=$$VNET_SUBNET_ID VNET_GUID=$$VNET_GUID az-mkvmssflex: ## Create VMSS Flex (optional, only if creating VMs referencing this VMSS) az vmss create --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --location $(AZURE_LOCATION) \ diff --git a/karpenter-values-template.yaml b/karpenter-values-template.yaml index c5124877c..da0e9100c 100644 --- a/karpenter-values-template.yaml +++ b/karpenter-values-template.yaml @@ -25,6 +25,8 @@ controller: value: ${NETWORK_POLICY} - name: VNET_SUBNET_ID value: ${VNET_SUBNET_ID} + - name: VNET_GUID + value: ${VNET_GUID} - name: NODE_IDENTITIES value: ${NODE_IDENTITIES} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index c3adfa6b6..1e59aa2d7 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -46,6 +46,7 @@ import ( webhooksalt "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/webhooks" "github.com/Azure/karpenter-provider-azure/pkg/auth" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" + "github.com/Azure/karpenter-provider-azure/pkg/consts" "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" @@ -88,8 +89,11 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azClient, err := instance.CreateAZClient(ctx, azConfig) lo.Must0(err, "creating Azure client") - vnetGUID, err := getVnetGUID(azConfig, options.FromContext(ctx).SubnetID) - lo.Must0(err, "getting VNET GUID") + if options.FromContext(ctx).VnetGUID == "" && options.FromContext(ctx).NetworkPluginMode == consts.NetworkPluginModeOverlay { + vnetGUID, err := getVnetGUID(azConfig, options.FromContext(ctx).SubnetID) + lo.Must0(err, "getting VNET GUID") + options.FromContext(ctx).VnetGUID = vnetGUID + } unavailableOfferingsCache := azurecache.NewUnavailableOfferings() pricingProvider := pricing.NewProvider( @@ -124,7 +128,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azConfig.KubeletIdentityClientID, azConfig.NodeResourceGroup, azConfig.Location, - vnetGUID, + options.FromContext(ctx).VnetGUID, options.FromContext(ctx).ProvisionMode, ) instanceTypeProvider := instancetype.NewDefaultProvider( diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 60cc81cfd..3f1c5c53f 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -66,14 +66,16 @@ type Options struct { ClusterID string KubeletClientTLSBootstrapToken string // => TLSBootstrapToken in bootstrap (may need to be per node/nodepool) SSHPublicKey string // ssh.publicKeys.keyData => VM SSH public key // TODO: move to v1alpha2.AKSNodeClass? - NetworkPlugin string // => NetworkPlugin in bootstrap - NetworkPolicy string // => NetworkPolicy in bootstrap - NetworkPluginMode string // => Network Plugin Mode is used to control the mode the network plugin should operate in. For example, "overlay" used with --network-plugin=azure will use an overlay network (non-VNET IPs) for pods in the cluster. Learn more about overlay networking here: https://learn.microsoft.com/en-us/azure/aks/azure-cni-overlay?tabs=kubectl#overview-of-overlay-networking - NetworkDataplane string - NodeIdentities []string // => Applied onto each VM - SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass - setFlags map[string]bool + NetworkPlugin string // => NetworkPlugin in bootstrap + NetworkPolicy string // => NetworkPolicy in bootstrap + NetworkPluginMode string // => Network Plugin Mode is used to control the mode the network plugin should operate in. For example, "overlay" used with --network-plugin=azure will use an overlay network (non-VNET IPs) for pods in the cluster. Learn more about overlay networking here: https://learn.microsoft.com/en-us/azure/aks/azure-cni-overlay?tabs=kubectl#overview-of-overlay-networking + NetworkDataplane string + + NodeIdentities []string // => Applied onto each VM + VnetGUID string // resource guid used by azure cni for identifying the right vnet + SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass + setFlags map[string]bool ProvisionMode string NodeBootstrappingServerURL string @@ -93,6 +95,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.NetworkPluginMode, "network-plugin-mode", env.WithDefaultString("NETWORK_PLUGIN_MODE", consts.NetworkPluginModeOverlay), "network plugin mode of the cluster.") fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.") fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.") + fs.StringVar(&o.VnetGUID, "vnet-guid", env.WithDefaultString("VNET_GUID", ""), "The vnet guid of the clusters vnet, only required by azure cni with overlay") fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR.") fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.") fs.StringVar(&o.ProvisionMode, "provision-mode", env.WithDefaultString("PROVISION_MODE", consts.ProvisionModeAKSScriptless), "[UNSUPPORTED] The provision mode for the cluster.") diff --git a/pkg/test/environment.go b/pkg/test/environment.go index bcf8ab48f..4cd5426f4 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -125,7 +125,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi "test-kubelet-identity-client-id", testOptions.NodeResourceGroup, region, - "test-vnet-guid", + testOptions.VnetGUID, testOptions.ProvisionMode, ) loadBalancerProvider := loadbalancer.NewProvider( diff --git a/pkg/test/options.go b/pkg/test/options.go index 4cdbbb3a4..1cac25a41 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -41,6 +41,7 @@ type OptionsFields struct { NodeResourceGroup *string ProvisionMode *string NodeBootstrappingServerURL *string + VnetGUID *string // UseSIG Flags not required by the self hosted offering UseSIG *bool @@ -63,6 +64,7 @@ func Options(overrides ...OptionsFields) *azoptions.Options { NetworkPlugin: lo.FromPtrOr(options.NetworkPlugin, "azure"), NetworkPluginMode: lo.FromPtrOr(options.NetworkPluginMode, "overlay"), NetworkPolicy: lo.FromPtrOr(options.NetworkPolicy, "cilium"), + VnetGUID: lo.FromPtrOr(options.VnetGUID, "test-vnet-guid"), NetworkDataplane: lo.FromPtrOr(options.NetworkDataplane, "cilium"), VMMemoryOverheadPercent: lo.FromPtrOr(options.VMMemoryOverheadPercent, 0.075), NodeIdentities: options.NodeIdentities,