diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index 12a14cbab..91b49e0a1 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -45,7 +45,7 @@ runs: run: az account set --subscription ${{ inputs.subscription-id }} - name: patch skaffold and cilium shell: bash - run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_LOCATION=${{ inputs.location }} make az-patch-skaffold-azureoverlay + run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_LOCATION=${{ inputs.location }} make az-patch-skaffold-azureoverlay az-patch-vnet-subnet-id - name: deploy karpenter to cluster shell: bash run: AZURE_ACR_NAME=${{ inputs.acr_name }} make az-run diff --git a/Makefile-az.mk b/Makefile-az.mk index a4a7173e8..833160660 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -15,7 +15,14 @@ KARPENTER_SERVICE_ACCOUNT_NAME ?= karpenter-sa AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME ?= karpentermsi KARPENTER_FEDERATED_IDENTITY_CREDENTIAL_NAME ?= KARPENTER_FID -az-all: az-login az-create-workload-msi az-mkaks-cilium az-create-federated-cred az-perm az-perm-acr az-patch-skaffold-azureoverlay az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload +CUSTOM_VNET_NAME ?= $(AZURE_CLUSTER_NAME)-vnet +CUSTOM_SUBNET_NAME ?= nodesubnet + + +az-all: az-login az-create-workload-msi az-mkaks-cilium az-create-federated-cred az-perm az-perm-acr az-patch-skaffold-azureoverlay az-patch-vnet-subnet-id az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload + +az-all-custom-vnet: az-login az-create-workload-msi az-mkaks-custom-vnet az-create-federated-cred az-perm az-perm-acr az-patch-skaffold-azureoverlay az-patch-subnet-custom az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload + az-all-savm: az-login az-mkaks-savm az-perm-savm az-patch-skaffold-azure az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload - StandaloneVirtualMachines az-login: ## Login into Azure @@ -44,6 +51,21 @@ az-mkaks-cilium: az-mkacr ## Create test AKS cluster (with --network-dataplane c az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter +az-mkvnet: ## Create a VNet with address range of 10.1.0.0/16 + az group create --name $(CUSTOM_VNET_NAME) --location $(AZURE_LOCATION) + az network vnet create --name $(CUSTOM_VNET_NAME) --resource-group $(AZURE_RESOURCE_GROUP)-vnet --location $(AZURE_LOCATION) --address-prefixes "10.1.0.0/16" + +az-mksubnet: ## Create a subnet with address range of 10.1.0.0/24 + az network vnet subnet create --name $(CUSTOM_SUBNET_NAME) --resource-group $(CUSTOM_VNET_NAME) --vnet-name $(CUSTOM_VNET_NAME) --address-prefixes "10.1.0.0/24" + +az-mkaks-custom-vnet: az-mkacr ## Create test AKS cluster with custom VNet + az aks create --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \ + --enable-managed-identity --node-count 3 --generate-ssh-keys -o none --network-dataplane cilium --network-plugin azure --network-plugin-mode overlay \ + --enable-oidc-issuer --enable-workload-identity \ + --vnet-subnet-id "/subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(CUSTOM_VNET_NAME)/providers/Microsoft.Network/virtualNetworks/$(CUSTOM_VNET_NAME)/subnets/$(CUSTOM_SUBNET_NAME)" + az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing + skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter + az-create-workload-msi: az-mkrg # create the workload MSI that is the backing for the karpenter pod auth az identity create --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --resource-group "${AZURE_RESOURCE_GROUP}" --location "${AZURE_LOCATION}" @@ -82,20 +104,31 @@ az-patch-skaffold: ## Update Azure client env vars and settings in skaffold con yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="KUBELET_BOOTSTRAP_TOKEN")).value = "$(BOOTSTRAP_TOKEN)"' skaffold.yaml yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="SSH_PUBLIC_KEY")).value = "$(SSH_PUBLIC_KEY)"' skaffold.yaml -az-patch-skaffold-kubenet: az-patch-skaffold az-fetch-network-info - $(eval AZURE_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].id")) - yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml - yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="NETWORK_PLUGIN").value) = "kubenet"' skaffold.yaml -az-patch-skaffold-azure: az-patch-skaffold az-fetch-network-info - $(eval AZURE_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId")) - yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml +az-patch-subnet-custom: + $(eval VNET_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId")) + $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) + $(eval SUBNET_RESOURCE_GROUP=$(shell az network vnet subnet show --id $(VNET_SUBNET_ID) | jq -r ".resourceGroup")) + az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(SUBNET_RESOURCE_GROUP) --role "Network Contributor" + yq e -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="VNET_SUBNET_ID")).value = "$(VNET_SUBNET_ID)"' skaffold.yaml -az-patch-skaffold-azureoverlay: az-patch-skaffold az-fetch-network-info - $(eval AZURE_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].id")) - yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml +az-patch-vnet-subnet-id: ## Patch VNET_SUBNET_ID in skaffold.yaml + $(eval VNET_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].id")) + $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) + $(eval SUBNET_RESOURCE_GROUP=$(shell az network vnet subnet show --id $(VNET_SUBNET_ID) | jq -r ".resourceGroup")) + az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(SUBNET_RESOURCE_GROUP) --role "Network Contributor" + yq e -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="VNET_SUBNET_ID")).value = "$(VNET_SUBNET_ID)"' skaffold.yaml + + +az-patch-skaffold-kubenet: az-patch-vnet-subnet-id az-patch-skaffold + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="NETWORK_PLUGIN").value) = "kubenet"' skaffold.yaml + +az-patch-skaffold-azure: az-patch-vnet-subnet-id az-patch-skaffold yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="NETWORK_PLUGIN").value) = "azure"' skaffold.yaml +az-patch-skaffold-azureoverlay: az-patch-skaffold + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="NETWORK_PLUGIN").value) = "azure"' skaffold.yaml + # old identity path is still the default, so need to override the values values with new logic. # TODO (chmcbrid): update the new logic path as the default. $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'clientId' -otsv)) @@ -108,12 +141,6 @@ az-patch-skaffold-azureoverlay: az-patch-skaffold az-fetch-network-info yq -i '.manifests.helm.releases[0].overrides.podLabels ."azure.workload.identity/use" = "true"' skaffold.yaml -az-fetch-network-info: - $(eval AZURE_VNET_NAME=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].name")) - yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_VNET_NAME")) .value = "$(AZURE_VNET_NAME)"' skaffold.yaml - $(eval AZURE_SUBNET_NAME=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].name")) - yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_NAME")) .value = "$(AZURE_SUBNET_NAME)"' skaffold.yaml - az-mkvmssflex: ## Create VMSS Flex (optional, only if creating VMs referencing this VMSS) az vmss create --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --location $(AZURE_LOCATION) \ --instance-count 0 --orchestration-mode Flexible --platform-fault-domain-count 1 --zones 1 2 3 @@ -128,8 +155,10 @@ az-perm: ## Create role assignments to let Karpenter manage VMs and Network az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Network Contributor" az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Managed Identity Operator" az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) --role "Network Contributor" # in some case we create vnet here + @echo Consider "make az-patch-skaffold"! + az-perm-savm: ## Create role assignments to let Karpenter manage VMs and Network # Note: savm has not been converted over to use a workload identity $(eval AZURE_OBJECT_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".identityProfile.kubeletidentity.objectId")) @@ -235,6 +264,9 @@ az-rmnodeclaims: ## kubectl delete all nodeclaims; don't wait for finalizers (us az-taintsystemnodes: ## Taint all system nodepool nodes kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --selector='kubernetes.azure.com/mode=system' --overwrite +az-taintnodes: + kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite + az-e2etests: ## Run e2etests kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite TEST_SUITE=Utilization make e2etests diff --git a/pkg/auth/config.go b/pkg/auth/config.go index b2db025a3..83c927663 100644 --- a/pkg/auth/config.go +++ b/pkg/auth/config.go @@ -80,10 +80,6 @@ type Config struct { ClusterName string `json:"clusterName" yaml:"clusterName"` //Config only for AKS NodeResourceGroup string `json:"nodeResourceGroup" yaml:"nodeResourceGroup"` - //SubnetId is the resource ID of the subnet that VM network interfaces should use - SubnetID string `json:"subnetId" yaml:"subnetId"` - VnetName string `json:"vnetName" yaml:"vnetName"` - SubnetName string `json:"subnetName" yaml:"subnetName"` } func (cfg *Config) PrepareConfig() error { @@ -108,10 +104,6 @@ func (cfg *Config) BaseVars() { cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD") cfg.ClusterName = os.Getenv("AZURE_CLUSTER_NAME") cfg.NodeResourceGroup = os.Getenv("AZURE_NODE_RESOURCE_GROUP") - cfg.SubnetID = os.Getenv("AZURE_SUBNET_ID") - cfg.SubnetName = os.Getenv("AZURE_SUBNET_NAME") - cfg.VnetName = os.Getenv("AZURE_VNET_NAME") - // cfg.VnetGuid = os.Getenv("AZURE_VNET_GUID") // This field needs to be resolved inside of karpenter, so we will get it in the azClient initialization } func (cfg *Config) prepareID() error { @@ -186,9 +178,6 @@ func (cfg *Config) TrimSpace() { cfg.AADClientCertPassword = strings.TrimSpace(cfg.AADClientCertPassword) cfg.ClusterName = strings.TrimSpace(cfg.ClusterName) cfg.NodeResourceGroup = strings.TrimSpace(cfg.NodeResourceGroup) - cfg.SubnetID = strings.TrimSpace(cfg.SubnetID) - cfg.SubnetName = strings.TrimSpace(cfg.SubnetName) - cfg.VnetName = strings.TrimSpace(cfg.VnetName) } func (cfg *Config) validate() error { @@ -199,9 +188,6 @@ func (cfg *Config) validate() error { {cfg.VMType, "VM type"}, // Even though the config doesnt use some of these, // its good to validate they were set in the environment - {cfg.SubnetID, "subnet ID"}, - {cfg.SubnetName, "subnet name"}, - {cfg.VnetName, "vnet name"}, } for _, field := range fields { diff --git a/pkg/auth/config_test.go b/pkg/auth/config_test.go index 3b35790d9..01231bd1b 100644 --- a/pkg/auth/config_test.go +++ b/pkg/auth/config_test.go @@ -40,9 +40,6 @@ func TestBuildAzureConfig(t *testing.T) { SubscriptionID: "12345", ResourceGroup: "my-rg", NodeResourceGroup: "my-node-rg", - SubnetID: "12345", - SubnetName: "my-subnet", - VnetName: "my-vnet", VMType: "vmss", }, wantErr: false, @@ -61,9 +58,6 @@ func TestBuildAzureConfig(t *testing.T) { SubscriptionID: "12345", ResourceGroup: "my-rg", NodeResourceGroup: "my-node-rg", - SubnetID: "12345", - SubnetName: "my-subnet", - VnetName: "my-vnet", VMType: "vm", }, wantErr: false, @@ -97,9 +91,6 @@ func TestBuildAzureConfig(t *testing.T) { SubscriptionID: "12345", ResourceGroup: "my-rg", NodeResourceGroup: "my-node-rg", - SubnetID: "12345", - SubnetName: "my-subnet", - VnetName: "my-vnet", VMType: "vmss", UseManagedIdentityExtension: true, UserAssignedIdentityID: "12345", diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 914ec91d1..4f7e70276 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -30,6 +30,8 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator/scheme" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/karpenter-provider-azure/pkg/apis" "github.com/Azure/karpenter-provider-azure/pkg/auth" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" @@ -40,6 +42,8 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate" "github.com/Azure/karpenter-provider-azure/pkg/providers/loadbalancer" "github.com/Azure/karpenter-provider-azure/pkg/providers/pricing" + "github.com/Azure/karpenter-provider-azure/pkg/utils" + armopts "github.com/Azure/karpenter-provider-azure/pkg/utils/opts" "sigs.k8s.io/karpenter/pkg/operator" ) @@ -64,11 +68,14 @@ type Operator struct { func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { azConfig, err := GetAZConfig() - lo.Must0(err, "creating Azure config") // TODO: I assume we prefer this over the cleaner azConfig := lo.Must(GetAzConfig()), as this has a helpful error message? + lo.Must0(err, "creating Azure config") // NOTE: we prefer this over the cleaner azConfig := lo.Must(GetAzConfig()), as when initializing the client there are helpful error messages in initializing clients and the azure config azClient, err := instance.CreateAZClient(ctx, azConfig) lo.Must0(err, "creating Azure client") + vnetGUID, err := getVnetGUID(azConfig, options.FromContext(ctx).SubnetID) + lo.Must0(err, "getting VNET GUID") + unavailableOfferingsCache := azurecache.NewUnavailableOfferings() pricingProvider := pricing.NewProvider( ctx, @@ -83,7 +90,10 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azClient.ImageVersionsClient, azConfig.Location, ) - imageResolver := imagefamily.New(operator.GetClient(), imageProvider) + imageResolver := imagefamily.New( + operator.GetClient(), + imageProvider, + ) launchTemplateProvider := launchtemplate.NewProvider( ctx, imageResolver, @@ -95,6 +105,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azConfig.UserAssignedIdentityID, azConfig.NodeResourceGroup, azConfig.Location, + vnetGUID, ) instanceTypeProvider := instancetype.NewProvider( azConfig.Location, @@ -116,7 +127,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont unavailableOfferingsCache, azConfig.Location, azConfig.NodeResourceGroup, - azConfig.SubnetID, + options.FromContext(ctx).SubnetID, azConfig.SubscriptionID, ) @@ -156,3 +167,28 @@ func getCABundle(restConfig *rest.Config) (*string, error) { } return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil } + +func getVnetGUID(cfg *auth.Config, subnetID string) (string, error) { + creds, err := auth.NewCredential(cfg) + if err != nil { + return "", err + } + opts := armopts.DefaultArmOpts() + vnetClient, err := armnetwork.NewVirtualNetworksClient(cfg.SubscriptionID, creds, opts) + if err != nil { + return "", err + } + + subnetParts, err := utils.GetVnetSubnetIDComponents(subnetID) + if err != nil { + return "", err + } + vnet, err := vnetClient.Get(context.Background(), subnetParts.ResourceGroupName, subnetParts.VNetName, nil) + if err != nil { + return "", err + } + if vnet.Properties == nil || vnet.Properties.ResourceGUID == nil { + return "", fmt.Errorf("vnet %s does not have a resource GUID", subnetParts.VNetName) + } + return *vnet.Properties.ResourceGUID, nil +} diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 9875d7189..5ede66943 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -63,11 +63,13 @@ type Options struct { VMMemoryOverheadPercent float64 ClusterID string KubeletClientTLSBootstrapToken string // => TLSBootstrapToken in bootstrap (may need to be per node/nodepool) - SSHPublicKey string // ssh.publicKeys.keyData => VM SSH public key // TODO: move to node template? + SSHPublicKey string // ssh.publicKeys.keyData => VM SSH public key // TODO: move to v1alpha2.AKSNodeClass? NetworkPlugin string // => NetworkPlugin in bootstrap NetworkPolicy string // => NetworkPolicy in bootstrap NodeIdentities []string // => Applied onto each VM + SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass + setFlags map[string]bool } @@ -79,6 +81,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.") fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", "azure"), "The network plugin used by the cluster.") fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.") + fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR") fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.") } diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go index 7b202d080..9ed1bb21c 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/operator/options/options_validation.go @@ -20,6 +20,7 @@ import ( "fmt" "net/url" + "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/go-playground/validator/v10" "go.uber.org/multierr" ) @@ -30,10 +31,19 @@ func (o Options) Validate() error { o.validateRequiredFields(), o.validateEndpoint(), o.validateVMMemoryOverheadPercent(), + o.validateVnetSubnetID(), validate.Struct(o), ) } +func (o Options) validateVnetSubnetID() error { + _, err := utils.GetVnetSubnetIDComponents(o.SubnetID) + if err != nil { + return fmt.Errorf("vnet-subnet-id is invalid: %w", err) + } + return nil +} + func (o Options) validateEndpoint() error { if o.ClusterEndpoint == "" { return nil @@ -67,5 +77,8 @@ func (o Options) validateRequiredFields() error { if o.SSHPublicKey == "" { return fmt.Errorf("missing field, ssh-public-key") } + if o.SubnetID == "" { + return fmt.Errorf("missing field, vnet-subnet-id") + } return nil } diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index f5d7ecfd9..b223e75f4 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -92,6 +92,7 @@ var _ = Describe("Options", func() { os.Setenv("NETWORK_PLUGIN", "env-network-plugin") os.Setenv("NETWORK_POLICY", "env-network-policy") os.Setenv("NODE_IDENTITIES", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid1,/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid2") + os.Setenv("VNET_SUBNET_ID", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub") fs = &coreoptions.FlagSet{ FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), } @@ -107,6 +108,7 @@ var _ = Describe("Options", func() { SSHPublicKey: lo.ToPtr("env-ssh-public-key"), NetworkPlugin: lo.ToPtr("env-network-plugin"), NetworkPolicy: lo.ToPtr("env-network-policy"), + SubnetID: lo.ToPtr("/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"), NodeIdentities: []string{"/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid1", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid2"}, })) }) diff --git a/pkg/providers/imagefamily/azlinux.go b/pkg/providers/imagefamily/azlinux.go index 5640becf4..3d36d8cc2 100644 --- a/pkg/providers/imagefamily/azlinux.go +++ b/pkg/providers/imagefamily/azlinux.go @@ -84,8 +84,9 @@ func (u AzureLinux) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, ta CABundle: caBundle, GPUNode: u.Options.GPUNode, GPUDriverVersion: u.Options.GPUDriverVersion, - // GPUImageSHA: u.Options.GPUImageSHA, image sha only applies to ubuntu - // SEE: https://github.com/Azure/AgentBaker/blob/f393d6e4d689d9204d6000c85623ad9b764e2a29/vhdbuilder/packer/install-dependencies.sh#L201 + // GPUImageSHA: u.Options.GPUImageSHA - GPU image SHA only applies to Ubuntu + // See: https://github.com/Azure/AgentBaker/blob/f393d6e4d689d9204d6000c85623ad9b764e2a29/vhdbuilder/packer/install-dependencies.sh#L201 + SubnetID: u.Options.SubnetID, }, Arch: u.Options.Arch, TenantID: u.Options.TenantID, diff --git a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go index 0ba59a481..807428f6e 100644 --- a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go @@ -21,10 +21,10 @@ import ( _ "embed" "encoding/base64" "fmt" - "os" "strings" "text/template" + "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "knative.dev/pkg/ptr" @@ -389,17 +389,8 @@ var ( } ) -// Node Labels for Vnet const ( - vnetDataPlaneLabel = "kubernetes.azure.com/ebpf-dataplane" - vnetNetworkNameLabel = "kubernetes.azure.com/network-name" - vnetSubnetNameLabel = "kubernetes.azure.com/network-subnet" - vnetSubscriptionIDLabel = "kubernetes.azure.com/network-subscription" - vnetGUIDLabel = "kubernetes.azure.com/nodenetwork-vnetguid" - vnetPodNetworkTypeLabel = "kubernetes.azure.com/podnetwork-type" - ciliumDataPlane = "cilium" - overlayNetworkType = "overlay" - globalAKSMirror = "https://acs-mirror.azureedge.net" + globalAKSMirror = "https://acs-mirror.azureedge.net" ) func (a AKS) aksBootstrapScript() (string, error) { @@ -450,7 +441,6 @@ func (a AKS) applyOptions(nbv *NodeBootstrapVariables) { // calculated values nbv.EnsureNoDupePromiscuousBridge = nbv.NeedsContainerd && nbv.NetworkPlugin == "kubenet" && nbv.NetworkPolicy != "calico" nbv.NetworkSecurityGroup = fmt.Sprintf("aks-agentpool-%s-nsg", a.ClusterID) - nbv.VirtualNetwork = fmt.Sprintf("aks-vnet-%s", a.ClusterID) nbv.RouteTable = fmt.Sprintf("aks-agentpool-%s-routetable", a.ClusterID) if a.GPUNode { @@ -464,21 +454,11 @@ func (a AKS) applyOptions(nbv *NodeBootstrapVariables) { kubeletLabels := lo.Assign(kubeletNodeLabelsBase, a.Labels) getAgentbakerGeneratedLabels(a.ResourceGroup, kubeletLabels) - //Adding vnet-related labels to the nodeLabels. - azureVnetGUID := os.Getenv("AZURE_VNET_GUID") - azureVnetName := os.Getenv("AZURE_VNET_NAME") - azureSubnetName := os.Getenv("AZURE_SUBNET_NAME") - - vnetLabels := map[string]string{ - vnetDataPlaneLabel: ciliumDataPlane, - vnetNetworkNameLabel: azureVnetName, - vnetSubnetNameLabel: azureSubnetName, - vnetSubscriptionIDLabel: a.SubscriptionID, - vnetGUIDLabel: azureVnetGUID, - vnetPodNetworkTypeLabel: overlayNetworkType, - } + subnetParts, _ := utils.GetVnetSubnetIDComponents(a.SubnetID) + nbv.Subnet = subnetParts.SubnetName + nbv.VirtualNetworkResourceGroup = subnetParts.ResourceGroupName + nbv.VirtualNetwork = subnetParts.VNetName - kubeletLabels = lo.Assign(kubeletLabels, vnetLabels) nbv.KubeletNodeLabels = strings.Join(lo.MapToSlice(kubeletLabels, func(k, v string) string { return fmt.Sprintf("%s=%s", k, v) }), ",") diff --git a/pkg/providers/imagefamily/bootstrap/bootstrap.go b/pkg/providers/imagefamily/bootstrap/bootstrap.go index 8505d1b41..2bf1fab8d 100644 --- a/pkg/providers/imagefamily/bootstrap/bootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/bootstrap.go @@ -32,6 +32,7 @@ type Options struct { GPUNode bool GPUDriverVersion string GPUImageSHA string + SubnetID string } // Bootstrapper can be implemented to generate a bootstrap script diff --git a/pkg/providers/imagefamily/resolver.go b/pkg/providers/imagefamily/resolver.go index 2c292e959..b318c7d50 100644 --- a/pkg/providers/imagefamily/resolver.go +++ b/pkg/providers/imagefamily/resolver.go @@ -94,7 +94,6 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, kubeletConfig.EvictionHard = map[string]string{ instancetype.MemoryAvailable: instanceType.Overhead.EvictionThreshold.Memory().String()} kubeletConfig.MaxPods = lo.ToPtr(getMaxPods(staticParameters.NetworkPlugin)) - logging.FromContext(ctx).Infof("Resolved image %s for instance type %s", imageID, instanceType.Name) template := &template.Parameters{ StaticParameters: staticParameters, diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go index 5c0177206..a7c3b8ee2 100644 --- a/pkg/providers/imagefamily/ubuntu_2204.go +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -85,6 +85,7 @@ func (u Ubuntu2204) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, ta GPUNode: u.Options.GPUNode, GPUDriverVersion: u.Options.GPUDriverVersion, GPUImageSHA: u.Options.GPUImageSHA, + SubnetID: u.Options.SubnetID, }, Arch: u.Options.Arch, TenantID: u.Options.TenantID, diff --git a/pkg/providers/instance/azure_client.go b/pkg/providers/instance/azure_client.go index 52509c17d..025e02caa 100644 --- a/pkg/providers/instance/azure_client.go +++ b/pkg/providers/instance/azure_client.go @@ -18,12 +18,6 @@ package instance import ( "context" - "fmt" - "os" - - // nolint SA1019 - deprecated package - - "github.com/samber/lo" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" @@ -113,18 +107,6 @@ func CreateAZClient(ctx context.Context, cfg *auth.Config) (*AZClient, error) { return azClient, nil } -func handleVNET(cfg *auth.Config, vnetClient *armnetwork.VirtualNetworksClient) error { - vnet, err := vnetClient.Get(context.Background(), cfg.NodeResourceGroup, cfg.VnetName, nil) - if err != nil { - return err - } - if vnet.Properties == nil || vnet.Properties.ResourceGUID == nil { - return fmt.Errorf("vnet %s does not have a resource GUID", cfg.VnetName) - } - os.Setenv("AZURE_VNET_GUID", lo.FromPtr(vnet.Properties.ResourceGUID)) - return nil -} - func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) (*AZClient, error) { cred, err := auth.NewCredential(cfg) if err != nil { @@ -143,14 +125,6 @@ func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) } klog.V(5).Infof("Created network interface client %v using token credential", interfacesClient) - vnetClient, err := armnetwork.NewVirtualNetworksClient(cfg.SubscriptionID, cred, opts) - if err != nil { - return nil, err - } - err = handleVNET(cfg, vnetClient) - if err != nil { - return nil, err - } virtualMachinesClient, err := armcompute.NewVirtualMachinesClient(cfg.SubscriptionID, cred, opts) if err != nil { return nil, err diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 9de6ac78a..6dd94b4a9 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -23,7 +23,6 @@ import ( "fmt" "io" "net/http" - "os" "strings" "testing" "time" @@ -110,15 +109,10 @@ var _ = AfterSuite(func() { }) var _ = Describe("InstanceType Provider", func() { - var nodeClass *v1alpha2.AKSNodeClass var nodePool *corev1beta1.NodePool BeforeEach(func() { - os.Setenv("AZURE_VNET_GUID", "test-vnet-guid") - os.Setenv("AZURE_VNET_NAME", "aks-vnet-00000000") - os.Setenv("AZURE_SUBNET_NAME", "test-subnet-name") - nodeClass = test.AKSNodeClass() nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ @@ -142,6 +136,37 @@ var _ = Describe("InstanceType Provider", func() { ExpectCleanedUp(ctx, env.Client) }) + Context("Subnet", func() { + It("should use the VNET_SUBNET_ID", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + nic := azureEnv.NetworkInterfacesAPI.NetworkInterfacesCreateOrUpdateBehavior.CalledWithInput.Pop() + Expect(nic).NotTo(BeNil()) + Expect(lo.FromPtr(nic.Interface.Properties.IPConfigurations[0].Properties.Subnet.ID)).To(Equal("/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub")) + }) + It("should produce all required azure cni labels", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + Expect(decodedString).To(SatisfyAll( + ContainSubstring("kubernetes.azure.com/ebpf-dataplane=cilium"), + ContainSubstring("kubernetes.azure.com/network-subnet=karpentersub"), + ContainSubstring("kubernetes.azure.com/nodenetwork-vnetguid=test-vnet-guid"), + ContainSubstring("kubernetes.azure.com/podnetwork-type=overlay"), + )) + }) + }) Context("VM Creation Failures", func() { It("should delete the network interface on failure to create the vm", func() { ErrMsg := "test error" @@ -592,31 +617,6 @@ var _ = Describe("InstanceType Provider", func() { }) }) - Context("Provisioner with VNetNodeLabel", func() { - It("should support provisioning with VNet node labels", func() { - ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) - ExpectScheduled(ctx, env.Client, pod) - - Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) - vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM - customData := *vm.Properties.OSProfile.CustomData - Expect(customData).ToNot(BeNil()) - decodedBytes, err := base64.StdEncoding.DecodeString(customData) - Expect(err).To(Succeed()) - decodedString := string(decodedBytes[:]) - Expect(decodedString).To(SatisfyAll( - ContainSubstring("kubernetes.azure.com/ebpf-dataplane=cilium"), - ContainSubstring("kubernetes.azure.com/network-name=aks-vnet-00000000"), - ContainSubstring("kubernetes.azure.com/network-subnet=test-subnet-name"), - ContainSubstring("kubernetes.azure.com/network-subscription=test-subscription"), - ContainSubstring("kubernetes.azure.com/nodenetwork-vnetguid=test-vnet-guid"), - ContainSubstring("kubernetes.azure.com/podnetwork-type=overlay"), - )) - }) - }) - Context("Unavailable Offerings", func() { It("should not allocate a vm in a zone marked as unavailable", func() { azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeSpot) diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 19725edef..3c5e5ac95 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -36,6 +36,14 @@ import ( const ( karpenterManagedTagKey = "karpenter.azure.com/cluster" + + networkDataplaneCilium = "cilium" + vnetDataPlaneLabel = "kubernetes.azure.com/ebpf-dataplane" + vnetSubnetNameLabel = "kubernetes.azure.com/network-subnet" + vnetGUIDLabel = "kubernetes.azure.com/nodenetwork-vnetguid" + vnetPodNetworkTypeLabel = "kubernetes.azure.com/podnetwork-type" + + networkModeOverlay = "overlay" ) type Template struct { @@ -54,12 +62,13 @@ type Provider struct { userAssignedIdentityID string resourceGroup string location string + vnetGUID string } // TODO: add caching of launch templates func NewProvider(_ context.Context, imageFamily *imagefamily.Resolver, imageProvider *imagefamily.Provider, caBundle *string, clusterEndpoint string, - tenantID, subscriptionID, userAssignedIdentityID, resourceGroup, location string, + tenantID, subscriptionID, userAssignedIdentityID, resourceGroup, location, vnetGUID string, ) *Provider { return &Provider{ imageFamily: imageFamily, @@ -71,12 +80,17 @@ func NewProvider(_ context.Context, imageFamily *imagefamily.Resolver, imageProv userAssignedIdentityID: userAssignedIdentityID, resourceGroup: resourceGroup, location: location, + vnetGUID: vnetGUID, } } func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceType *cloudprovider.InstanceType, additionalLabels map[string]string) (*Template, error) { - staticParameters := p.getStaticParameters(ctx, instanceType, nodeClass, lo.Assign(nodeClaim.Labels, additionalLabels)) + staticParameters, err := p.getStaticParameters(ctx, instanceType, nodeClass, lo.Assign(nodeClaim.Labels, additionalLabels)) + if err != nil { + return nil, err + } + kubeServerVersion, err := p.imageProvider.KubeServerVersion(ctx) if err != nil { return nil, err @@ -94,11 +108,26 @@ func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeC return launchTemplate, nil } -func (p *Provider) getStaticParameters(ctx context.Context, instanceType *cloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass, labels map[string]string) *parameters.StaticParameters { +func (p *Provider) getStaticParameters(ctx context.Context, instanceType *cloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass, labels map[string]string) (*parameters.StaticParameters, error) { var arch string = corev1beta1.ArchitectureAmd64 if err := instanceType.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64))); err == nil { arch = corev1beta1.ArchitectureArm64 } + // TODO: make conditional on either Azure CNI Overlay or pod subnet + vnetLabels, err := p.getVnetInfoLabels(ctx, nodeClass) + if err != nil { + return nil, err + } + labels = lo.Assign(labels, vnetLabels) + + // TODO: Make conditional on epbf dataplane + // This label is required for the cilium agent daemonset because + // we select the nodes for the daemonset based on this label + // - key: kubernetes.azure.com/ebpf-dataplane + // operator: In + // values: + // - cilium + labels[vnetDataPlaneLabel] = networkDataplaneCilium return ¶meters.StaticParameters{ ClusterName: options.FromContext(ctx).ClusterName, @@ -120,7 +149,8 @@ func (p *Provider) getStaticParameters(ctx context.Context, instanceType *cloudp KubeletClientTLSBootstrapToken: options.FromContext(ctx).KubeletClientTLSBootstrapToken, NetworkPlugin: options.FromContext(ctx).NetworkPlugin, NetworkPolicy: options.FromContext(ctx).NetworkPolicy, - } + SubnetID: options.FromContext(ctx).SubnetID, + }, nil } func (p *Provider) createLaunchTemplate(_ context.Context, options *parameters.Parameters) (*Template, error) { @@ -147,3 +177,17 @@ func mergeTags(tags ...map[string]string) (result map[string]*string) { return strings.ReplaceAll(key, "/", "_"), to.StringPtr(value) }) } + +func (p *Provider) getVnetInfoLabels(ctx context.Context, _ *v1alpha2.AKSNodeClass) (map[string]string, error) { + // TODO(bsoghigian): this should be refactored to lo.Ternary(nodeClass.Spec.VnetSubnetID != nil, lo.FromPtr(nodeClass.Spec.VnetSubnetID), os.Getenv("AZURE_SUBNET_ID")) when we add VnetSubnetID to the nodeclass + vnetSubnetComponents, err := utils.GetVnetSubnetIDComponents(options.FromContext(ctx).SubnetID) + if err != nil { + return nil, err + } + vnetLabels := map[string]string{ + vnetSubnetNameLabel: vnetSubnetComponents.SubnetName, + vnetGUIDLabel: p.vnetGUID, + vnetPodNetworkTypeLabel: networkModeOverlay, + } + return vnetLabels, nil +} diff --git a/pkg/providers/launchtemplate/parameters/types.go b/pkg/providers/launchtemplate/parameters/types.go index b228ef5b7..238ce0710 100644 --- a/pkg/providers/launchtemplate/parameters/types.go +++ b/pkg/providers/launchtemplate/parameters/types.go @@ -41,6 +41,9 @@ type StaticParameters struct { NetworkPolicy string KubernetesVersion string + // VNET + SubnetID string + Tags map[string]string Labels map[string]string } diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 2cf3fe0c0..87a942ef3 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -41,7 +41,9 @@ func init() { corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) } -var resourceGroup = "test-resourceGroup" +var ( + resourceGroup = "test-resourceGroup" +) type Environment struct { // API @@ -116,6 +118,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi "test-userAssignedIdentity", resourceGroup, region, + "test-vnet-guid", ) loadBalancerProvider := loadbalancer.NewProvider( loadBalancersAPI, @@ -137,10 +140,10 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi launchTemplateProvider, loadBalancerProvider, unavailableOfferingsCache, - region, // region - resourceGroup, // resourceGroup - "", // subnet - "", // subscriptionID + region, + resourceGroup, + testOptions.SubnetID, + "", // subscriptionID ) return &Environment{ diff --git a/pkg/test/options.go b/pkg/test/options.go index 0fcb6a251..de910df89 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -35,6 +35,7 @@ type OptionsFields struct { NetworkPolicy *string VMMemoryOverheadPercent *float64 NodeIdentities []string + SubnetID *string } func Options(overrides ...OptionsFields) *azoptions.Options { @@ -51,8 +52,9 @@ func Options(overrides ...OptionsFields) *azoptions.Options { KubeletClientTLSBootstrapToken: lo.FromPtrOr(options.KubeletClientTLSBootstrapToken, "test-token"), SSHPublicKey: lo.FromPtrOr(options.SSHPublicKey, "test-ssh-public-key"), NetworkPlugin: lo.FromPtrOr(options.NetworkPlugin, "azure"), - NetworkPolicy: lo.FromPtrOr(options.NetworkPolicy, ""), + NetworkPolicy: lo.FromPtrOr(options.NetworkPolicy, "cilium"), VMMemoryOverheadPercent: lo.FromPtrOr(options.VMMemoryOverheadPercent, 0.075), NodeIdentities: options.NodeIdentities, + SubnetID: lo.FromPtrOr(options.SubnetID, "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"), } } diff --git a/pkg/utils/subnet_parser.go b/pkg/utils/subnet_parser.go new file mode 100644 index 000000000..73c4ea821 --- /dev/null +++ b/pkg/utils/subnet_parser.go @@ -0,0 +1,57 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "strings" +) + +// this parsing function replaces three different functions in different packages that all had bugs. Please don't use a regex to parse these +type vnetSubnetResource struct { + SubscriptionID string + ResourceGroupName string + VNetName string + SubnetName string +} + +// GetSubnetResourceID constructs the subnet resource id +func GetSubnetResourceID(subscriptionID, resourceGroupName, virtualNetworkName, subnetName string) string { + // an example subnet resource: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName} + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", subscriptionID, resourceGroupName, virtualNetworkName, subnetName) +} + +func GetVnetSubnetIDComponents(vnetSubnetID string) (vnetSubnetResource, error) { + parts := strings.Split(vnetSubnetID, "/") + if len(parts) != 11 { + return vnetSubnetResource{}, fmt.Errorf("invalid vnet subnet id: %s", vnetSubnetID) + } + + vs := vnetSubnetResource{ + SubscriptionID: parts[2], + ResourceGroupName: parts[4], + VNetName: parts[8], + SubnetName: parts[10], + } + + //this is a cheap way of ensure all the names match + mirror := GetSubnetResourceID(vs.SubscriptionID, vs.ResourceGroupName, vs.VNetName, vs.SubnetName) + if !strings.EqualFold(mirror, vnetSubnetID) { + return vnetSubnetResource{}, fmt.Errorf("invalid vnet subnet id: %s", vnetSubnetID) + } + return vs, nil +} diff --git a/pkg/utils/subnet_parser_test.go b/pkg/utils/subnet_parser_test.go new file mode 100644 index 000000000..0fb55ab32 --- /dev/null +++ b/pkg/utils/subnet_parser_test.go @@ -0,0 +1,127 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCustomvnet(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "GetVnetSubnetIDComponents") +} + +func Benchmark(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := GetVnetSubnetIDComponents("/subscriptions/00000000-0000-0000-0000-0000000000/resourceGroups/myrg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/default1") + if err != nil { + b.Fatal(err) + } + } +} + +var _ = Describe("GetVnetSubnetIDComponents", func() { + It("should return correct subnet id components", func() { + subnetResource, err := GetVnetSubnetIDComponents("/subscriptions/00000000-0000-0000-0000-0000000000/resourceGroups/myrg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/default1") + Expect(err).ToNot(HaveOccurred()) + subscriptionID := subnetResource.SubscriptionID + resourceGroupName := subnetResource.ResourceGroupName + vNetName := subnetResource.VNetName + subnetName := subnetResource.SubnetName + + Expect(subscriptionID).To(Equal("00000000-0000-0000-0000-0000000000")) + Expect(resourceGroupName).To(Equal("myrg")) + Expect(vNetName).To(Equal("my-vnet")) + Expect(subnetName).To(Equal("default1")) + }) + It("should return error when unable to parse vnet subnet id", func() { + // "/subscriptions/00000000-0000-0000-0000-0000000000/resourceGroups/myrg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/default1" + customVnetSubnetID := "someSubnetID" // invalid format + _, err := GetVnetSubnetIDComponents(customVnetSubnetID) + Expect(err).To(HaveOccurred()) + + // "resourceGr" instead of "resourceGroups" in customVnetSubnetID + customVnetSubnetID = "/subscriptions/00000000-0000-0000-0000-0000000000/resourceGr/myrg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/default1" + _, err = GetVnetSubnetIDComponents(customVnetSubnetID) + Expect(err).To(HaveOccurred()) + }) + + It("Is reflexive", func() { + vnetsubnetid := GetSubnetResourceID("sam", "red", "violet", "subaru") + vnet, err := GetVnetSubnetIDComponents(vnetsubnetid) + Expect(err).To(BeNil()) + + Expect(vnet.SubscriptionID).To(Equal("sam")) + Expect(vnet.ResourceGroupName).To(Equal("red")) + Expect(vnet.VNetName).To(Equal("violet")) + Expect(vnet.SubnetName).To(Equal("subaru")) + }) + + It("real world weirdness (subnets is repeated broke old regex)", func() { + vnetsubnetid := "/subscriptions/00000000-0000-0000-0000-0000000000/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/sillygeese-VNET/subnets/subnets/AKSMgmtv2-Subnet" + _, err := GetVnetSubnetIDComponents(vnetsubnetid) + Expect(err).ToNot(BeNil()) + }) + + It("Is case insensitive (subnetparser.GetVnetSubnetIDComponents)", func() { + vnetsubnetid := "/SubscRiptionS/mySubscRiption/ResourceGroupS/myResourceGroup/ProviDerS/MicrOsofT.NetWorK/VirtualNetwOrkS/myVirtualNetwork/SubNetS/mySubnet" + vnet, err := GetVnetSubnetIDComponents(vnetsubnetid) + Expect(err).ToNot(HaveOccurred()) + Expect(vnet.SubscriptionID).To(Equal("mySubscRiption")) + Expect(vnet.ResourceGroupName).To(Equal("myResourceGroup")) + Expect(vnet.VNetName).To(Equal("myVirtualNetwork")) + Expect(vnet.SubnetName).To(Equal("mySubnet")) + }) + + It("Fails when appropriate", func() { + _, err := GetVnetSubnetIDComponents("what/a/bunch/of/junk") + Expect(err).ToNot(BeNil()) + _, err = GetVnetSubnetIDComponents("/subscriptions/sam/resourceGroups/red/providers/Microsoft.Network/virtualNetworks/soclose") + Expect(err).ToNot(BeNil()) + }) + + It("Test GetVNETSubnetIDComponents", func() { + vnetSubnetID := "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME" + vs, err := GetVnetSubnetIDComponents(vnetSubnetID) + Expect(err).To(BeNil()) + Expect(vs.SubscriptionID).To(Equal("SUB_ID")) + Expect(vs.ResourceGroupName).To(Equal("RG_NAME")) + Expect(vs.VNetName).To(Equal("VNET_NAME")) + Expect(vs.SubnetName).To(Equal("SUBNET_NAME")) + + // case-insensitive match + vnetSubnetID = "/SubscriPtioNS/SUB_ID/REsourceGroupS/RG_NAME/ProViderS/MicrosoFT.NetWorK/VirtualNetWorKS/VNET_NAME/SubneTS/SUBNET_NAME" + vs, err = GetVnetSubnetIDComponents(vnetSubnetID) + Expect(err).To(BeNil()) + Expect(vs.SubscriptionID).To(Equal("SUB_ID")) + Expect(vs.ResourceGroupName).To(Equal("RG_NAME")) + Expect(vs.VNetName).To(Equal("VNET_NAME")) + Expect(vs.SubnetName).To(Equal("SUBNET_NAME")) + + //wtwo bad ones + vnetSubnetID = "/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME" + _, err = GetVnetSubnetIDComponents(vnetSubnetID) + Expect(err).ToNot(BeNil()) + + vnetSubnetID = "badVnetSubnetID" + _, err = GetVnetSubnetIDComponents(vnetSubnetID) + Expect(err).ToNot(BeNil()) + }) +}) diff --git a/skaffold.yaml b/skaffold.yaml index 10c99a523..6584a5a05 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -59,14 +59,10 @@ manifests: value: "Please run make az-all" - name: AZURE_NODE_RESOURCE_GROUP value: "Please run make az-all" - - name: AZURE_SUBNET_ID # the id of subnet to create network interfaces on + - name: VNET_SUBNET_ID # the id of subnet to create network interfaces on value: "Please run make az-all" - name: LEADER_ELECT # disable leader election for better debugging experience value: "false" - - name: AZURE_VNET_NAME - value: "Please run make az-all" - - name: AZURE_SUBNET_NAME - value: "Please run make az-all" # disable HTTP/2 to reduce ARM throttling on large-scale tests; # with this in place write (and read) QPS can be increased too #- name: GODEBUG