load environment file and override cloud config

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/go-logr/logr v1.4.2
 	github.com/google/uuid v1.6.0
 	github.com/onsi/ginkgo/v2 v2.22.0
-	github.com/onsi/gomega v1.36.0
+	github.com/onsi/gomega v1.36.1
 	github.com/prometheus/client_golang v1.20.5
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
@@ -51,7 +51,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubelet v0.31.3
 	k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078
-	sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.2.7
+	sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.2.11
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient/cache v0.1.0
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.1.8
 	sigs.k8s.io/yaml v1.4.0

go.sum

@@ -212,8 +212,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
 github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
-github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y=
-github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw=
+github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -455,8 +455,8 @@ k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 h1:jGnCPejIetjiy2gqaJ5V0NLwTpF4w
 k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
-sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.2.7 h1:dgpy4qVuTsrgtjg0ZHcXll8DUe/aYTNBY01CpqG7qQ0=
-sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.2.7/go.mod h1:i/hFmBZ8SSJxneGtIfWKU3HTzqumpOU0WjJVeaWupUs=
+sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.2.11 h1:ydoTact4K6ji7p3AC+7suEifsGsItxQXcuLbACPxCOU=
+sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.2.11/go.mod h1:hQhOHrM/+jMh0k+YfV5GVKyqG5skeu4AOjqBEW3xYio=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/cache v0.1.0 h1:Cfi7bDwK2MRRWljNIn1my+LYTfD7JnxM4MSCVwtKxoM=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/cache v0.1.0/go.mod h1:mMfz57EOng50xYz6te2a7cPL5iXHiyNk7cjyosCDtkg=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.1.8 h1:fwg1J+nRRXkcA0lIJ/a4fJPg7tNggRH1pPOukthuAGw=

pkg/credentialprovider/azure_credentials.go

@@ -18,10 +18,8 @@ package credentialprovider
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"regexp"
 	"strings"
 	"time"
@@ -61,6 +59,7 @@ type CredentialProvider interface {
 type acrProvider struct {
 	config         *providerconfig.AzureClientConfig
 	environment    *azclient.Environment
+	clientOption   *policy.ClientOptions
 	credential     azcore.TokenCredential
 	registryMirror map[string]string // Registry mirror relation: source registry -> target registry
 }
@@ -82,22 +81,9 @@ func NewAcrProviderFromConfig(configFile string, registryMirrorStr string) (Cred
 	if err != nil {
 		return nil, fmt.Errorf("failed to load config: %w", err)
 	}
-
-	var envConfig azclient.Environment
-	envFilePath, ok := os.LookupEnv(azclient.EnvironmentFilepathName)
-	if ok {
-		content, err := os.ReadFile(envFilePath)
-		if err != nil {
-			return nil, err
-		}
-		if err = json.Unmarshal(content, &envConfig); err != nil {
-			return nil, err
-		}
-	}
-
 	var managedIdentityCredential azcore.TokenCredential
 
-	clientOption, err := azclient.GetAzCoreClientOption(&config.ARMClientConfig)
+	clientOption, env, err := azclient.GetAzCoreClientOption(&config.ARMClientConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -121,7 +107,8 @@ func NewAcrProviderFromConfig(configFile string, registryMirrorStr string) (Cred
 	return &acrProvider{
 		config:         config,
 		credential:     managedIdentityCredential,
-		environment:    &envConfig,
+		clientOption:   clientOption,
+		environment:    env,
 		registryMirror: parseRegistryMirror(registryMirrorStr),
 	}, nil
 }
@@ -201,11 +188,9 @@ func (a *acrProvider) GetCredentials(ctx context.Context, image string, _ []stri
 
 // getFromACR gets credentials from ACR.
 func (a *acrProvider) getFromACR(ctx context.Context, loginServer string) (string, string, error) {
-	config, err := azclient.GetAzureCloudConfig(&a.config.ARMClientConfig)
-	if err != nil {
-		return "", "", err
-	}
+	config := a.clientOption.Cloud
 	var armAccessToken azcore.AccessToken
+	var err error
 	if armAccessToken, err = a.credential.GetToken(ctx, policy.TokenRequestOptions{
 		Scopes: []string{
 			strings.TrimRight(config.Services[azcontainerregistry.ServiceName].Audience, "/") + "/.default",

pkg/provider/azure.go

@@ -27,10 +27,9 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/Azure/go-autorest/autorest/azure"
-
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -104,7 +103,7 @@ var (
 // Cloud holds the config and clients
 type Cloud struct {
 	azureconfig.Config
-	Environment azure.Environment
+	Environment *azclient.Environment
 
 	SubnetsClient                   subnetclient.Interface
 	InterfacesClient                interfaceclient.Interface
@@ -352,23 +351,17 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C
 		config.ClusterServiceSharedLoadBalancerHealthProbePath = consts.ClusterServiceLoadBalancerHealthProbeDefaultPath
 	}
 
-	env, err := azureconfig.ParseAzureEnvironment(config.Cloud, config.ResourceManagerEndpoint, config.IdentitySystem)
-	if err != nil {
-		return err
-	}
-
 	// Initialize rate limiting config options.
 	azureconfig.InitializeCloudProviderRateLimitConfig(&config.CloudProviderRateLimitConfig)
 
 	resourceRequestBackoff := az.setCloudProviderBackoffDefaults(config)
 
-	err = az.setLBDefaults(config)
+	err := az.setLBDefaults(config)
 	if err != nil {
 		return err
 	}
 
 	az.Config = *config
-	az.Environment = *env
 	az.ResourceRequestBackoff = resourceRequestBackoff
 	az.Metadata, err = NewInstanceMetadataService(consts.ImdsServer)
 	if err != nil {
@@ -407,7 +400,22 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C
 			return err
 		}
 	}
-	servicePrincipalToken, err := azureconfig.GetServicePrincipalToken(&config.AzureClientConfig, env, env.ServiceManagementEndpoint)
+
+	clientOption, env, err := azclient.GetAzCoreClientOption(&az.ARMClientConfig)
+	if err != nil {
+		return err
+	}
+	az.Environment = env
+
+	aadEndpoint := env.ActiveDirectoryEndpoint
+	if aadEndpoint == "" {
+		aadEndpoint = clientOption.Cloud.ActiveDirectoryAuthorityHost
+	}
+	armAudience := env.ServiceManagementEndpoint
+	if armAudience == "" {
+		armAudience = string(clientOption.Cloud.Services[cloud.ResourceManager].Audience[0])
+	}
+	servicePrincipalToken, err := azureconfig.GetServicePrincipalToken(&config.AzureClientConfig, aadEndpoint, armAudience)
 	if errors.Is(err, azureconfig.ErrorNoAuth) {
 		// Only controller-manager would lazy-initialize from secret, and credentials are required for such case.
 		if fromSecret {
@@ -440,7 +448,7 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C
 	}
 	az.AuthProvider = authProvider
 	// If uses network resources in different AAD Tenant, then prepare corresponding Service Principal Token for VM/VMSS client and network resources client
-	multiTenantServicePrincipalToken, networkResourceServicePrincipalToken, err := az.getAuthTokenInMultiTenantEnv(servicePrincipalToken, authProvider)
+	multiTenantServicePrincipalToken, networkResourceServicePrincipalToken, err := az.getAuthTokenInMultiTenantEnv(servicePrincipalToken, &clientOption.Cloud, authProvider)
 	if err != nil {
 		return err
 	}
@@ -458,7 +466,7 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C
 			networkTenantCred := authProvider.GetNetworkAzIdentity()
 			az.NetworkClientFactory, err = azclient.NewClientFactory(&azclient.ClientFactoryConfig{
 				SubscriptionID: az.NetworkResourceSubscriptionID,
-			}, &az.ARMClientConfig, networkTenantCred)
+			}, &az.ARMClientConfig, clientOption.Cloud, networkTenantCred)
 			if err != nil {
 				return err
 			}
@@ -468,7 +476,7 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *config.C
 		}
 		az.ComputeClientFactory, err = azclient.NewClientFactory(&azclient.ClientFactoryConfig{
 			SubscriptionID: az.SubscriptionID,
-		}, &az.ARMClientConfig, cred)
+		}, &az.ARMClientConfig, clientOption.Cloud, cred)
 		if err != nil {
 			return err
 		}
@@ -625,16 +633,24 @@ func (az *Cloud) setLBDefaults(config *azureconfig.Config) error {
 	return nil
 }
 
-func (az *Cloud) getAuthTokenInMultiTenantEnv(_ *adal.ServicePrincipalToken, authProvider *azclient.AuthProvider) (adal.MultitenantOAuthTokenProvider, adal.OAuthTokenProvider, error) {
+func (az *Cloud) getAuthTokenInMultiTenantEnv(_ *adal.ServicePrincipalToken, cloudConfig *cloud.Configuration, authProvider *azclient.AuthProvider) (adal.MultitenantOAuthTokenProvider, adal.OAuthTokenProvider, error) {
 	var err error
 	var multiTenantOAuthToken adal.MultitenantOAuthTokenProvider
 	var networkResourceServicePrincipalToken adal.OAuthTokenProvider
 	if az.Config.UsesNetworkResourceInDifferentTenant() {
-		multiTenantOAuthToken, err = azureconfig.GetMultiTenantServicePrincipalToken(&az.Config.AzureClientConfig, &az.Environment, authProvider)
+		aadEndpoint := az.Environment.ActiveDirectoryEndpoint
+		if aadEndpoint == "" {
+			aadEndpoint = cloudConfig.ActiveDirectoryAuthorityHost
+		}
+		armAudience := az.Environment.ServiceManagementEndpoint
+		if armAudience == "" {
+			armAudience = string(cloudConfig.Services[cloud.ResourceManager].Audience[0])
+		}
+		multiTenantOAuthToken, err = azureconfig.GetMultiTenantServicePrincipalToken(&az.Config.AzureClientConfig, aadEndpoint, armAudience, authProvider)
 		if err != nil {
 			return nil, nil, err
 		}
-		networkResourceServicePrincipalToken, err = azureconfig.GetNetworkResourceServicePrincipalToken(&az.Config.AzureClientConfig, &az.Environment, authProvider)
+		networkResourceServicePrincipalToken, err = azureconfig.GetNetworkResourceServicePrincipalToken(&az.Config.AzureClientConfig, aadEndpoint, armAudience, authProvider)
 		if err != nil {
 			return nil, nil, err
 		}

pkg/provider/azure_fakes.go

@@ -107,6 +107,15 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) {
 		nodePrivateIPs:           map[string]*utilsets.IgnoreCaseSet{},
 		routeCIDRs:               map[string]string{},
 		eventRecorder:            &record.FakeRecorder{},
+		Environment: &azclient.Environment{
+			Name:                       "AzurePublicCloud",
+			ServiceManagementEndpoint:  "https://management.core.windows.net/",
+			ResourceManagerEndpoint:    "https://management.azure.com/",
+			ActiveDirectoryEndpoint:    "https://login.microsoftonline.com/",
+			StorageEndpointSuffix:      "core.windows.net",
+			ContainerRegistryDNSSuffix: "azurecr.io",
+			TokenAudience:              "https://management.azure.com/",
+		},
 	}
 	az.DisksClient = mockdiskclient.NewMockInterface(ctrl)
 	az.InterfacesClient = mockinterfaceclient.NewMockInterface(ctrl)

pkg/provider/config/azure_auth.go

@@ -76,7 +76,7 @@ type AzureClientConfig struct {
 // If NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
 // than only azure clients except VM/VMSS and network resource ones use this method to fetch Token.
 // For tokens for VM/VMSS and network resource ones, please check GetMultiTenantServicePrincipalToken and GetNetworkResourceServicePrincipalToken.
-func GetServicePrincipalToken(config *AzureClientConfig, env *azure.Environment, resource string) (*adal.ServicePrincipalToken, error) {
+func GetServicePrincipalToken(config *AzureClientConfig, aadEndpoint, resource string) (*adal.ServicePrincipalToken, error) {
 	logger := klog.Background().WithName("GetServicePrincipalToken")
 	var tenantID string
 	if strings.EqualFold(config.IdentitySystem, consts.ADFSIdentitySystem) {
@@ -85,13 +85,9 @@ func GetServicePrincipalToken(config *AzureClientConfig, env *azure.Environment,
 		tenantID = config.TenantID
 	}
 
-	if resource == "" {
-		resource = env.ServiceManagementEndpoint
-	}
-
 	if config.UseFederatedWorkloadIdentityExtension {
 		logger.V(2).Info("Setup ARM general resource token provider", "method", "workload_identity")
-		oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.TenantID, nil)
+		oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(aadEndpoint, config.TenantID, nil)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create the OAuth config: %w", err)
 		}
@@ -104,7 +100,7 @@ func GetServicePrincipalToken(config *AzureClientConfig, env *azure.Environment,
 			return string(jwt), nil
 		}
 
-		token, err := adal.NewServicePrincipalTokenFromFederatedTokenCallback(*oauthConfig, config.AADClientID, jwtCallback, env.ResourceManagerEndpoint)
+		token, err := adal.NewServicePrincipalTokenFromFederatedTokenCallback(*oauthConfig, config.AADClientID, jwtCallback, resource)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create a workload identity token: %w", err)
 		}
@@ -140,7 +136,7 @@ func GetServicePrincipalToken(config *AzureClientConfig, env *azure.Environment,
 			resource)
 	}
 
-	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, tenantID, nil)
+	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(aadEndpoint, tenantID, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error creating the OAuth config: %w", err)
 	}
@@ -185,7 +181,7 @@ func GetServicePrincipalToken(config *AzureClientConfig, env *azure.Environment,
 // PrimaryToken of the returned multi-tenant token is for the AAD Tenant specified by TenantID, and AuxiliaryToken of the returned multi-tenant token is for the AAD Tenant specified by NetworkResourceTenantID.
 //
 // Azure VM/VMSS clients use this multi-tenant token, in order to operate those VM/VMSS in AAD Tenant specified by TenantID, and meanwhile in their payload they are referencing network resources (e.g. Load Balancer, Network Security Group, etc.) in AAD Tenant specified by NetworkResourceTenantID.
-func GetMultiTenantServicePrincipalToken(config *AzureClientConfig, env *azure.Environment, authProvider *azclient.AuthProvider) (adal.MultitenantOAuthTokenProvider, error) {
+func GetMultiTenantServicePrincipalToken(config *AzureClientConfig, aadEndpoint string, resource string, authProvider *azclient.AuthProvider) (adal.MultitenantOAuthTokenProvider, error) {
 	logger := klog.Background().WithName("GetMultiTenantServicePrincipalToken")
 
 	err := config.ValidateForMultiTenant()
@@ -194,7 +190,7 @@ func GetMultiTenantServicePrincipalToken(config *AzureClientConfig, env *azure.E
 	}
 
 	multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(
-		env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
+		aadEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("creating the multi-tenant OAuth config: %w", err)
 	}
@@ -205,7 +201,7 @@ func GetMultiTenantServicePrincipalToken(config *AzureClientConfig, env *azure.E
 			multiTenantOAuthConfig,
 			config.AADClientID,
 			config.AADClientSecret,
-			env.ServiceManagementEndpoint)
+			resource)
 	}
 
 	if len(config.AADClientCertPath) > 0 {
@@ -223,7 +219,7 @@ func GetMultiTenantServicePrincipalToken(config *AzureClientConfig, env *azure.E
 			config.AADClientID,
 			certificate,
 			privateKey,
-			env.ServiceManagementEndpoint)
+			resource)
 	}
 
 	if authProvider.ComputeCredential != nil && authProvider.NetworkCredential != nil {
@@ -247,15 +243,15 @@ func GetMultiTenantServicePrincipalToken(config *AzureClientConfig, env *azure.E
 // and this method creates a new service principal token for network resources tenant based on the configuration.
 //
 // Azure network resource (Load Balancer, Public IP, Route Table, Network Security Group and their sub level resources) clients use this multi-tenant token, in order to operate resources in AAD Tenant specified by NetworkResourceTenantID.
-func GetNetworkResourceServicePrincipalToken(config *AzureClientConfig, env *azure.Environment, authProvider *azclient.AuthProvider) (adal.OAuthTokenProvider, error) {
+func GetNetworkResourceServicePrincipalToken(config *AzureClientConfig, aadEndpoint string, resource string, authProvider *azclient.AuthProvider) (adal.OAuthTokenProvider, error) {
 	logger := klog.Background().WithName("GetNetworkResourceServicePrincipalToken")
 
 	err := config.ValidateForMultiTenant()
 	if err != nil {
 		return nil, fmt.Errorf("got error(%w) in getting network resources service principal token", err)
 	}
 
-	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil)
+	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(aadEndpoint, config.NetworkResourceTenantID, nil)
 	if err != nil {
 		return nil, fmt.Errorf("creating the OAuth config for network resources tenant: %w", err)
 	}
@@ -266,7 +262,7 @@ func GetNetworkResourceServicePrincipalToken(config *AzureClientConfig, env *azu
 			*oauthConfig,
 			config.AADClientID,
 			config.AADClientSecret,
-			env.ServiceManagementEndpoint)
+			resource)
 	}
 
 	if len(config.AADClientCertPath) > 0 {
@@ -284,7 +280,7 @@ func GetNetworkResourceServicePrincipalToken(config *AzureClientConfig, env *azu
 			config.AADClientID,
 			certificate,
 			privateKey,
-			env.ServiceManagementEndpoint)
+			resource)
 	}
 
 	if authProvider.ComputeCredential != nil && authProvider.NetworkCredential != nil {