refresh GH action ID token during pipeline runs (#1214)

* refresh GH action ID token during pipeline runs mitigation for Azure/azure-cli#28708 Signed-off-by: Gerd Oberlechner <[email protected]> * provide env vars to gh actions Signed-off-by: Gerd Oberlechner <[email protected]> * timeout handling Signed-off-by: Gerd Oberlechner <[email protected]> --------- Signed-off-by: Gerd Oberlechner <[email protected]>
Azure · Feb 3, 2025 · 2f56c56 · 2f56c56
1 parent 24eb849
commit 2f56c56
Show file tree

Hide file tree

Showing 14 changed files with 207 additions and 44 deletions.
diff --git a/.github/workflows/aro-hcp-cd.yml b/.github/workflows/aro-hcp-cd.yml
@@ -4,6 +4,11 @@
     DEPLOY_ENV: dev
     PERSIST: true
     SKIP_CONFIRM: true
+    AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+    AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+    AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+    #AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+    PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
   on:
     workflow_dispatch:
     pull_request:

diff --git a/.github/workflows/environment-infra-cd.yml b/.github/workflows/environment-infra-cd.yml
@@ -4,6 +4,11 @@
     DEPLOY_ENV: ${{ inputs.deploy_env }}
     SKIP_CONFIRM: true
     PERSIST: true
+    AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+    AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+    AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+    #AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+    PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
   on:
     workflow_call:
       inputs:
@@ -90,12 +95,8 @@
         - name: 'Deploy Cluster'
           run: |
             cd dev-infrastructure/
-            az account get-access-token --output none
             PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc
 
-            # grant GH action user access to resources
-            # PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc.dev-role-assignments
-
         - name: 'Az CLI login again'
           uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
           with:
@@ -106,7 +107,6 @@
         - name: 'Deploy rest'
           run: |
             cd dev-infrastructure/
-            az account get-access-token --output none
             PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc.aks.admin-access
 
         - name: 'CS PR check MSI'
@@ -154,7 +154,6 @@
         - name: 'Deploy or Update'
           run: |
             cd dev-infrastructure/
-            az account get-access-token --output none
             PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make mgmt
 
         - name: 'Az CLI login again'
@@ -167,5 +166,4 @@
         - name: 'Deploy rest'
           run: |
             cd dev-infrastructure/
-            az account get-access-token --output none
             PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make mgmt.aks.admin-access
diff --git a/.github/workflows/services-cd.yml b/.github/workflows/services-cd.yml
@@ -3,6 +3,11 @@
   env:
     DEPLOY_ENV: ${{ inputs.deploy_env }}
     SKIP_CONFIRM: true
+    AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+    AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+    AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+    #AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+    PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
   on:
     workflow_call:
       inputs:
@@ -33,8 +38,6 @@
   jobs:
 
     deploy_to_service_cluster:
-      env:
-        PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
       permissions:
         id-token: 'write'
         contents: 'read'
@@ -112,8 +115,6 @@
             ./svc-deploy.sh ${DEPLOY_ENV} cluster-service svc deploy-pr-env-deps
 
     deploy_to_management_cluster:
-      env:
-        PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
       permissions:
         id-token: 'write'
         contents: 'read'

diff --git a/.github/workflows/services-ci.yml b/.github/workflows/services-ci.yml
@@ -2,6 +2,11 @@
   name: ARO HCP Dev Environment Continuous Deployment
   env:
     DEPLOY_ENV: dev
+    AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+    AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+    AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+    #AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+    PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
   on:
     workflow_call:
       inputs:

diff --git a/.github/workflows/services-pr-check.yml b/.github/workflows/services-pr-check.yml
@@ -3,6 +3,11 @@
   env:
     DEPLOY_ENV: dev
     SKIP_CONFIRM: true
+    AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+    AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+    AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+    #AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/
+    PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
   on:
     pull_request:
       paths:
@@ -33,8 +38,6 @@
 
   jobs:
     service_cluster_pr_check:
-      env:
-        PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
       permissions:
         id-token: 'write'
         contents: 'read'
@@ -47,13 +50,6 @@
         - name: "install azure-cli"
           uses: "Azure/ARO-HCP@main"
 
-        - name: 'Az CLI login'
-          uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
-          with:
-            client-id: ${{ secrets.AZURE_CLIENT_ID }}
-            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
         - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0
 
         # Used to deploy Cluster Service
@@ -105,8 +101,6 @@
             make metrics.dry_run
 
     mgmt_cluster_pr_check:
-      env:
-        PRINCIPAL_ID: ${{ secrets.GHA_PRINCIPAL_ID }}
       permissions:
         id-token: 'write'
         contents: 'read'
@@ -119,13 +113,6 @@
         - name: "install azure-cli"
           uses: "Azure/ARO-HCP@main"
 
-        - name: 'Az CLI login'
-          uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
-          with:
-            client-id: ${{ secrets.AZURE_CLIENT_ID }}
-            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
         - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0
 
         # Used to deploy Cluster Service

diff --git a/tooling/templatize/cmd/pipeline/run/cmd.go b/tooling/templatize/cmd/pipeline/run/cmd.go
@@ -4,6 +4,8 @@ import (
 	"context"
 
 	"github.com/spf13/cobra"
+
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/azauth"
 )
 
 func NewCommand() (*cobra.Command, error) {
@@ -31,5 +33,9 @@ func runPipeline(ctx context.Context, opts *RawRunOptions) error {
 	if err != nil {
 		return err
 	}
+	err = azauth.SetupAzureAuth(ctx)
+	if err != nil {
+		return err
+	}
 	return completed.RunPipeline(ctx)
 }
diff --git a/tooling/templatize/internal/end2end/e2e.go b/tooling/templatize/internal/end2end/e2e.go
@@ -9,9 +9,9 @@ import (
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/azauth"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
@@ -112,7 +112,7 @@ func (e *e2eImpl) UseRandomRG() func() error {
 		if err != nil {
 			return err
 		}
-		cred, err := azidentity.NewDefaultAzureCredential(nil)
+		cred, err := azauth.GetAzureTokenCredentials()
 		if err != nil {
 			return err
 		}

diff --git a/tooling/templatize/internal/end2end/e2e_test.go b/tooling/templatize/internal/end2end/e2e_test.go
@@ -10,10 +10,10 @@ import (
 	"gotest.tools/v3/assert"
 
 	"github.com/Azure/ARO-HCP/tooling/templatize/cmd/pipeline/run"
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/azauth"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 )
@@ -107,7 +107,7 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
 	subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)")
 	assert.NilError(t, err)
 
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azauth.GetAzureTokenCredentials()
 	assert.NilError(t, err)
 
 	zonesClient, err := armdns.NewZonesClient(subsriptionID, cred, nil)
@@ -314,7 +314,7 @@ resource newRG 'Microsoft.Resources/resourceGroups@2024-03-01' = {
 	subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)")
 	assert.NilError(t, err)
 
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azauth.GetAzureTokenCredentials()
 	assert.NilError(t, err)
 
 	rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil)
@@ -354,7 +354,7 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
 	subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)")
 	assert.NilError(t, err)
 
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azauth.GetAzureTokenCredentials()
 	assert.NilError(t, err)
 
 	zonesClient, err := armdns.NewZonesClient(subsriptionID, cred, nil)

diff --git a/tooling/templatize/pkg/aks/admin.go b/tooling/templatize/pkg/aks/admin.go
@@ -9,14 +9,15 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	armauthorization "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3"
 	"github.com/google/uuid"
 	auth "github.com/microsoft/kiota-authentication-azure-go"
 	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/azauth"
 )
 
 const (
@@ -99,7 +100,7 @@ func getCurrentUserObjectID(ctx context.Context) (string, error) {
 	}
 
 	// Create a Graph client using Azure Credentials
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azauth.GetAzureTokenCredentials()
 	if err != nil {
 		return "", fmt.Errorf("failed to obtain a credential: %w", err)
 	}
@@ -130,7 +131,7 @@ func getCurrentUserObjectID(ctx context.Context) (string, error) {
 
 func assignClusterAdminRBACRole(ctx context.Context, subscriptionID, resourceGroupName, aksClusterName, userObjectID, roleID string) error {
 	// Create a new Azure identity client
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azauth.GetAzureTokenCredentials()
 	if err != nil {
 		return fmt.Errorf("failed to obtain a credential: %w", err)
 	}

diff --git a/tooling/templatize/pkg/aks/kubeconfig.go b/tooling/templatize/pkg/aks/kubeconfig.go
@@ -6,8 +6,9 @@ import (
 	"os"
 	"os/exec"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
+
+	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/azauth"
 )
 
 func GetKubeConfig(ctx context.Context, subscriptionID, resourceGroupName, aksClusterName string) (string, error) {
@@ -16,7 +17,7 @@ func GetKubeConfig(ctx context.Context, subscriptionID, resourceGroupName, aksCl
 	}
 
 	// Create a new Azure identity client
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	cred, err := azauth.GetAzureTokenCredentials()
 	if err != nil {
 		return "", fmt.Errorf("failed to obtain a credential: %v", err)
 	}

diff --git a/tooling/templatize/pkg/azauth/auth.go b/tooling/templatize/pkg/azauth/auth.go
@@ -0,0 +1,37 @@
+package azauth
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+)
+
+func SetupAzureAuth(ctx context.Context) error {
+	if githubAuthSupported() {
+		err := setupGithubAzureFederationAuthRefresher(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to setup GitHub Azure Federation Auth Refresher: %w", err)
+		}
+	}
+	return nil
+}
+
+func GetAzureTokenCredentials() (azcore.TokenCredential, error) {
+	azCLI, err := azidentity.NewAzureCLICredential(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	def, err := azidentity.NewDefaultAzureCredential(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	chain, err := azidentity.NewChainedTokenCredential([]azcore.TokenCredential{azCLI, def}, nil)
+	if err != nil {
+		return nil, err
+	}
+	return chain, nil
+}