Merge branch 'main' into patch-library-672

.github/workflows/update-policy.yml

@@ -4,7 +4,7 @@ name: Update Library Templates
 # yamllint disable-line rule:truthy
 on:
   schedule:
-    - cron: "0 8 * * 1-5"
+    - cron: "0 8 * * 1"
   workflow_dispatch:
     inputs:
       enterprise-scale-repository-branch:
@@ -42,11 +42,11 @@ jobs:
           path: ${{ env.remote_repository }}
           ref: ${{ env.remote_repository_branch }}
 
-      - uses: tibdex/github-app-token@v2
+      - uses: actions/create-github-app-token@v1
         id: generate-token
         with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
 
       - name: Configure local git
         run: |
@@ -106,6 +106,14 @@ jobs:
           git push origin ${{ env.branch_name }}
         working-directory: ${{ github.repository }}
 
+      - name: close and comment out of date prs
+        if: steps.git_status.outputs.changes > 0
+        run: |
+          PULL_REQUESTS=$(gh pr list --search "${{ env.pr_title }}" --json number,headRefName)
+          echo "$PULL_REQUESTS" | jq -r '.[] | .number' | xargs -I {} gh pr close {} --delete-branch --comment "Out of date PR, closing and deleting branch"
+        env:
+          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
+
       - name: Create pull request
         if: steps.git_status.outputs.changes > 0
         run: |

tests/pipelines/templates/tests-backend.yml

@@ -1,10 +1,11 @@
 ---
 steps:
-  - task: Bash@3
+  - task: AzureCLI@2
     name: prepare_backend
     displayName: "Prepare Backend Storage"
     inputs:
-      targetType: "inline"
-      script: "make azp-backend"
-    env:
-      ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
+      azureSubscription: ado-mscet-cae-estf
+      scriptLocation: scriptPath
+      scriptPath: "tests/scripts/azp-backend.sh"
+      scriptType: bash
+      failOnStandardError: true

tests/pipelines/templates/tests-common.yml

@@ -11,16 +11,11 @@ steps:
       targetType: "inline"
       script: "make tf-install"
 
-  - task: GoTool@0
-    displayName: "Install Go"
-    inputs:
-      version: "1.22.3"
-    condition: and(succeeded(), eq('${{ parameters.run_type }}', 'unit'))
-
-  - task: Bash@3
+  - task: AzureCLI@2
     displayName: "Prepare Terraform Environment"
     inputs:
-      targetType: "inline"
-      script: "make tf-prepare"
-    env:
-      ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
+      azureSubscription: ado-mscet-cae-estf
+      scriptLocation: scriptPath
+      scriptPath: "tests/scripts/tf-prepare.sh"
+      scriptType: bash
+      failOnStandardError: true

tests/pipelines/templates/tests-loop.yml

@@ -6,41 +6,54 @@ parameters:
     type: string
 
 steps:
-  - task: Bash@3
+  - task: AzureCLI@2
     displayName: "[terraform init]"
     inputs:
-      targetType: "inline"
-      script: "make tf-init"
+      scriptType: bash
+      scriptLocation: scriptPath
+      scriptPath: "tests/scripts/tf-init.sh"
+      failOnStandardError: true
+      addSpnToEnvironment: true
+      azureSubscription: ado-mscet-cae-estf
     env:
-      ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
       TEST_MODULE_PATH: "${{ parameters.module_path }}"
     condition: and(succeeded(), in('${{ parameters.run_type }}', 'unit', 'e2e', 'destroy'))
 
-  - task: Bash@3
+  - task: AzureCLI@2
     displayName: "[terraform plan]"
     inputs:
-      targetType: "inline"
-      script: "make tf-plan"
+      scriptType: bash
+      scriptLocation: scriptPath
+      scriptPath: "tests/scripts/tf-plan.sh"
+      failOnStandardError: true
+      addSpnToEnvironment: true
+      azureSubscription: ado-mscet-cae-estf
     env:
-      ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
       TEST_MODULE_PATH: "${{ parameters.module_path }}"
     condition: and(succeeded(), in('${{ parameters.run_type }}', 'unit'))
 
-  - task: Bash@3
+  - task: AzureCLI@2
     displayName: "[terraform apply]"
     inputs:
-      targetType: "inline"
-      script: "make tf-apply"
+      scriptType: bash
+      scriptLocation: scriptPath
+      failOnStandardError: true
+      addSpnToEnvironment: true
+      scriptPath: "tests/scripts/tf-apply.sh"
+      azureSubscription: ado-mscet-cae-estf
     env:
-      ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
       TEST_MODULE_PATH: "${{ parameters.module_path }}"
     condition: and(succeeded(), eq('${{ parameters.run_type }}', 'e2e'))
 
-  - task: Bash@3
+  - task: AzureCLI@2
     displayName: "[terraform destroy]"
     inputs:
-      targetType: "inline"
-      script: "make tf-destroy"
+      scriptType: bash
+      scriptLocation: scriptPath
+      failOnStandardError: true
+      addSpnToEnvironment: true
+      scriptPath: "tests/scripts/tf-destroy.sh"
+      azureSubscription: ado-mscet-cae-estf
     env:
       ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
       TEST_MODULE_PATH: "${{ parameters.module_path }}"

tests/pipelines/templates/tests-strategy.yml

@@ -1,11 +1,13 @@
 ---
 steps:
-  - task: PowerShell@2
+  - task: AzurePowerShell@5
     name: build_strategy
     displayName: "Generate Build Strategy"
     inputs:
-      targetType: "inline"
-      script: "make azp-strategy"
+      azureSubscription: ado-mscet-cae-estf
+      scriptType: FilePath
+      scriptPath: "tests/scripts/azp-strategy.ps1"
+      failOnStandardError: true
+      azurePowerShellVersion: 'LatestVersion'  # Adding version specification for clarity
     env:
-      ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
       BILLING_SCOPE: $(BILLING_SCOPE)

tests/pipelines/tests-e2e.yml

@@ -4,7 +4,7 @@ name: "Tests (E2E)"
 trigger: none
 
 pool:
-  vmImage: ubuntu-20.04
+  vmImage: ubuntu-22.04
 
 variables:
   - group: csu-tf-environment

tests/pipelines/tests-unit.yml

@@ -4,7 +4,7 @@ name: "Tests (Unit)"
 trigger: none
 
 pool:
-  vmImage: ubuntu-20.04
+  vmImage: ubuntu-22.04
 
 variables:
   - group: csu-tf-environment

tests/scripts/azp-backend.sh

@@ -5,20 +5,9 @@ set -e
 # Shell Script
 # - Terraform Create or Update Azure Backend Storage
 #
-
-echo "==> Authenticating cli..."
-az login \
-    --service-principal \
-    --tenant "$ARM_TENANT_ID" \
-    --username "$ARM_CLIENT_ID" \
-    --password "$ARM_CLIENT_SECRET" \
-    --query [?isDefault]
-
 echo "==> Setting active Subscription..."
 az account set \
     --subscription "$ARM_SUBSCRIPTION_ID"
-az account list \
-    --query "[?isDefault]"
 
 echo "==> Create or update Resource Group..."
 RSG_NAME="$DEFAULT_PREFIX"
@@ -40,6 +29,7 @@ SA_NAME=$(
 )
 SA_ID=$(
     az storage account create \
+        --only-show-errors \
         --name "$SA_NAME" \
         --resource-group "$RSG_NAME" \
         --location "$PRIMARY_LOCATION" \
@@ -56,13 +46,15 @@ echo "##vso[task.setVariable variable=STORAGE_ACCOUNT_NAME;isOutput=true]$SA_NAM
 
 echo "==> Create or update Storage Account permissions..."
 az role assignment create \
+    --only-show-errors \
     --role 'Storage Blob Data Contributor' \
     --assignee "$ARM_CLIENT_ID" \
     --scope "$SA_ID"
 
 echo "==> Create or update Storage Account container..."
 SC_NAME="tfstate"
 az storage container create \
+    --only-show-errors \
     --name "$SC_NAME" \
     --auth-mode 'login' \
     --account-name "$SA_NAME" \

tests/scripts/azp-strategy.ps1

@@ -20,7 +20,6 @@ $ErrorActionPreference = "Stop"
 Write-Information "==> Generating Azure Pipelines Strategy Matrix..." -InformationAction Continue
 
 $jsonDepth = 4
-$terraformUrl = "https://api.github.com/repos/hashicorp/terraform/tags"
 
 function Get-RandomId {
   [CmdletBinding()]
@@ -38,13 +37,7 @@ function Get-RandomId {
 #     1.3.*  (latest 1)
 ########################################
 
-$terraformVersionsResponse = Invoke-RestMethod -Method Get -Uri $terraformUrl -FollowRelLink
-$terraformVersionsAll = $terraformVersionsResponse.name -replace "v", ""
-
-$terraformVersions = @("1.7.0")
-$terraformVersions += $terraformVersionsAll | Where-Object { $_ -match "^1(\.\d{1,2}){1,2}$" } | Select-Object -First 1
-
-$terraformVersions = $terraformVersions | Sort-Object
+$terraformVersions = @("1.11.0")
 
 $terraformVersionsCount = $terraformVersions.Count
 
@@ -54,8 +47,7 @@ $terraformVersionsCount = $terraformVersions.Count
 # - Latest Versions: (latest 1)
 #######################################
 
-$azurermProviderVersionBase = "3.108.0"
-$azurermProviderVersionLatest = "3.116.0"
+$azurermProviderVersionBase = "3.117.0"
 
 #######################################
 # Generate Subscription Aliases
@@ -69,20 +61,6 @@ Import-Module -Name "Az.Accounts" -Force
 
 Write-Information "==> Getting Subscription Aliases..." -InformationAction Continue
 
-Write-Verbose "Switching Azure Context using Client ID [$($env:ARM_CLIENT_ID)]."
-$Credential = New-Object System.Management.Automation.PSCredential (
-  $($env:ARM_CLIENT_ID),
-  $($env:ARM_CLIENT_SECRET | ConvertTo-SecureString -AsPlainText -Force)
-)
-$ctx = Connect-AzAccount `
-  -ServicePrincipal `
-  -Tenant $($env:ARM_TENANT_ID) `
-  -SubscriptionId $($env:ARM_SUBSCRIPTION_ID) `
-  -Credential $Credential `
-  -WarningAction SilentlyContinue
-
-Write-Information " Successfully authenticated account ($($ctx.Context.Account.Id))." -InformationAction Continue
-
 Write-Verbose "Checking for Management Subscription Aliases."
 $subscriptionAliasesManagement = [PSCustomObject]@{}
 for ($i = 1; $i -lt (($terraformVersionsCount * 2) + 1); $i++) {
@@ -149,9 +127,7 @@ $matrixObject = [PSCustomObject]@{}
 for ($i = 0; $i -lt $terraformVersionsCount; $i++) {
   $terraformVersion = $terraformVersions[$i]
   $jobId1 = ($i * 2) + 1
-  $jobId2 = ($i * 2) + 2
   $jobName1 = "$jobId1. (TF: $terraformVersion, AZ: $azurermProviderVersionBase)"
-  $jobName2 = "$jobId2. (TF: $terraformVersion, AZ: $azurermProviderVersionLatest)"
   $matrixObject | Add-Member `
     -NotePropertyName $jobName1 `
     -NotePropertyValue @{
@@ -163,17 +139,6 @@ for ($i = 0; $i -lt $terraformVersionsCount; $i++) {
     TF_SUBSCRIPTION_ID_CONNECTIVITY = ($subscriptionAliasesConnectivity."csu-tf-connectivity-$jobId1")
   }
   Write-Information " Added job to matrix ($($jobName1))." -InformationAction Continue
-  $matrixObject | Add-Member `
-    -NotePropertyName $jobName2 `
-    -NotePropertyValue @{
-    TF_ROOT_ID                      = Get-RandomId
-    TF_VERSION                      = $terraformVersion
-    TF_AZ_VERSION                   = $azurermProviderVersionLatest
-    TF_JOB_ID                       = $jobId2
-    TF_SUBSCRIPTION_ID_MANAGEMENT   = ($subscriptionAliasesManagement."csu-tf-management-$jobId2")
-    TF_SUBSCRIPTION_ID_CONNECTIVITY = ($subscriptionAliasesConnectivity."csu-tf-connectivity-$jobId2")
-  }
-  Write-Information " Added job to matrix ($($jobName2))." -InformationAction Continue
 }
 
 # Convert PSCustomObject to JSON.

tests/scripts/tf-destroy.sh

@@ -24,14 +24,6 @@ status=$?
 
 if [ $status -ne 0 ]; then
 
-    echo "==> Authenticating cli..."
-    az login \
-        --service-principal \
-        --tenant "$ARM_TENANT_ID" \
-        --username "$ARM_CLIENT_ID" \
-        --password "$ARM_CLIENT_SECRET" \
-        --query [?isDefault]
-
     IFS=$'\n'
 
     TF_ROOT_ID=("$TF_ROOT_ID")

tests/scripts/tf-init.sh

@@ -29,6 +29,9 @@ terraform {
     storage_account_name = "$STORAGE_ACCOUNT_NAME"
     container_name       = "$STORAGE_CONTAINER_NAME"
     key                  = "terraform-$TF_ROOT_ID.tfstate"
+    use_azuread_auth     = true
+    use_oidc             = true
+    client_id            = "$ARM_CERTIFICATE_CLIENT_ID"
   }
 }
 TFCONFIG
@@ -37,30 +40,31 @@ echo "==> Creating providers_override.tf with subscription configuration and cre
 cat >providers_override.tf <<TFCONFIG
 provider "azurerm" {
   features {}
+
+  use_oidc        = true
+  tenant_id       = "$ARM_TENANT_ID"
+  client_id       = "$ARM_CERTIFICATE_CLIENT_ID"
+  subscription_id = "$TF_SUBSCRIPTION_ID_MANAGEMENT"
 }
 
 provider "azurerm" {
   features {}
 
-  alias                       = "connectivity"
-  subscription_id             = "$TF_SUBSCRIPTION_ID_CONNECTIVITY"
-  client_id                   = "$ARM_CERTIFICATE_CLIENT_ID"
-  client_certificate_path     = "$ARM_CERTIFICATE_PATH"
-  client_certificate_password = "$ARM_CERTIFICATE_PASSWORD"
-  tenant_id                   = "$ARM_TENANT_ID"
+  alias           = "connectivity"
+  subscription_id = "$TF_SUBSCRIPTION_ID_CONNECTIVITY"
+  client_id       = "$ARM_CERTIFICATE_CLIENT_ID"
+  use_oidc        = true
 }
 
 provider "azurerm" {
   features {}
 
-  alias                       = "management"
-  subscription_id             = "$TF_SUBSCRIPTION_ID_MANAGEMENT"
-  client_id                   = "$ARM_CERTIFICATE_CLIENT_ID"
-  client_certificate_path     = "$ARM_CERTIFICATE_PATH"
-  client_certificate_password = "$ARM_CERTIFICATE_PASSWORD"
-  tenant_id                   = "$ARM_TENANT_ID"
+  alias           = "management"
+  subscription_id = "$TF_SUBSCRIPTION_ID_MANAGEMENT"
+  client_id       = "$ARM_CERTIFICATE_CLIENT_ID"
+  use_oidc        = true
 }
 TFCONFIG
 
-echo "==> Initializaing Terraform workspace..."
+echo "==> Initializing Terraform workspace..."
 terraform init