Use a static storage account for OIDC issuer

Re-enables KIND-based integration tests.

Closes Azure#4166.

.github/workflows/pre-release-tests.yaml

@@ -36,7 +36,7 @@ jobs:
           echo "container_id=$container_id" >> $GITHUB_ENV
 
       - name: Run Pre Release Tests
-        run: docker exec -e HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task controller:test-upgrade
+        run: docker exec -e HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID -e KIND_OIDC_STORAGE_ACCOUNT_RG -e KIND_OIDC_STORAGE_ACCOUNT "${{ env.container_id }}" task controller:test-upgrade
         env:
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

Taskfile.yml

@@ -274,10 +274,8 @@ tasks:
   controller:ci-integration-tests:
     cmds:
       - task: cleanup-old-live-azure-resources
-      # TODO: These two test suites have been commented out temporarily
-      # TODO: due to https://github.com/Azure/azure-service-operator/issues/4166
-      # - task: controller:test-integration-kind-ci
-      # - task: controller:test-multitenant-integration-kind-ci
+      - task: controller:test-integration-kind-ci
+      - task: controller:test-multitenant-integration-kind-ci
       - task: cleanup-old-live-azure-resources  # Run this on both sides of the test pass to do our best to avoid leaking something
 
   controller:ci-live:
@@ -963,9 +961,10 @@ tasks:
     desc: Creates a workload identity secret for testing purposes
     cmds:
       - "{{.SCRIPTS_ROOT}}/deploy-testing-secret.sh workloadidentity"
+    vars:
+      DIR: "{{.DIR | default .KIND_WORKLOAD_IDENTITY_PATH}}"
     env:
       # Override the AZURE_CLIENT_ID env variable here
-      DIR: "{{.DIR | default .KIND_WORKLOAD_IDENTITY_PATH}}"
       AZURE_MI_CLIENT_ID:  # TODO: Ideally would override AZURE_CLIENT_ID here but we can't because of https://github.com/go-task/task/issues/1038
         sh: "cat {{.DIR}}/azure/miclientid.txt"
 
@@ -1278,3 +1277,13 @@ tasks:
     dir: v2/tools/mangle-test-json
     cmds:
       - go build
+
+  provision-oidc-storage-acct:
+    desc: Creates a storage account for use in OIDC federation with a random name in resource group "aso-wi-storage"
+    dir: "v2/"
+    cmds:
+      - '{{.SCRIPTS_ROOT}}/workloadidentitystorage/deploy.sh'
+    env:
+      KIND_OIDC_STORAGE_ACCOUNT_RG: aso-wi-storage
+      KIND_OIDC_STORAGE_ACCOUNT:
+          sh: echo "asowi$(openssl rand -hex 6)"

docs/hugo/content/contributing/add-a-new-code-generated-resource.md

@@ -297,7 +297,7 @@ More information on the naming convention can be found in that folders [README](
 
 ### Record the test passing
 
-See [the code generator test README](../testing/#running-integration-tests) for how to run tests and record their HTTP interactions to allow replay.
+See [the code generator test README](../testing/#running-envtest-integration-tests) for how to run tests and record their HTTP interactions to allow replay.
 
 ## Add a new sample
 

docs/hugo/content/contributing/testing.md

@@ -2,9 +2,12 @@
 title: Testing
 ---
 
-## Running integration tests
+## Running envtest integration tests
 
-Basic use: run `task controller:test-integration-envtest`.
+**Basic use:** `task controller:test-integration-envtest`.
+
+These are sometimes also called `envtest` tests, because they use 
+[envtest](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest).
 
 ### Required variables
 
@@ -64,3 +67,38 @@ or, with a timeout:
 ```bash
 TIMEOUT=10m TEST_FILTER=<test_name_regex> task controller:test-integration-envtest
 ```
+
+## Running integration tests in a KIND cluster
+
+| Environment Variable         | Value                                                                          |
+|------------------------------|--------------------------------------------------------------------------------|
+| AZURE_SUBSCRIPTION_ID        | The Azure Subscription ID                                                      |
+| AZURE_TENANT_ID              | The Azure Tenant ID                                                            |
+| KIND_OIDC_STORAGE_ACCOUNT_RG | The resource group containing the Azure storage account for OIDC federation    |
+| KIND_OIDC_STORAGE_ACCOUNT    | The Azure storage account name. Storage account names must be globally unique. |
+
+ASO CI uses:
+```
+KIND_OIDC_STORAGE_ACCOUNT_RG=asov2-ci
+KIND_OIDC_STORAGE_ACCOUNT=asowistorage
+```
+
+**Pre-requisite:** Provision a storage account for OIDC federated token exchange with Azure for your KIND clusters.
+This can be done by running `task provision-oidc-storage-acct`, which will create a storage account with a random
+name in the `aso-wi-storage` resource group.
+
+Note: If you want to run the tests in a different subscription than the one containing the OIDC storage account,
+set the `KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION` variable to the subscription ID of the storage account, after
+which you can run tests in a different subscription if you like.
+
+**Basic use (create KIND + run tests):** `task controller:test-integration-kind`
+
+**Basic use (create KIND cluster only):** `task controller:kind-create-helm-workload-identity`
+
+There are a _few_ tests that can't be run in envtest, because envtest doesn't offer a full Kubernetes environment.
+For these tests we have a separate suite of tests that should be run in a real Kubernetes environment, located at
+[v2/test](https://github.com/Azure/azure-service-operator/tree/main/v2/test).
+
+It's easiest, fastest, and cheapest to run these tests in a KIND cluster, although they can also be run in
+a full Kubernetes cluster (such as an AKS cluster). It can sometimes be useful to have a KIND cluster for local
+testing as well.

scripts/v2/create-kind-wi-storage.sh

@@ -7,7 +7,6 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-
 print_usage() {
   echo "Usage: create-kind-wi-storage.sh -d <DIRECTORY> -p <PREFIX>"
 }
@@ -29,35 +28,48 @@ if [[ -z "$DIR" ]] || [[ -z "$PREFIX" ]]; then
   exit 1
 fi
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source ${SCRIPT_DIR}/workloadidentitystorage/vars.sh
+
 # Generate names and save in files
 mkdir -p "${DIR}/azure"
+
+# Note that this resource group isn't used by this script anymore but is still used by make-mi-fic.py
+# to produce a new MI + FIC.
 RESOURCE_GROUP="${PREFIX}-rg-wi$(openssl rand -hex 6)"
-AZURE_STORAGE_ACCOUNT="asowi$(openssl rand -hex 6)"
-# This $web container is a special container that serves static web content without requiring public access enablement.
-# See https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blob-static-website
-AZURE_STORAGE_CONTAINER="\$web"
 
 # If somehow the files already exist then the resource also already exists and we shouldn't do anything
-if [ -f "$DIR/azure/rg.txt" ]; then
+if [ -f "$DIR/azure/oidcid.txt" ]; then
   # Nothing to do, no existing rg
+  echo "Using existing OIDC key $(cat ${DIR}/azure/oidcid.txt)"
+  exit 0
+fi
+
+if [ -f "$DIR/azure/rg.txt" ]; then
+   # Nothing to do, no existing rg
   echo "Using existing RG $(cat ${DIR}/azure/rg.txt)"
   exit 0
 fi
 
+# This is just an identifier to differentiate different blobs from one another so that we can support
+# multiple OIDC issuers in a single storage account. It's not actually involved in the authentication flow anywhere
+# it's purely ASO bookkeeping.
+OIDC_IDENTIFIER="asowi$(openssl rand -hex 6)"
+echo ${OIDC_IDENTIFIER} > "${DIR}/azure/oidcid.txt"
+
+# Save the RG we're using too
 echo ${RESOURCE_GROUP} > "${DIR}/azure/rg.txt"
 
 # Generate the OIDC keys
 openssl genrsa -out "$DIR/sa.key" 2048
 openssl rsa -in "$DIR/sa.key" -pubout -out "$DIR/sa.pub"
 
-az group create -l westus -n "${RESOURCE_GROUP}" --tags "CreatedAt=$(date --utc +"%Y-%m-%dT%H:%M:%SZ")"
-
-az storage account create --resource-group "${RESOURCE_GROUP}" --name "${AZURE_STORAGE_ACCOUNT}" --allow-blob-public-access false
-# Enable static website serving
-az storage blob service-properties update --account-name "${AZURE_STORAGE_ACCOUNT}" --static-website
-az storage container create --account-name "${AZURE_STORAGE_ACCOUNT}" --name "${AZURE_STORAGE_CONTAINER}"
+if [ -z "${KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION-}" ]; then
+  KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION=$(az account show --output tsv --query id)
+fi
 
-ISSUER_URL=$(az storage account show --name "${AZURE_STORAGE_ACCOUNT}" -o json | jq -r .primaryEndpoints.web)
+# There's already a trailing / so we don't need to add one between the web endpoint and the OIDC Identifier
+ISSUER_URL="$(az storage account show --subscription ${KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION} --name "${KIND_OIDC_STORAGE_ACCOUNT}" -o json | jq -r .primaryEndpoints.web)${OIDC_IDENTIFIER}"
 echo "${ISSUER_URL}" > "${DIR}/azure/saissuer.txt"
 
 cat <<EOF > "${DIR}/openid-configuration.json"
@@ -76,16 +88,24 @@ cat <<EOF > "${DIR}/openid-configuration.json"
 }
 EOF
 
+CREATION_TIME="$(date --utc +"%Y-%m-%dT%H:%M:%SZ")"
+
+az group create -l westus -n "${RESOURCE_GROUP}" --tags "CreatedAt=${CREATION_TIME}"
+
 az storage blob upload \
-  --account-name "${AZURE_STORAGE_ACCOUNT}" \
-  --container-name "${AZURE_STORAGE_CONTAINER}" \
+  --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" \
+  --container-name "${KIND_OIDC_STORAGE_CONTAINER}" \
   --file "${DIR}/openid-configuration.json" \
-  --name .well-known/openid-configuration
+  --tags "CreatedAt=${CREATION_TIME}" \
+  --name "${OIDC_IDENTIFIER}/.well-known/openid-configuration" \
+  --auth-mode login
 
 azwi jwks --public-keys "${DIR}/sa.pub" --output-file "${DIR}/jwks.json"
 
 az storage blob upload \
-  --account-name "${AZURE_STORAGE_ACCOUNT}" \
-  --container-name "${AZURE_STORAGE_CONTAINER}" \
+  --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" \
+  --container-name "${KIND_OIDC_STORAGE_CONTAINER}" \
   --file "${DIR}/jwks.json" \
-  --name openid/v1/jwks
+  --tags "CreatedAt=${CREATION_TIME}" \
+  --name "${OIDC_IDENTIFIER}/openid/v1/jwks" \
+  --auth-mode login

scripts/v2/delete-kind-wi-storage.sh

@@ -28,19 +28,8 @@ if [[ -z "$DIR" ]]; then
   exit 1
 fi
 
-EXISTS=$(kind get clusters -q | grep "^asov2-wi$" || true)
-
-if [ -f "$DIR/azure/rg.txt" ]; then
-  RESOURCE_GROUP=$(cat $DIR/azure/rg.txt)
-else
-  # Nothing to do, no existing rg
-  exit 0
-fi
-
-if [[ -z "$EXISTS" ]]; then
-  # Nothing to do, no match
-  exit 0
-fi
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source ${SCRIPT_DIR}/workloadidentitystorage/vars.sh
 
 if [ -f "$DIR/azure/fic.txt" ]; then
   # Need to manually delete the identity FIC
@@ -57,10 +46,25 @@ if [ -f "$DIR/azure/roleassignmentid.txt" ]; then
   az role assignment delete --ids "${ROLE_ASSIGNMENT_ID}"
 fi
 
-if [ $(az group exists --name ${RESOURCE_GROUP}) = true ]; then
-  echo "Deleting resourceGroup: ${RESOURCE_GROUP}"
-  az group delete --name ${RESOURCE_GROUP} -y
-  echo "Done deleting resourceGroup: ${RESOURCE_GROUP}"
+if [ -f "$DIR/azure/oidcid.txt" ]; then
+  OIDC_IDENTIFIER=$(cat $DIR/azure/oidcid.txt)
+  echo "Deleting blobs associated with OIDC ID: ${OIDC_IDENTIFIER}"
+  az storage blob delete-batch \
+    --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" \
+    --source "${KIND_OIDC_STORAGE_CONTAINER}" \
+    --pattern "${OIDC_IDENTIFIER}/*" \
+    --auth-mode login
+  echo "Done deleting blobs associated with OIDC ID: ${OIDC_IDENTIFIER}"
+fi
+
+if [ -f "$DIR/azure/rg.txt" ]; then
+  RESOURCE_GROUP=$(cat $DIR/azure/rg.txt)
+
+  if [ $(az group exists --name ${RESOURCE_GROUP}) = true ]; then
+    echo "Deleting resourceGroup: ${RESOURCE_GROUP}"
+    az group delete --name ${RESOURCE_GROUP} -y
+    echo "Done deleting resourceGroup: ${RESOURCE_GROUP}"
+  fi
 fi
 
 echo "Deleting directory ${DIR}"

scripts/v2/workloadidentitystorage/deploy.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+print_usage() {
+  echo "Usage: deploy.sh [-r]"
+  echo "    -r: If the ASO CI subscription should have role assigned to it. Only should be used in ASO CI contexts."
+  echo "Note that this deploys to the currently selected az cli subscription"
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source ${SCRIPT_DIR}/vars.sh
+
+ASSIGN_CI_ROLE=0
+
+while getopts 'r' flag; do
+  case "${flag}" in
+    r) ASSIGN_CI_ROLE=1 ;;
+    *) print_usage
+       exit 1 ;;
+  esac
+done
+
+if [ -z "${KIND_OIDC_STORAGE_ACCOUNT_RG}" -o -z "${KIND_OIDC_STORAGE_ACCOUNT}" ]; then
+  echo "Missing required environment variables KIND_OIDC_STORAGE_ACCOUNT_RG and KIND_OIDC_STORAGE_ACCOUNT"
+fi
+
+az group create --name "${KIND_OIDC_STORAGE_ACCOUNT_RG}" --location westus
+
+az storage account create --resource-group "${KIND_OIDC_STORAGE_ACCOUNT_RG}" --name "${KIND_OIDC_STORAGE_ACCOUNT}" \
+  --allow-blob-public-access false \
+  --allow-shared-key-access false
+
+# Enable static website serving
+az storage blob service-properties update --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" --static-website --auth-mode login
+az storage container create --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" --name "${KIND_OIDC_STORAGE_CONTAINER}" --auth-mode login
+
+# Individual identity assignments not managed here but instead managed through the portal
+
+if [[ "$ASSIGN_CI_ROLE" -eq 1 ]]; then
+  # Assign permissions to the CI identity
+  AZURE_SUBSCRIPTION_ID=$(az account show --output tsv --query id)
+
+  IDENTITY_OBJECT_ID=$(az identity show -g 1es-pool -n aso-ci-identity --query principalId -otsv)
+  az role assignment create \
+    --assignee-object-id ${IDENTITY_OBJECT_ID} \
+    --assignee-principal-type "ServicePrincipal" \
+    --role "Storage Blob Data Contributor" \
+    --scope /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${KIND_OIDC_STORAGE_ACCOUNT_RG}/providers/Microsoft.Storage/storageAccounts/${KIND_OIDC_STORAGE_ACCOUNT}
+fi
+
+echo "Set the following environment variables to use this storage account"
+echo "export KIND_OIDC_STORAGE_ACCOUNT_RG=${KIND_OIDC_STORAGE_ACCOUNT_RG}"
+echo "export KIND_OIDC_STORAGE_ACCOUNT=${KIND_OIDC_STORAGE_ACCOUNT}"

scripts/v2/workloadidentitystorage/vars.sh

@@ -0,0 +1,3 @@
+# This $web container is a special container that serves static web content without requiring public access enablement.
+# See https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blob-static-website
+KIND_OIDC_STORAGE_CONTAINER="\$web"