diff --git a/.github/workflows/pre-release-tests.yaml b/.github/workflows/pre-release-tests.yaml index de88cacc7fa..37762f85d8b 100644 --- a/.github/workflows/pre-release-tests.yaml +++ b/.github/workflows/pre-release-tests.yaml @@ -36,7 +36,7 @@ jobs: echo "container_id=$container_id" >> $GITHUB_ENV - name: Run Pre Release Tests - run: docker exec -e HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task controller:test-upgrade + run: docker exec -e HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID -e KIND_OIDC_STORAGE_ACCOUNT_RG -e KIND_OIDC_STORAGE_ACCOUNT "${{ env.container_id }}" task controller:test-upgrade env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/Taskfile.yml b/Taskfile.yml index 32f0978f415..d03c709b49a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -274,10 +274,8 @@ tasks: controller:ci-integration-tests: cmds: - task: cleanup-old-live-azure-resources - # TODO: These two test suites have been commented out temporarily - # TODO: due to https://github.com/Azure/azure-service-operator/issues/4166 - # - task: controller:test-integration-kind-ci - # - task: controller:test-multitenant-integration-kind-ci + - task: controller:test-integration-kind-ci + - task: controller:test-multitenant-integration-kind-ci - task: cleanup-old-live-azure-resources # Run this on both sides of the test pass to do our best to avoid leaking something controller:ci-live: @@ -963,9 +961,10 @@ tasks: desc: Creates a workload identity secret for testing purposes cmds: - "{{.SCRIPTS_ROOT}}/deploy-testing-secret.sh workloadidentity" + vars: + DIR: "{{.DIR | default .KIND_WORKLOAD_IDENTITY_PATH}}" env: # Override the AZURE_CLIENT_ID env variable here - DIR: "{{.DIR | default .KIND_WORKLOAD_IDENTITY_PATH}}" AZURE_MI_CLIENT_ID: # TODO: Ideally would override AZURE_CLIENT_ID here but we can't because of https://github.com/go-task/task/issues/1038 sh: "cat {{.DIR}}/azure/miclientid.txt" @@ -1278,3 +1277,13 @@ tasks: dir: v2/tools/mangle-test-json cmds: - go build + + provision-oidc-storage-acct: + desc: Creates a storage account for use in OIDC federation with a random name in resource group "aso-wi-storage" + dir: "v2/" + cmds: + - '{{.SCRIPTS_ROOT}}/workloadidentitystorage/deploy.sh' + env: + KIND_OIDC_STORAGE_ACCOUNT_RG: aso-wi-storage + KIND_OIDC_STORAGE_ACCOUNT: + sh: echo "asowi$(openssl rand -hex 6)" diff --git a/docs/hugo/content/contributing/add-a-new-code-generated-resource.md b/docs/hugo/content/contributing/add-a-new-code-generated-resource.md index b789a81dfcb..4af2020f9e7 100644 --- a/docs/hugo/content/contributing/add-a-new-code-generated-resource.md +++ b/docs/hugo/content/contributing/add-a-new-code-generated-resource.md @@ -297,7 +297,7 @@ More information on the naming convention can be found in that folders [README]( ### Record the test passing -See [the code generator test README](../testing/#running-integration-tests) for how to run tests and record their HTTP interactions to allow replay. +See [the code generator test README](../testing/#running-envtest-integration-tests) for how to run tests and record their HTTP interactions to allow replay. ## Add a new sample diff --git a/docs/hugo/content/contributing/testing.md b/docs/hugo/content/contributing/testing.md index 1cd1275b4b3..f68df7e8c22 100644 --- a/docs/hugo/content/contributing/testing.md +++ b/docs/hugo/content/contributing/testing.md @@ -2,9 +2,12 @@ title: Testing --- -## Running integration tests +## Running envtest integration tests -Basic use: run `task controller:test-integration-envtest`. +**Basic use:** `task controller:test-integration-envtest`. + +These are sometimes also called `envtest` tests, because they use +[envtest](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest). ### Required variables @@ -64,3 +67,38 @@ or, with a timeout: ```bash TIMEOUT=10m TEST_FILTER= task controller:test-integration-envtest ``` + +## Running integration tests in a KIND cluster + +| Environment Variable | Value | +|------------------------------|--------------------------------------------------------------------------------| +| AZURE_SUBSCRIPTION_ID | The Azure Subscription ID | +| AZURE_TENANT_ID | The Azure Tenant ID | +| KIND_OIDC_STORAGE_ACCOUNT_RG | The resource group containing the Azure storage account for OIDC federation | +| KIND_OIDC_STORAGE_ACCOUNT | The Azure storage account name. Storage account names must be globally unique. | + +ASO CI uses: +``` +KIND_OIDC_STORAGE_ACCOUNT_RG=asov2-ci +KIND_OIDC_STORAGE_ACCOUNT=asowistorage +``` + +**Pre-requisite:** Provision a storage account for OIDC federated token exchange with Azure for your KIND clusters. +This can be done by running `task provision-oidc-storage-acct`, which will create a storage account with a random +name in the `aso-wi-storage` resource group. + +Note: If you want to run the tests in a different subscription than the one containing the OIDC storage account, +set the `KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION` variable to the subscription ID of the storage account, after +which you can run tests in a different subscription if you like. + +**Basic use (create KIND + run tests):** `task controller:test-integration-kind` + +**Basic use (create KIND cluster only):** `task controller:kind-create-helm-workload-identity` + +There are a _few_ tests that can't be run in envtest, because envtest doesn't offer a full Kubernetes environment. +For these tests we have a separate suite of tests that should be run in a real Kubernetes environment, located at +[v2/test](https://github.com/Azure/azure-service-operator/tree/main/v2/test). + +It's easiest, fastest, and cheapest to run these tests in a KIND cluster, although they can also be run in +a full Kubernetes cluster (such as an AKS cluster). It can sometimes be useful to have a KIND cluster for local +testing as well. diff --git a/scripts/v2/create-kind-wi-storage.sh b/scripts/v2/create-kind-wi-storage.sh index d5c071fde4a..1fb6260fbbd 100755 --- a/scripts/v2/create-kind-wi-storage.sh +++ b/scripts/v2/create-kind-wi-storage.sh @@ -7,7 +7,6 @@ set -o errexit set -o nounset set -o pipefail - print_usage() { echo "Usage: create-kind-wi-storage.sh -d -p " } @@ -29,35 +28,48 @@ if [[ -z "$DIR" ]] || [[ -z "$PREFIX" ]]; then exit 1 fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source ${SCRIPT_DIR}/workloadidentitystorage/vars.sh + # Generate names and save in files mkdir -p "${DIR}/azure" + +# Note that this resource group isn't used by this script anymore but is still used by make-mi-fic.py +# to produce a new MI + FIC. RESOURCE_GROUP="${PREFIX}-rg-wi$(openssl rand -hex 6)" -AZURE_STORAGE_ACCOUNT="asowi$(openssl rand -hex 6)" -# This $web container is a special container that serves static web content without requiring public access enablement. -# See https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blob-static-website -AZURE_STORAGE_CONTAINER="\$web" # If somehow the files already exist then the resource also already exists and we shouldn't do anything -if [ -f "$DIR/azure/rg.txt" ]; then +if [ -f "$DIR/azure/oidcid.txt" ]; then # Nothing to do, no existing rg + echo "Using existing OIDC key $(cat ${DIR}/azure/oidcid.txt)" + exit 0 +fi + +if [ -f "$DIR/azure/rg.txt" ]; then + # Nothing to do, no existing rg echo "Using existing RG $(cat ${DIR}/azure/rg.txt)" exit 0 fi +# This is just an identifier to differentiate different blobs from one another so that we can support +# multiple OIDC issuers in a single storage account. It's not actually involved in the authentication flow anywhere +# it's purely ASO bookkeeping. +OIDC_IDENTIFIER="asowi$(openssl rand -hex 6)" +echo ${OIDC_IDENTIFIER} > "${DIR}/azure/oidcid.txt" + +# Save the RG we're using too echo ${RESOURCE_GROUP} > "${DIR}/azure/rg.txt" # Generate the OIDC keys openssl genrsa -out "$DIR/sa.key" 2048 openssl rsa -in "$DIR/sa.key" -pubout -out "$DIR/sa.pub" -az group create -l westus -n "${RESOURCE_GROUP}" --tags "CreatedAt=$(date --utc +"%Y-%m-%dT%H:%M:%SZ")" - -az storage account create --resource-group "${RESOURCE_GROUP}" --name "${AZURE_STORAGE_ACCOUNT}" --allow-blob-public-access false -# Enable static website serving -az storage blob service-properties update --account-name "${AZURE_STORAGE_ACCOUNT}" --static-website -az storage container create --account-name "${AZURE_STORAGE_ACCOUNT}" --name "${AZURE_STORAGE_CONTAINER}" +if [ -z "${KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION-}" ]; then + KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION=$(az account show --output tsv --query id) +fi -ISSUER_URL=$(az storage account show --name "${AZURE_STORAGE_ACCOUNT}" -o json | jq -r .primaryEndpoints.web) +# There's already a trailing / so we don't need to add one between the web endpoint and the OIDC Identifier +ISSUER_URL="$(az storage account show --subscription ${KIND_OIDC_STORAGE_ACCOUNT_SUBSCRIPTION} --name "${KIND_OIDC_STORAGE_ACCOUNT}" -o json | jq -r .primaryEndpoints.web)${OIDC_IDENTIFIER}" echo "${ISSUER_URL}" > "${DIR}/azure/saissuer.txt" cat < "${DIR}/openid-configuration.json" @@ -76,16 +88,24 @@ cat < "${DIR}/openid-configuration.json" } EOF +CREATION_TIME="$(date --utc +"%Y-%m-%dT%H:%M:%SZ")" + +az group create -l westus -n "${RESOURCE_GROUP}" --tags "CreatedAt=${CREATION_TIME}" + az storage blob upload \ - --account-name "${AZURE_STORAGE_ACCOUNT}" \ - --container-name "${AZURE_STORAGE_CONTAINER}" \ + --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" \ + --container-name "${KIND_OIDC_STORAGE_CONTAINER}" \ --file "${DIR}/openid-configuration.json" \ - --name .well-known/openid-configuration + --tags "CreatedAt=${CREATION_TIME}" \ + --name "${OIDC_IDENTIFIER}/.well-known/openid-configuration" \ + --auth-mode login azwi jwks --public-keys "${DIR}/sa.pub" --output-file "${DIR}/jwks.json" az storage blob upload \ - --account-name "${AZURE_STORAGE_ACCOUNT}" \ - --container-name "${AZURE_STORAGE_CONTAINER}" \ + --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" \ + --container-name "${KIND_OIDC_STORAGE_CONTAINER}" \ --file "${DIR}/jwks.json" \ - --name openid/v1/jwks + --tags "CreatedAt=${CREATION_TIME}" \ + --name "${OIDC_IDENTIFIER}/openid/v1/jwks" \ + --auth-mode login diff --git a/scripts/v2/delete-kind-wi-storage.sh b/scripts/v2/delete-kind-wi-storage.sh index 82a55c9e8b6..07358e7d203 100755 --- a/scripts/v2/delete-kind-wi-storage.sh +++ b/scripts/v2/delete-kind-wi-storage.sh @@ -28,19 +28,8 @@ if [[ -z "$DIR" ]]; then exit 1 fi -EXISTS=$(kind get clusters -q | grep "^asov2-wi$" || true) - -if [ -f "$DIR/azure/rg.txt" ]; then - RESOURCE_GROUP=$(cat $DIR/azure/rg.txt) -else - # Nothing to do, no existing rg - exit 0 -fi - -if [[ -z "$EXISTS" ]]; then - # Nothing to do, no match - exit 0 -fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source ${SCRIPT_DIR}/workloadidentitystorage/vars.sh if [ -f "$DIR/azure/fic.txt" ]; then # Need to manually delete the identity FIC @@ -57,10 +46,25 @@ if [ -f "$DIR/azure/roleassignmentid.txt" ]; then az role assignment delete --ids "${ROLE_ASSIGNMENT_ID}" fi -if [ $(az group exists --name ${RESOURCE_GROUP}) = true ]; then - echo "Deleting resourceGroup: ${RESOURCE_GROUP}" - az group delete --name ${RESOURCE_GROUP} -y - echo "Done deleting resourceGroup: ${RESOURCE_GROUP}" +if [ -f "$DIR/azure/oidcid.txt" ]; then + OIDC_IDENTIFIER=$(cat $DIR/azure/oidcid.txt) + echo "Deleting blobs associated with OIDC ID: ${OIDC_IDENTIFIER}" + az storage blob delete-batch \ + --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" \ + --source "${KIND_OIDC_STORAGE_CONTAINER}" \ + --pattern "${OIDC_IDENTIFIER}/*" \ + --auth-mode login + echo "Done deleting blobs associated with OIDC ID: ${OIDC_IDENTIFIER}" +fi + +if [ -f "$DIR/azure/rg.txt" ]; then + RESOURCE_GROUP=$(cat $DIR/azure/rg.txt) + + if [ $(az group exists --name ${RESOURCE_GROUP}) = true ]; then + echo "Deleting resourceGroup: ${RESOURCE_GROUP}" + az group delete --name ${RESOURCE_GROUP} -y + echo "Done deleting resourceGroup: ${RESOURCE_GROUP}" + fi fi echo "Deleting directory ${DIR}" diff --git a/scripts/v2/workloadidentitystorage/deploy.sh b/scripts/v2/workloadidentitystorage/deploy.sh new file mode 100755 index 00000000000..7bb279a6f07 --- /dev/null +++ b/scripts/v2/workloadidentitystorage/deploy.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +print_usage() { + echo "Usage: deploy.sh [-r]" + echo " -r: If the ASO CI subscription should have role assigned to it. Only should be used in ASO CI contexts." + echo "Note that this deploys to the currently selected az cli subscription" +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source ${SCRIPT_DIR}/vars.sh + +ASSIGN_CI_ROLE=0 + +while getopts 'r' flag; do + case "${flag}" in + r) ASSIGN_CI_ROLE=1 ;; + *) print_usage + exit 1 ;; + esac +done + +if [ -z "${KIND_OIDC_STORAGE_ACCOUNT_RG}" -o -z "${KIND_OIDC_STORAGE_ACCOUNT}" ]; then + echo "Missing required environment variables KIND_OIDC_STORAGE_ACCOUNT_RG and KIND_OIDC_STORAGE_ACCOUNT" +fi + +az group create --name "${KIND_OIDC_STORAGE_ACCOUNT_RG}" --location westus + +az storage account create --resource-group "${KIND_OIDC_STORAGE_ACCOUNT_RG}" --name "${KIND_OIDC_STORAGE_ACCOUNT}" \ + --allow-blob-public-access false \ + --allow-shared-key-access false + +# Enable static website serving +az storage blob service-properties update --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" --static-website --auth-mode login +az storage container create --account-name "${KIND_OIDC_STORAGE_ACCOUNT}" --name "${KIND_OIDC_STORAGE_CONTAINER}" --auth-mode login + +# Individual identity assignments not managed here but instead managed through the portal + +if [[ "$ASSIGN_CI_ROLE" -eq 1 ]]; then + # Assign permissions to the CI identity + AZURE_SUBSCRIPTION_ID=$(az account show --output tsv --query id) + + IDENTITY_OBJECT_ID=$(az identity show -g 1es-pool -n aso-ci-identity --query principalId -otsv) + az role assignment create \ + --assignee-object-id ${IDENTITY_OBJECT_ID} \ + --assignee-principal-type "ServicePrincipal" \ + --role "Storage Blob Data Contributor" \ + --scope /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${KIND_OIDC_STORAGE_ACCOUNT_RG}/providers/Microsoft.Storage/storageAccounts/${KIND_OIDC_STORAGE_ACCOUNT} +fi + +echo "Set the following environment variables to use this storage account" +echo "export KIND_OIDC_STORAGE_ACCOUNT_RG=${KIND_OIDC_STORAGE_ACCOUNT_RG}" +echo "export KIND_OIDC_STORAGE_ACCOUNT=${KIND_OIDC_STORAGE_ACCOUNT}" diff --git a/scripts/v2/workloadidentitystorage/vars.sh b/scripts/v2/workloadidentitystorage/vars.sh new file mode 100644 index 00000000000..82d0e85d229 --- /dev/null +++ b/scripts/v2/workloadidentitystorage/vars.sh @@ -0,0 +1,3 @@ +# This $web container is a special container that serves static web content without requiring public access enablement. +# See https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blob-static-website +KIND_OIDC_STORAGE_CONTAINER="\$web"