From 4bf10ee9236e895c231dd9ab06fd7ad49763344d Mon Sep 17 00:00:00 2001 From: Matthew Christopher Date: Wed, 15 May 2024 16:55:34 -0700 Subject: [PATCH] Update local tests and CI tests to use az login token or managed identity, not service principal (#4003) * Update Taskfile to use az cli token rather than SP - Update az login step to use an interactive login, rather than an SP. Interactive login is good for 12 hours. - Remove SP-based kind cluster helpers and replace usages with workload-identity based. - Refactor make-mi-fic.sh to Python. This is required to manage the more limited permissions that an interactive token has with repsect to calling 'az role assignment list'. * HTTP Recording tests should use CLI or Env credentials Tested to ensure that az login allows tests to be re-recorded * Add unit tests with mocked azidentity calls The idea is that this will replace actual live testing of the various authentication modes. We trust that, if we call the azidentity SDK with the correct parameters, it does the correct thing. * Fix issue with credential format documentation We documented the AUTH_MODE: "workloadidentity" for workload identity, but in actuality the code doesn't require this be set for WI on any secret but the global secret. * Remove use of MULTITENANT secret Now using a dynamically created UserAssignedIdentity instead of a hardcoded static multitenant secret. There are two advantages here: 1. Less setup to run a test on a new subscription because fewer inputs to the CI job. 2. No static secrets that can possibly be leaked or compromised. * Use MSI for CI instead of service principal - Updated cipool BICEP to provision a managed identity and give it the required roles. - Updated Taskfile to support MSI-based az login when running in the context of a GitHub action. - Removed all reference to AZURE_CLIENT_ID and AZURE_CLIENT_SECRET from GitHub workflows. * Update documentation to be more clear about authentication mode Be explicit about which mode we're using in the various installation instructions, and provide links to the detailed authentication documentation so that users are more easily exposed to other authentication options (formats and scopes). Also update devcontainer raw cmdline to map azcli token into container to reduce number of required az logins --- .../workflows/create-release-experimental.yml | 4 +- .github/workflows/create-release-official.yml | 8 +- .github/workflows/live-validation.yml | 12 +- .github/workflows/pr-validation-fork.yml | 8 +- .github/workflows/pr-validation.yml | 8 +- .github/workflows/pre-release-tests.yaml | 8 +- Taskfile.yml | 193 ++++++++++-------- docs/hugo/content/_index.md | 36 +++- .../contributing/create-a-new-release.md | 88 ++++---- .../content/contributing/developer-setup.md | 2 +- .../running-a-development-version.md | 10 +- docs/hugo/content/contributing/testing.md | 31 +-- .../guide/authentication/credential-format.md | 2 - .../content/guide/installing-from-yaml.md | 17 +- scripts/v2/cipool/deploy.bicep | 38 +++- scripts/v2/cipool/identity.bicep | 29 +++ scripts/v2/delete-kind-wi-storage.sh | 3 +- scripts/v2/delete-role-assignment.sh | 36 ++++ .../v2/deploy-multitenant-testing-secret.sh | 19 +- scripts/v2/deploy-testing-secret.sh | 2 +- scripts/v2/make-aks.sh | 16 +- scripts/v2/make-mi-fic.py | 160 +++++++++++++++ scripts/v2/make-mi-fic.sh | 92 --------- v2/cmd/controller/app/setup.go | 2 +- v2/internal/identity/credential_provider.go | 31 ++- .../identity/credential_provider_test.go | 146 ++++++++++++- .../identity/token_credential_provider.go | 39 ++++ .../token_credential_provider_test.go | 48 +++++ .../reconcilers/arm/arm_client_cache_test.go | 4 +- v2/internal/testcommon/creds/creds.go | 58 ++++-- .../testcommon/kube_test_context_envtest.go | 2 +- v2/test/single_operator_multitenant_test.go | 189 ++++++++++------- 32 files changed, 923 insertions(+), 418 deletions(-) create mode 100644 scripts/v2/cipool/identity.bicep create mode 100755 scripts/v2/delete-role-assignment.sh create mode 100755 scripts/v2/make-mi-fic.py delete mode 100755 scripts/v2/make-mi-fic.sh create mode 100644 v2/internal/identity/token_credential_provider.go create mode 100644 v2/internal/identity/token_credential_provider_test.go diff --git a/.github/workflows/create-release-experimental.yml b/.github/workflows/create-release-experimental.yml index cecf8b00dcc..d359a26c09f 100644 --- a/.github/workflows/create-release-experimental.yml +++ b/.github/workflows/create-release-experimental.yml @@ -95,11 +95,9 @@ jobs: - name: Docker login run: | container_id=${{env.container_id}} - docker exec -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e DOCKER_REGISTRY "$container_id" task VERSION=${{ env.ARTIFACT_VERSION }} LATEST_VERSION_TAG=${{ env.ARTIFACT_VERSION }} docker-login + docker exec -e DOCKER_REGISTRY "$container_id" task VERSION=${{ env.ARTIFACT_VERSION }} LATEST_VERSION_TAG=${{ env.ARTIFACT_VERSION }} docker-login env: DOCKER_REGISTRY: ${{ secrets.REGISTRY_LOGIN }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} - name: Build, tag and push docker image run: | diff --git a/.github/workflows/create-release-official.yml b/.github/workflows/create-release-official.yml index ec75d168de3..42d04d21ddf 100644 --- a/.github/workflows/create-release-official.yml +++ b/.github/workflows/create-release-official.yml @@ -52,11 +52,9 @@ jobs: - name: Docker login run: | container_id=${{env.container_id}} - docker exec -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e DOCKER_REGISTRY "$container_id" task docker-login + docker exec -e DOCKER_REGISTRY "$container_id" task docker-login env: DOCKER_REGISTRY: ${{ secrets.REGISTRY_LOGIN }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} - name: Build, tag and push docker image run: | @@ -68,11 +66,9 @@ jobs: - name: Protect image run: | container_id=${{env.container_id}} - docker exec -e DOCKER_PUSH_TARGET -e DOCKER_REGISTRY -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID "$container_id" task controller:acr-protect-image + docker exec -e GITHUB_ACTIONS -e DOCKER_PUSH_TARGET -e DOCKER_REGISTRY -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "$container_id" task controller:acr-protect-image env: DOCKER_PUSH_TARGET: ${{ secrets.REGISTRY_PUBLIC }} DOCKER_REGISTRY: ${{ secrets.REGISTRY_LOGIN }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/.github/workflows/live-validation.yml b/.github/workflows/live-validation.yml index d65f39b2164..4ca58b81fca 100644 --- a/.github/workflows/live-validation.yml +++ b/.github/workflows/live-validation.yml @@ -34,25 +34,17 @@ jobs: echo "container_id=$container_id" >> $GITHUB_ENV - name: Run CI tasks against live resources - run: docker exec --env HOSTROOT=$GITHUB_WORKSPACE -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_CLIENT_SECRET_MULTITENANT -e AZURE_CLIENT_ID_MULTITENANT -e AZURE_CLIENT_ID_CERT_AUTH -e AZURE_CLIENT_SECRET_CERT_AUTH "${{env.container_id }}" task ci-live + run: docker exec --env HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "${{env.container_id }}" task ci-live env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - AZURE_CLIENT_SECRET_MULTITENANT: ${{ secrets.AZURE_CLIENT_SECRET_MULTITENANT }} - AZURE_CLIENT_ID_MULTITENANT: ${{ secrets.AZURE_CLIENT_ID_MULTITENANT }} - AZURE_CLIENT_ID_CERT_AUTH: ${{ secrets.AZURE_CLIENT_ID_CERT_AUTH }} - AZURE_CLIENT_SECRET_CERT_AUTH: ${{ secrets.AZURE_CLIENT_SECRET_CERT_AUTH }} # We explicitly do not upload coverage for live tests as it messes with the diffs for PRs, # since they will not exercize the authorization codepaths. - name: Cleanup test resources if: always() - run: docker exec -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task cleanup-azure-resources + run: docker exec -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task cleanup-azure-resources env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/.github/workflows/pr-validation-fork.yml b/.github/workflows/pr-validation-fork.yml index 560a64245ae..a1256ea481c 100644 --- a/.github/workflows/pr-validation-fork.yml +++ b/.github/workflows/pr-validation-fork.yml @@ -107,16 +107,10 @@ jobs: - name: Run integration tests run: | container_id=${{ env.container_id }} - docker exec --env HOSTROOT=$GITHUB_WORKSPACE -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_CLIENT_SECRET_MULTITENANT -e AZURE_CLIENT_ID_MULTITENANT -e AZURE_CLIENT_ID_CERT_AUTH -e AZURE_CLIENT_SECRET_CERT_AUTH "$container_id" task controller:ci-integration-tests + docker exec --env HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "$container_id" task controller:ci-integration-tests env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - AZURE_CLIENT_SECRET_MULTITENANT: ${{ secrets.AZURE_CLIENT_SECRET_MULTITENANT }} - AZURE_CLIENT_ID_MULTITENANT: ${{ secrets.AZURE_CLIENT_ID_MULTITENANT }} - AZURE_CLIENT_ID_CERT_AUTH: ${{ secrets.AZURE_CLIENT_ID_CERT_AUTH }} - AZURE_CLIENT_SECRET_CERT_AUTH: ${{ secrets.AZURE_CLIENT_SECRET_CERT_AUTH }} if: steps.check-changes.outputs.code-changed == 'true' # Update check run called "integration-fork" diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 1a6da9499b4..9c58e0dba6c 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -236,16 +236,10 @@ jobs: - name: Run integration tests run: | container_id=${{ env.container_id }} - docker exec --env HOSTROOT=$GITHUB_WORKSPACE -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_CLIENT_SECRET_MULTITENANT -e AZURE_CLIENT_ID_MULTITENANT -e AZURE_CLIENT_ID_CERT_AUTH -e AZURE_CLIENT_SECRET_CERT_AUTH "$container_id" task controller:ci-integration-tests + docker exec --env HOSTROOT=$GITHUB_WORKSPACE -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "$container_id" task controller:ci-integration-tests env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - AZURE_CLIENT_SECRET_MULTITENANT: ${{ secrets.AZURE_CLIENT_SECRET_MULTITENANT }} - AZURE_CLIENT_ID_MULTITENANT: ${{ secrets.AZURE_CLIENT_ID_MULTITENANT }} - AZURE_CLIENT_ID_CERT_AUTH: ${{ secrets.AZURE_CLIENT_ID_CERT_AUTH }} - AZURE_CLIENT_SECRET_CERT_AUTH: ${{ secrets.AZURE_CLIENT_SECRET_CERT_AUTH }} if: steps.check-changes.outputs.code-changed == 'true' diff --git a/.github/workflows/pre-release-tests.yaml b/.github/workflows/pre-release-tests.yaml index e2c1e6af540..968988f574b 100644 --- a/.github/workflows/pre-release-tests.yaml +++ b/.github/workflows/pre-release-tests.yaml @@ -36,18 +36,14 @@ jobs: echo "container_id=$container_id" >> $GITHUB_ENV - name: Run Pre Release Tests - run: docker exec -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task controller:test-upgrade + run: docker exec -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task controller:test-upgrade env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Cleanup test resources if: always() - run: docker exec -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task cleanup-azure-resources + run: docker exec -e GITHUB_ACTIONS -e AZURE_TENANT_ID -e AZURE_SUBSCRIPTION_ID "${{ env.container_id }}" task cleanup-azure-resources env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/Taskfile.yml b/Taskfile.yml index 2659cf6c28b..fdd00685ff4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -55,6 +55,9 @@ vars: ACR_NAME: sh: "echo $(hostname)asoacr | sed 's/[^[:alpha:]]//g'" # Only alphanumeric allowed for acr names + KIND_WORKLOAD_IDENTITY_PATH: "v2/out/kind-identity" + AKS_WORKLOAD_IDENTITY_PATH: "v2/out/aks-identity" + tasks: default: deps: @@ -324,35 +327,47 @@ tasks: # -race fails at the moment in gopter - possibly due to our shared generator variable? - go test -short -tags=noexit -timeout 10m -covermode atomic -coverprofile=controller-coverage.out -json -coverpkg="./..." -run '{{default ".*" .TEST_FILTER}}' ./... > '{{.TEST_OUT}}/controller-unit-tests.json' - controller:test-upgrade-pre: - desc: Test upgrading {{.CONTROLLER_APP}} and helm chart for new release + controller:test-upgrade-setup: + desc: Setup for upgrade test dir: "{{.CONTROLLER_ROOT}}" cmds: - - task: controller:kind-create + - task: controller:kind-create-wi + - task: controller:create-mi-for-workload-identity - task: controller:install-cert-manager - task: controller:docker-push-local - task: controller:gen-helm-manifest # We need the below to wait until cert-manager is up, otherwise the subsequent installation of webhooks fails. See https://cert-manager.io/next-docs/installation/verify/ - "cmctl check api --wait=2m" + + controller:test-upgrade-pre: + desc: Test upgrading {{.CONTROLLER_APP}} and helm chart for new release + dir: "{{.CONTROLLER_ROOT}}" + cmds: - "helm repo add asov2 https://raw.githubusercontent.com/Azure/azure-service-operator/main/v2/charts" - "helm repo update" - - "helm upgrade --install --devel aso2 asov2/azure-service-operator \ + - "helm upgrade --install aso2 asov2/azure-service-operator \ --create-namespace \ --namespace={{.ASO_NAMESPACE}} \ --set azureSubscriptionID=$AZURE_SUBSCRIPTION_ID \ --set azureTenantID=$AZURE_TENANT_ID \ - --set azureClientID=$AZURE_CLIENT_ID \ - --set azureClientSecret=$AZURE_CLIENT_SECRET \ + --set azureClientID={{.AZURE_MI_CLIENT_ID}} \ + --set useWorkloadIdentityAuth=true \ --set crdPattern=*" - "kubectl create namespace pre-release" - task: controller:wait-for-operator-ready - "go test -timeout 15m -count=1 -v -run Test_Pre_Release_ResourceCanBeCreated_BeforeUpgrade ./test/pre-release" + vars: + AZURE_MI_CLIENT_ID: + sh: "cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt" controller:test-upgrade-post: desc: Test upgrading {{.CONTROLLER_APP}} and helm chart for new release dir: "{{.CONTROLLER_ROOT}}" cmds: - - task: controller:install-helm + - task: controller:install-helm-wi + vars: + AZURE_MI_CLIENT_ID: + sh: "cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt" - "go test -timeout 15m -count=1 -v -run Test_Pre_Release_ResourceCanBeCreated_AfterUpgrade ./test/pre-release" - task controller:kind-delete @@ -360,6 +375,7 @@ tasks: desc: Test upgrading {{.CONTROLLER_APP}} and helm chart for new release dir: "{{.CONTROLLER_ROOT}}" cmds: + - task: controller:test-upgrade-setup - task: controller:test-upgrade-pre - task: controller:test-upgrade-post @@ -694,21 +710,6 @@ tasks: cmds: - "{{.SCRIPTS_ROOT}}/generate-helm-manifest.sh {{.LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE}} {{.PUBLIC_REGISTRY}} {{.LATEST_VERSION_TAG}} `pwd`/" - controller:install-helm: - desc: Generate and install helm chart on cluster - dir: "{{.CONTROLLER_ROOT}}" - cmds: - - "helm upgrade --install --set azureSubscriptionID=$AZURE_SUBSCRIPTION_ID \ - --set azureTenantID=$AZURE_TENANT_ID \ - --set azureClientID=$AZURE_CLIENT_ID \ - --set azureClientSecret=$AZURE_CLIENT_SECRET \ - --set image.repository={{.IMAGE_REPOSITORY}} \ - --set crdPattern=* \ - aso2 -n {{.ASO_NAMESPACE}} --create-namespace ./charts/azure-service-operator/" - - task: controller:wait-for-operator-ready - vars: - IMAGE_REPOSITORY: "{{default .LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE .IMAGE_REPOSITORY}}" - controller:install-helm-wi: desc: Generate and install helm chart on cluster using workload identity dir: "{{.CONTROLLER_ROOT}}" @@ -717,14 +718,30 @@ tasks: --set azureTenantID=$AZURE_TENANT_ID \ --set azureClientID={{.AZURE_MI_CLIENT_ID}} \ --set useWorkloadIdentityAuth=true \ - --set image.repository={{.LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE}} \ + --set image.repository={{.IMAGE_REPOSITORY}} \ --set crdPattern=* \ aso2 -n {{.ASO_NAMESPACE}} --create-namespace ./charts/azure-service-operator/" - task: controller:wait-for-operator-ready vars: - # Override the AZURE_CLIENT_ID env variable here + IMAGE_REPOSITORY: "{{default .LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE .IMAGE_REPOSITORY}}" + + controller:install-helm-single-tenant: + desc: Generate and install helm chart on cluster using workload identity + dir: "{{.CONTROLLER_ROOT}}" + cmds: + - "helm upgrade --install --set azureSubscriptionID=$AZURE_SUBSCRIPTION_ID \ + --set azureTenantID=$AZURE_TENANT_ID \ + --set azureClientID={{.AZURE_MI_CLIENT_ID}} \ + --set image.repository={{.LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE}} \ + --set useWorkloadIdentityAuth=true \ + --set multitenant.enable=true \ + --set azureOperatorMode=watchers \ + --set-string azureTargetNamespaces='{t1-items,t1-more}' \ + aso2 -n tenant1-system --create-namespace ./charts/azure-service-operator/" + vars: + IMAGE_REPOSITORY: "{{default .LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE .IMAGE_REPOSITORY}}" AZURE_MI_CLIENT_ID: - sh: "cat v2/out/workload-identity/azure/miclientid.txt" + sh: "cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt #controller:install-helm-single-tenant" # Comment required to disambiguate sh usage, see: https://github.com/go-task/task/issues/524 controller:delete-helm: desc: Delete helm release @@ -736,9 +753,11 @@ tasks: run: always cmds: - "kind delete cluster --name=asov2" - - "{{.SCRIPTS_ROOT}}/delete-kind-wi-storage.sh -d v2/out/workload-identity" + - "{{.SCRIPTS_ROOT}}/delete-kind-wi-storage.sh -d {{.KIND_WORKLOAD_IDENTITY_PATH}}" - "kind delete cluster --name=asov2-wi" + # This is somewhat useless now because we're not using service principal to install ASO anymore + # but keeping it around in case there's some other use for a bare kind cluster. controller:kind-create: desc: Creates a kind cluster and local Docker registry. Images in the local registry can be pulled in the kind cluster run: always @@ -750,36 +769,38 @@ tasks: controller:kind-create-wi: desc: Creates a kind cluster and local Docker registry with OIDC + workload identity enabled. Images in the local registry can be pulled in the kind cluster. run: always - dir: "v2/" deps: - az-login cmds: - - "rm -rf out/workload-identity" - - "mkdir -p out/workload-identity" - - "{{.SCRIPTS_ROOT}}/create-kind-wi-storage.sh -d out/workload-identity -p {{.TEST_LIVE_RESOURCE_PREFIX}}" + - "rm -rf {{.KIND_WORKLOAD_IDENTITY_PATH}}" + - "mkdir -p {{.KIND_WORKLOAD_IDENTITY_PATH}}" + - "{{.SCRIPTS_ROOT}}/create-kind-wi-storage.sh -d {{.KIND_WORKLOAD_IDENTITY_PATH}} -p {{.TEST_LIVE_RESOURCE_PREFIX}}" - "export KIND_CLUSTER_NAME=asov2-wi && \ - export SERVICE_ACCOUNT_ISSUER=$(cat out/workload-identity/azure/saissuer.txt) && \ + export SERVICE_ACCOUNT_ISSUER=$(cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/saissuer.txt) && \ {{.SCRIPTS_ROOT}}/kind-with-registry.sh" status: - "kind get clusters | grep \"^asov2-wi$\"" env: SERVICE_ACCOUNT_KEY_FILE: - sh: "echo ${HOSTROOT:-$PWD}/v2/out/workload-identity/sa.pub" # Have to use HOSTROOT here for mounting files when in docker-in-docker as paths are relative to host + sh: "echo ${HOSTROOT:-$PWD}/{{.KIND_WORKLOAD_IDENTITY_PATH}}/sa.pub" # Have to use HOSTROOT here for mounting files when in docker-in-docker as paths are relative to host SERVICE_ACCOUNT_SIGNING_KEY_FILE: - sh: "echo ${HOSTROOT:-$PWD}/v2/out/workload-identity/sa.key" + sh: "echo ${HOSTROOT:-$PWD}/{{.KIND_WORKLOAD_IDENTITY_PATH}}/sa.key" controller:create-mi-for-workload-identity: - desc: Creates a managed identity and federated identity credential for user in a kind cluster - dir: "v2/" + desc: Creates a managed identity and federated identity credential and stores their details in the specified path deps: - az-login cmds: - - "{{.SCRIPTS_ROOT}}/make-mi-fic.sh -g {{.RESOURCE_GROUP}} -i {{.ISSUER}} -d ./out/workload-identity" + - "{{.SCRIPTS_ROOT}}/make-mi-fic.py -g {{.RESOURCE_GROUP}} -i {{.ISSUER}} -s {{.SUBJECT}} -d ./{{.DIR}}" vars: - RESOURCE_GROUP: - sh: "cat ./out/workload-identity/azure/rg.txt" - ISSUER: - sh: "cat ./out/workload-identity/azure/saissuer.txt" + DIR: "{{default .KIND_WORKLOAD_IDENTITY_PATH .DIR}}" + RESOURCE_GROUP_FILE: + sh: "cat ./{{.DIR}}/azure/rg.txt" + RESOURCE_GROUP: "{{default .RESOURCE_GROUP_FILE .RESOURCE_GROUP}}" + ISSUER_FILE: + sh: "cat ./{{.DIR}}/azure/saissuer.txt" + ISSUER: "{{default .ISSUER_FILE .ISSUER}}" + SUBJECT: '{{default "system:serviceaccount:azureserviceoperator-system:azureserviceoperator-default" .SUBJECT}}' controller:aks-create: desc: Creates an AKS cluster @@ -787,7 +808,7 @@ tasks: deps: - az-login cmds: - - "{{.SCRIPTS_ROOT}}/make-aks.sh -l {{.LOCATION}} -g {{.RESOURCE_GROUP_NAME}} -a {{.ACR_NAME}} -c {{.CLUSTER_NAME}}" + - "{{.SCRIPTS_ROOT}}/make-aks.sh -l {{.LOCATION}} -g {{.RESOURCE_GROUP_NAME}} -a {{.ACR_NAME}} -c {{.CLUSTER_NAME}} -d {{.AKS_WORKLOAD_IDENTITY_PATH}}" status: - az aks show --resource-group {{.RESOURCE_GROUP_NAME}} --name {{.CLUSTER_NAME}} > /dev/null 2>&1 vars: @@ -802,6 +823,8 @@ tasks: - az-login cmds: - "az group delete --name {{.RESOURCE_GROUP_NAME}} -y" + - "{{.SCRIPTS_ROOT}}/delete-role-assignment.sh -d {{.AKS_WORKLOAD_IDENTITY_PATH}}" + - "rm -rf {{.AKS_WORKLOAD_IDENTITY_PATH}}" vars: RESOURCE_GROUP_NAME: "{{.HOSTNAME}}-aso-rg" @@ -812,6 +835,10 @@ tasks: - az-login cmds: - task: controller:aks-create + - task: controller:create-mi-for-workload-identity + vars: + DIR: "{{.AKS_WORKLOAD_IDENTITY_PATH}}" + RESOURCE_GROUP: "{{.HOSTNAME}}-aso-rg" - task: controller:install-cert-manager - "az acr login --name {{.ACR_NAME}}" - task: controller:docker-push-multiarch @@ -821,9 +848,11 @@ tasks: - task: controller:gen-helm-manifest # We need the below to wait until cert-manager is up, otherwise the subsequent installation of webhooks fails. See https://cert-manager.io/next-docs/installation/verify/ - "cmctl check api --wait=2m" - - task: controller:install-helm + - task: controller:install-helm-wi vars: IMAGE_REPOSITORY: "{{.ACR_NAME}}.azurecr.io/{{.CONTROLLER_DOCKER_IMAGE}}" + AZURE_MI_CLIENT_ID: + sh: "cat {{.AKS_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt" controller:scan-image: desc: Builds a local image and scans the image using trivy @@ -879,7 +908,16 @@ tasks: env: # Override the AZURE_CLIENT_ID env variable here AZURE_MI_CLIENT_ID: # TODO: Ideally would override AZURE_CLIENT_ID here but we can't because of https://github.com/go-task/task/issues/1038 - sh: "cat v2/out/workload-identity/azure/miclientid.txt" + sh: "cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt" + + controller:deploy-multitenant-testing-secret: + desc: Deploy a multitenant testing secret + cmds: + - "{{.SCRIPTS_ROOT}}/deploy-multitenant-testing-secret.sh" + env: + # Override the AZURE_CLIENT_ID env variable here + AZURE_MI_CLIENT_ID: # TODO: Ideally would override AZURE_CLIENT_ID here but we can't because of https://github.com/go-task/task/issues/1038 + sh: "cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt" controller:wait-for-operator-ready: desc: Waits for the operator and associated CRDs to be ready @@ -887,18 +925,6 @@ tasks: cmds: - "{{.SCRIPTS_ROOT}}/wait-for-operator-ready.sh -n '{{default .ASO_NAMESPACE .NAMESPACE}}' {{.ARGS}}" - controller:kind-create-with-service-principal: - desc: Creates a local kind cluster with ASO using service principal auth. - cmds: - - task: controller:kind-create - - task: controller:install-cert-manager - - task: controller:docker-push-local - # We need the below to wait until cert-manager is up, otherwise the subsequent installation of webhooks fails. See https://cert-manager.io/next-docs/installation/verify/ - - "cmctl check api --wait=2m" - - task: controller:install - - task: controller:make-sp-secret - - task: controller:wait-for-operator-ready - controller:kind-create-with-workload-identity: desc: Creates a local kind cluster in workload identity mode with ASO using a managed identity. cmds: @@ -912,12 +938,14 @@ tasks: - task: controller:make-workload-identity-secret - task: controller:wait-for-operator-ready + # TODO: This currently doesn't work because the installation via raw YAML doesn't pass crdpattern. controller:kind-create-multitenant-cluster: desc: Creates a local kind cluster with ASO installed in multitenant configuration. deps: - controller:make-multitenant-files cmds: - - task: controller:kind-create + - task: controller:kind-create-wi + - task: controller:create-mi-for-workload-identity - task: controller:install-cert-manager - task: controller:docker-push-local # We need the below to wait until cert-manager is up, otherwise the subsequent installation of webhooks fails. See https://cert-manager.io/next-docs/installation/verify/ @@ -929,7 +957,7 @@ tasks: # Install tenant yaml - with same image switch - "cat v2/bin/multitenant-tenant_{{.VERSION}}.yaml | sed -e 's|image: mcr.microsoft.com/k8s/azureserviceoperator:.*|image: localhost:5000/azureserviceoperator:latest|' | kubectl apply --server-side=true -f -" # Make sp secret for tenant watching some target namespaces. - - "{{.SCRIPTS_ROOT}}/deploy-multitenant-testing-secret.sh" + - task: controller:deploy-multitenant-testing-secret # Create the namespaces that the tenant is watching. - "kubectl create namespace t1-items" - "kubectl create namespace t1-more" @@ -943,7 +971,17 @@ tasks: # This below is added to test if ASO works fine in namespaces other than 'azureserviceoperator-system" NAMESPACE: "custom-namespace" cmds: - - task: controller:kind-create + - task: controller:kind-create-wi + - task: controller:create-mi-for-workload-identity + vars: + # Use the tenant1-system namespace as that's where the single tenant operator will be installed + SUBJECT: "system:serviceaccount:tenant1-system:azureserviceoperator-default" + # We have to set RESOURCE_GROUP and ISSUER here because if we don't, the cached variables + # will be used from the create-mi-for-workload-identity task in ci. See: https://github.com/go-task/task/issues/524 + RESOURCE_GROUP: + sh: "cat ./{{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/rg.txt #controller:kind-create-multitenant-cluster-helm" + ISSUER: + sh: "cat ./{{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/saissuer.txt #controller:kind-create-multitenant-cluster-helm" - task: controller:install-cert-manager - task: controller:docker-push-local - task: controller:gen-helm-manifest @@ -957,33 +995,13 @@ tasks: - task: controller:wait-for-operator-ready vars: NAMESPACE: "{{.NAMESPACE}}" - # Install tenant chart - - "helm upgrade --install --set azureSubscriptionID=$AZURE_SUBSCRIPTION_ID \ - --set azureTenantID=$AZURE_TENANT_ID \ - --set azureClientID=$AZURE_CLIENT_ID \ - --set azureClientSecret=$AZURE_CLIENT_SECRET \ - --set image.repository={{.LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE}} \ - --set multitenant.enable=true \ - --set azureOperatorMode=watchers \ - --set-string azureTargetNamespaces='{t1-items,t1-more}' \ - aso2 -n tenant1-system --create-namespace ./charts/azure-service-operator/" + - task: controller:install-helm-single-tenant # Create the namespaces that the tenant is watching. - "kubectl create namespace t1-items" - "kubectl create namespace t1-more" # Wait for the other operator to be ready - not using controller:wait-for-operator-ready because that also waits for CRDs, etc which this doesn't install - "kubectl wait --for=condition=ready --timeout=2m pod -n tenant1-system -l control-plane=controller-manager" - controller:kind-create-helm: - desc: Creates a local kind cluster with ASO helm chart built and installed. - cmds: - - task: controller:kind-create - - task: controller:install-cert-manager - - task: controller:docker-push-local - - task: controller:gen-helm-manifest - # We need the below to wait until cert-manager is up, otherwise the subsequent installation of webhooks fails. See https://cert-manager.io/next-docs/installation/verify/ - - "cmctl check api --wait=2m" - - task: controller:install-helm - controller:kind-create-helm-workload-identity: desc: Creates a local kind cluster with Workload identity installed alongside ASO helm chart using UMI auth. cmds: @@ -995,6 +1013,9 @@ tasks: # We need the below to wait until cert-manager is up, otherwise the subsequent installation of webhooks fails. See https://cert-manager.io/next-docs/installation/verify/ - "cmctl check api --wait=2m" - task: controller:install-helm-wi + vars: + AZURE_MI_CLIENT_ID: + sh: "cat {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/miclientid.txt" # Stub retained until migration of workflows complete controller:gen-crd-docs: @@ -1123,16 +1144,22 @@ tasks: - '{{.SCRIPTS_ROOT}}/delete-old-resourcegroups.sh -p "{{.TEST_LIVE_RESOURCE_PREFIX}}" -a 10800' az-login: - desc: Runs AZ login - cmds: - - az login --service-principal -u {{.AZURE_CLIENT_ID}} -p {{.AZURE_CLIENT_SECRET}} --tenant {{.AZURE_TENANT_ID}} > /dev/null + desc: Runs AZ login. In the context of GitHub actions, uses an MSI token, in a local (non-actions) context, uses an interactive login + cmds: + # For interactive login: az cli has 2 tokens, an access token (valid for ~60m) and a refresh token (valid for ~12h) + # If the access token is expired, but the refresh token is still valid, the below command will get a new access token + # via the refresh token without asking for re-authentication. + # If the refresh token is expired, re-authentication is required. + # In practice this means that you should be asked for authentication once a (working) day, unless you're working for more + # than 12 hours in which case you'll be prompted twice. + - if [ "${GITHUB_ACTIONS}" = true ]; then az login --identity; elif ! az account get-access-token --tenant {{.AZURE_TENANT_ID}} --query "expiresOn" --output tsv > /dev/null; then az login; fi - az account set --subscription {{.AZURE_SUBSCRIPTION_ID}} docker-login: desc: Docker login cmds: - 'if [ -z "{{.DOCKER_REGISTRY}}" ]; then echo "Error: DOCKER_REGISTRY must be set"; exit 1; fi' - - docker login {{.DOCKER_REGISTRY}} --username {{.AZURE_CLIENT_ID}} --password {{.AZURE_CLIENT_SECRET}} + - az acr login --name {{.DOCKER_REGISTRY}} header-check: desc: Ensure all files have an appropriate license header. diff --git a/docs/hugo/content/_index.md b/docs/hugo/content/_index.md index e6e5ffe5c56..ffb6ef67286 100644 --- a/docs/hugo/content/_index.md +++ b/docs/hugo/content/_index.md @@ -34,7 +34,9 @@ ASO supports more than 150 different Azure resources, with more added every rele ### Installation -1. Install [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) on the cluster using the following command. +#### Install cert-manager on the cluster + +See [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) if you'd like to learn more about the project. ``` bash $ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.1/cert-manager.yaml @@ -52,7 +54,9 @@ cert-manager-webhook-c4b5687dc-x66bj 1/1 Running 0 1m (Alternatively, you can wait for cert-manager to be ready with `cmctl check api --wait=2m` - see the [cert-manager documentation](https://cert-manager.io/docs/usage/cmctl/) for more information about `cmctl`.) -2. Install [the latest **v2+** Helm chart](https://github.com/Azure/azure-service-operator/tree/main/v2/charts): +#### Install the latest **v2+** Helm chart + +The latest v2+ Helm chart can be found [here](https://github.com/Azure/azure-service-operator/tree/main/v2/charts) {{< tabpane text=true left=true >}} {{% tab header="**Shell**:" disabled=true /%}} @@ -96,10 +100,20 @@ See [CRD management](https://azure.github.io/azure-service-operator/guide/crd-ma Alternatively you can install from the [release YAML directly](https://azure.github.io/azure-service-operator/guide/installing-from-yaml/). -3. Create an Azure Service Principal. You'll need this to grant Azure Service Operator permissions to create resources - in your subscription. +#### Create a Managed Identity or Service Principal + +This identity or service principal will be used by ASO to authenticate with Azure. +You'll need this to grant the identity or Service Principal permissions to create resources in your subscription. + +{{% alert title="Note" %}} +We show steps for using a Service Principal below, as it's easiest to get started with, but recommend using a +Managed Identity with [Azure Workload Identity]( {{< relref "credential-format#azure-workload-identity" >}} ) for +use-cases other than testing. + +See [Security best practices]({{< relref "security" >}}) for the full list of security best practices. +{{% /alert %}} - First, set the following environment variables to your Azure Tenant ID and Subscription ID with your values: +First, set the following environment variables to your Azure Tenant ID and Subscription ID with your values: {{< tabpane text=true left=true >}} {{% tab header="**Shell**:" disabled=true /%}} @@ -189,7 +203,14 @@ C:\> SET AZURE_CLIENT_SECRET= {{% /tab %}} {{< /tabpane >}} -Then create a secret named `aso-credential` in the namespace you'd like to create ASO resources in. +#### Create the Azure Service Operator namespaced secret + +The secret must be named `aso-credential` and be created in the namespace you'd like to create ASO resources in. + +{{% alert title="Note" %}} +To learn about the ASO global secret, which works across all namespaces, or per-resource secrets, see +[authentication documentation](https://azure.github.io/azure-service-operator/guide/authentication/). +{{% /alert %}} {{< tabpane text=true left=true >}} {{% tab header="**Shell**:" disabled=true /%}} @@ -250,9 +271,6 @@ Then run: `kubectl apply -f secret.yaml` {{% /tab %}} {{< /tabpane >}} -To learn more about other authentication options, see the -[authentication documentation](https://azure.github.io/azure-service-operator/guide/authentication/). - ### Usage Once the controller has been installed in your cluster, you should be able to see the ASO pod running. diff --git a/docs/hugo/content/contributing/create-a-new-release.md b/docs/hugo/content/contributing/create-a-new-release.md index b6205878d63..24d6a4197fc 100644 --- a/docs/hugo/content/contributing/create-a-new-release.md +++ b/docs/hugo/content/contributing/create-a-new-release.md @@ -47,34 +47,36 @@ service Swagger specifications. We must validate each breaking change so that we Perform a simple smoke test to make sure the new release is capable of starting up and creating some Azure resources. -1. Create a kind cluster +1. Ensure you have the following environment variables exported: + * AZURE_SUBSCRIPTION_ID + * AZURE_TENANT_ID + +2. Create a kind cluster ``` bash - task controller:kind-create + task controller:kind-create-wi + ``` + +3. Enable workload identity on that cluster: + ``` + task: controller:create-mi-for-workload-identity ``` -2. Install cert-manager +4. Install cert-manager ``` bash task controller:install-cert-manager ``` -3. Create the namespace for the operator +5. Create the namespace for the operator ``` bash kubectl create namespace azureserviceoperator-system ``` -4. Ensure you have a [Service Principal](https://azure.github.io/azure-service-operator/#installation) and export these environment variables: - - * AZURE_SUBSCRIPTION - * AZURE_TENANT_ID - * AZURE_CLIENT_ID - * AZURE_CLIENT_SECRET - -5. Create a secret in your cluster with the Service Principal credentials for ASO to use: +6. Create a secret in your cluster with the Workload Identity credentials for ASO to use: ``` bash - task controller:make-sp-secret + task controller:make-workload-identity-secret ``` -6. Download asoctl +7. Download asoctl ``` bash curl -L https://github.com/Azure/azure-service-operator/releases/latest/download/asoctl-linux-amd64.gz -o asoctl.gz @@ -82,51 +84,65 @@ Perform a simple smoke test to make sure the new release is capable of starting chmod +x asoctl ``` -7. Use asoctl to install the new release +8. Use asoctl to install the new release ``` bash ./asoctl export template --version v2.7.0 --crd-pattern "resources.azure.com/*;network.azure.com/*" | kubectl apply -f - ``` -8. Watch while ASO starts +9. Watch while ASO starts ``` bash kubectl get all -n azureserviceoperator-system ``` -9. Create a resource group and a vnet in it (the vnet is to check that conversion webhooks are working, since there aren't any for RGs): +10. Create a resource group and a vnet in it (the vnet is to check that conversion webhooks are working, since there aren't any for RGs): - ``` bash - kubectl apply -f v2/samples/resources/v1api/v1api20200601_resourcegroup.yaml - kubectl apply -f v2/samples/network/v1api20201101/v1api20201101_virtualnetwork.yaml - ``` -10. Make sure they deploy successfully - check in the portal as well. + ``` bash + kubectl apply -f v2/samples/resources/v1api/v1api20200601_resourcegroup.yaml + kubectl apply -f v2/samples/network/v1api20201101/v1api20201101_virtualnetwork.yaml + ``` +11. Make sure they deploy successfully - check in the portal as well. ## Create and test the Helm chart -> **Note:** A PR that does this should be automatically generated when a new release is published. -> These steps are documented here in case that process fails. +{{% alert title="Note" %}} +A PR that does this should be automatically generated when a new release is published. +These steps are documented here in case that process fails. +{{% /alert %}} 1. Create a new branch from `` HEAD 2. Generate helm manifest for new release: `task controller:gen-helm-manifest` 3. Check the version in `/v2/charts/azure-service-operator/Chart.yaml` if matches with the latest release tag. -4. Install helm chart: +4. Create a kind cluster + ``` bash + task controller:kind-create-wi + ``` +5. Enable workload identity on that cluster: + ``` + task: controller:create-mi-for-workload-identity + ``` +6. Install cert-manager + ``` bash + task controller:install-cert-manager + ``` +7. Install helm chart: ``` helm install --set azureSubscriptionID=$AZURE_SUBSCRIPTION_ID \ --set azureTenantID=$AZURE_TENANT_ID \ - --set azureClientSecret=$AZURE_CLIENT_SECRET \ --set azureClientID=$AZURE_CLIENT_ID \ + --set useWorkloadIdentityAuth=true \ asov2 -n azureserviceoperator-system --create-namespace ./v2/charts/azure-service-operator/. ``` -5. Wait for the chart installation. -6. Wait for it to start: `k get all -n azureserviceoperator-system` -7. Create a resource group and a vnet in it (the vnet is to check that conversion webhooks are working, since there aren't any for RGs): - ``` - k apply -f v2/samples/resources/v1beta/v1beta20200601_resourcegroup.yaml - k apply -f v2/samples/network/v1beta/v1beta20201101_virtualnetwork.yaml - ``` -8. Make sure they deploy successfully - check in the portal as well. -9. If installed successfully, commit the files under `v2/charts/azure-service-operator`. -10. Send a PR. +8. Wait for the chart installation. +9. Wait for it to start: `k get all -n azureserviceoperator-system` +10. Create a resource group and a vnet in it (the vnet is to check that conversion webhooks are working, since there aren't any for RGs): + ``` + k apply -f v2/samples/resources/v1beta/v1beta20200601_resourcegroup.yaml + k apply -f v2/samples/network/v1beta/v1beta20201101_virtualnetwork.yaml + ``` +11. Make sure they deploy successfully - check in the portal as well. +12. If installed successfully, commit the files under `v2/charts/azure-service-operator`. +13. Send a PR. ## Update Resource Documentation diff --git a/docs/hugo/content/contributing/developer-setup.md b/docs/hugo/content/contributing/developer-setup.md index fadfe1ff644..80ca0ef970d 100644 --- a/docs/hugo/content/contributing/developer-setup.md +++ b/docs/hugo/content/contributing/developer-setup.md @@ -68,7 +68,7 @@ $ docker build $(git rev-parse --show-toplevel)/.devcontainer -t asodev:latest … image will be created … $ # After that you can start a terminal in the development container with: -$ docker run --env-file ~/work/envs.env --env HOSTROOT=$(git rev-parse --show-toplevel) -v $(git rev-parse --show-toplevel):/go/src -w /go/src -u $(id -u ${USER}):$(id -g ${USER}) --group-add $(stat -c '%g' /var/run/docker.sock) -v /var/run/docker.sock:/var/run/docker.sock --network=host -it asodev:latest /bin/bash +$ docker run --env-file ~/work/envs.env --env HOSTROOT=$(git rev-parse --show-toplevel) -v $(git rev-parse --show-toplevel):/go/src -w /go/src -u $(id -u ${USER}):$(id -g ${USER}) --group-add $(stat -c '%g' /var/run/docker.sock) -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.kube:/home/vscode/.kube -v $HOME/.azure/:/home/vscode/.azure/ --network=host -it asodev:latest /bin/bash ``` Note: If you mount the source like this from a Windows folder, performance will be poor as file operations between the container and Windows are very slow. diff --git a/docs/hugo/content/contributing/running-a-development-version.md b/docs/hugo/content/contributing/running-a-development-version.md index 7a70c644480..67dc54951a4 100644 --- a/docs/hugo/content/contributing/running-a-development-version.md +++ b/docs/hugo/content/contributing/running-a-development-version.md @@ -7,13 +7,11 @@ title: Running a Development Version If you would like to try something out but do not want to write an integration test, you can run your local version of the operator locally in a [kind](https://kind.sigs.k8s.io) cluster. -Before launching `kind`, make sure that your shell has the `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, -and `AZURE_CLIENT_SECRET` environment variables set. See [testing](../testing/#recordreplay) for more details about them. +Before launching `kind`, make sure that your shell has the `AZURE_SUBSCRIPTION_ID` and `AZURE_TENANT_ID` +environment variables set. See [testing](../testing/#recordreplay) for more details about them. -Once you've set the environment variables above, run one of the following commands to create a `kind` cluster: - -1. Service Principal authentication cluster: `task controller:kind-create-with-service-principal`. -2. AAD Pod Identity authentication enabled cluster (emulates Managed Identity): `controller:kind-create-with-podidentity`. +Once you've set the environment variables above, create a `kind` cluster: +- `task controller:kind-create-with-workload-identity`. You can use `kubectl` to interact with the local `kind` cluster. diff --git a/docs/hugo/content/contributing/testing.md b/docs/hugo/content/contributing/testing.md index 44662a3f177..b504a83d463 100644 --- a/docs/hugo/content/contributing/testing.md +++ b/docs/hugo/content/contributing/testing.md @@ -17,10 +17,11 @@ To do this, delete the recordings for the failing tests (under `{test-dir}/recor include with your change. All authentication and subscription information is removed from the recording. To run the test and produce a new recording you will need to have set the required authentication environment variables -for an Azure Service Principal: `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`. -This Service Principal will need access to the subscription to create and delete resources. +`AZURE_SUBSCRIPTION_ID` and `AZURE_TENANT_ID`, _and_ logged in via `az login` (or you just use the `task` commands +mentioned below and it will prompt you to `az login` if needed for that specific command). +Note that you must be `Owner` on the subscription to execute some tests in record mode. -A few tests also need the `TEST_BILLING_ID` variable set to a valid Azure Billing ID when running in record mode. +A few tests also need the `TEST_BILLING_ID` environment variable set to a valid Azure Billing ID when running in record mode. In replay mode this variable is never required. Note that the billing ID is redacted from all recording files so that the resulting file can be replayed by anybody, even somebody who does not know the Billing ID the test was recorded with. @@ -31,33 +32,11 @@ set `TIMEOUT` to a suitable value when running task. For example, to give your t TIMEOUT=60m task controller:test-integration-envtest ``` -If you need to create a new Azure Service Principal, run the following commands: - -```console -$ az login -… follow the instructions … -$ az account set --subscription {the subscription ID you would like to use} -Creating a role assignment under the scope of "/subscriptions/{subscription ID you chose}" -… -$ az ad sp create-for-rbac --role contributor --name {the name you would like to use} -{ - "appId": "…", - "displayName": "{name you chose}", - "name": "{name you chose}", - "password": "…", - "tenant": "…" -} -``` -The output contains `appId` (`AZURE_CLIENT_ID`), `password` (`AZURE_CLIENT_SECRET`), and `tenant` (`AZURE_TENANT_ID`). -Store these somewhere safe as the password cannot be viewed again, only reset. The Service Principal will be created as -a “contributor” to your subscription which means it can create and delete resources, so -**ensure you keep the secrets secure**. - ### Running live tests If you want to skip all recordings and run all tests directly against live Azure resources, you can use the `controller:test-integration-envtest-live` task. This will also require you to set the authentication environment -variables, as detailed above. +variables and `az login`, as detailed above. ### Running a single test By default `task controller:test-integration-envtest` and its variants run all tests. This is often undesirable diff --git a/docs/hugo/content/guide/authentication/credential-format.md b/docs/hugo/content/guide/authentication/credential-format.md index 3bd7e42dd28..4afb7429f39 100644 --- a/docs/hugo/content/guide/authentication/credential-format.md +++ b/docs/hugo/content/guide/authentication/credential-format.md @@ -132,7 +132,6 @@ stringData: AZURE_SUBSCRIPTION_ID: "$AZURE_SUBSCRIPTION_ID" AZURE_TENANT_ID: "$AZURE_TENANT_ID" AZURE_CLIENT_ID: "$AZURE_CLIENT_ID" - AUTH_MODE: "workloadidentity" EOF ``` @@ -155,7 +154,6 @@ stringData: AZURE_SUBSCRIPTION_ID: "$AZURE_SUBSCRIPTION_ID" AZURE_TENANT_ID: "$AZURE_TENANT_ID" AZURE_CLIENT_ID: "$AZURE_CLIENT_ID" - AUTH_MODE: "workloadidentity" EOF ``` diff --git a/docs/hugo/content/guide/installing-from-yaml.md b/docs/hugo/content/guide/installing-from-yaml.md index 0dcbb7a4137..75d6046e195 100644 --- a/docs/hugo/content/guide/installing-from-yaml.md +++ b/docs/hugo/content/guide/installing-from-yaml.md @@ -4,8 +4,8 @@ weight: -4 --- ## Prerequisites 1. You have installed Cert Manager as per the [installation instructions](../../#installation) up to the "install from Helm" step. -2. You have the `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` environment variables set from the - [installation instructions](../../#installation). +2. You have followed the [instructions for creating a Managed Identity or Service Principal](../../#create-a-managed-identity-or-service-principal) + and set the appropriate environment variables. ## Installation (operator) @@ -19,9 +19,9 @@ weight: -4 `--crd-pattern "resources.azure.com/*;containerservice.azure.com/*;keyvault.azure.com/*;managedidentity.azure.com/*;eventhub.azure.com/*"`. For more information about what `--crd-pattern` means, see [CRD management in ASO]({{}}). -3. Create the Azure Service Operator v2 secret. This secret contains the identity that Azure Service Operator will run as. - Make sure that you have the 4 environment variables from the [Helm installation instructions](../../#installation) set. - To learn more about other authentication options, see the [authentication documentation](../authentication/): +3. Create the Azure Service Operator v2 global secret. This secret contains the identity that Azure Service Operator will run as. + Make sure that you have the 4 environment variables from the + [create a service principal step of the Helm instructions](../../#create-a-managed-identity-or-service-principal) set. ```bash cat <" +} + +DIR= + +while getopts 'd:g:' flag; do + case "${flag}" in + d) DIR="${OPTARG}" ;; + *) print_usage + exit 1 ;; + esac +done + + +if [[ -z "$DIR" ]]; then + print_usage + exit 1 +fi + +if [ -f "$DIR/azure/roleassignmentid.txt" ]; then + # Need to delete the role assignment as well so we don't leak them + ROLE_ASSIGNMENT_ID=$(cat $DIR/azure/roleassignmentid.txt) + echo "Deleting role assignment: ${ROLE_ASSIGNMENT_ID}" + az role assignment delete --ids "${ROLE_ASSIGNMENT_ID}" +fi diff --git a/scripts/v2/deploy-multitenant-testing-secret.sh b/scripts/v2/deploy-multitenant-testing-secret.sh index c8dfec3afc4..15d69c24872 100755 --- a/scripts/v2/deploy-multitenant-testing-secret.sh +++ b/scripts/v2/deploy-multitenant-testing-secret.sh @@ -7,7 +7,23 @@ set -o errexit set -o nounset set -o pipefail -cat < [-g -a -c ]" + echo "Usage: make-aks.sh -l -d [-g -a -c ]" } # TODO maybe just make all of these arguments required? @@ -14,19 +14,21 @@ RESOURCE_GROUP= ACR_NAME= CLUSTER_NAME= LOCATION= -while getopts 'g:l:a:c:' flag; do +DIR= +while getopts 'g:l:a:c:d:' flag; do case "${flag}" in g) RESOURCE_GROUP="${OPTARG}" ;; l) LOCATION="${OPTARG}" ;; a) ACR_NAME="${OPTARG}" ;; c) CLUSTER_NAME="${OPTARG}" ;; + d) DIR="${OPTARG}" ;; *) print_usage exit 1 ;; esac done # Deal with required parameters -if [[ -z "$LOCATION" ]]; then +if [[ -z "$LOCATION" ]] || [[ -z "$DIR" ]]; then print_usage exit 1 fi @@ -52,6 +54,12 @@ az acr create --resource-group ${RESOURCE_GROUP} --name ${ACR_NAME} --sku Basic echo "Creating AKS cluster: ${CLUSTER_NAME}..." az aks create --resource-group ${RESOURCE_GROUP} --name ${CLUSTER_NAME} --attach-acr ${ACR_NAME} \ --enable-managed-identity --node-count 3 --generate-ssh-keys --network-dataplane cilium \ - --network-plugin azure --network-plugin-mode overlay --tier standard -o none + --network-plugin azure --network-plugin-mode overlay --tier standard -o none \ + --enable-oidc-issuer az aks get-credentials --resource-group ${RESOURCE_GROUP} --name ${CLUSTER_NAME} --overwrite-existing +# Make a directory to save OIDC issuer details to +mkdir -p "${DIR}/azure" + +# Get OIDC issuer URL and save it +az aks show --resource-group ${RESOURCE_GROUP} --name ${CLUSTER_NAME} --query "oidcIssuerProfile.issuerUrl" -otsv > "${DIR}/azure/saissuer.txt" diff --git a/scripts/v2/make-mi-fic.py b/scripts/v2/make-mi-fic.py new file mode 100755 index 00000000000..8e54c42f50d --- /dev/null +++ b/scripts/v2/make-mi-fic.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +import subprocess +import json +import os +import random +import string +import argparse +import time +import datetime + + +def create_role_assignment(subscription_id: str, object_id: str) -> str: + cmd = [ + "az", + "role", + "assignment", + "create", + "--assignee-object-id", object_id, + "--assignee-principal-type", "ServicePrincipal", + "--role", "Owner", + "--subscription", subscription_id, + "--scope", f"/subscriptions/{subscription_id}" + ] + result = subprocess.check_output(cmd, text=True) + return json.loads(result)["id"] + + +def retry_create_role_assignment(subscription_id: str, object_id: str) -> str: + start_time = datetime.datetime.utcnow() + while True: + try: + return create_role_assignment(subscription_id, object_id) + except subprocess.CalledProcessError: + if datetime.datetime.utcnow() - start_time > datetime.timedelta(minutes=1): + raise Exception("timed out waiting for role assignment creation") + print("Retrying role assignment creation...") + time.sleep(5) + + +def generate_random_string(length=12) -> str: + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) + + +def get_az_identity_show_args(resource_group: str, identity_name: str, query: str) -> [str]: + return [ + "az", + "identity", + "show", + "--resource-group", + resource_group, + "--name", + identity_name, + "--query", + "'{}'".format(query), + "-otsv"] + + +def main(): + # Create the parser + parser = argparse.ArgumentParser( + description='Make a managed identity, Federated Identity Credentials, and assign associated permissions') + + # Add arguments + parser.add_argument('-g', '--resource-group', type=str, help='Resource group name') + parser.add_argument('-i', '--issuer', type=str, help='Issuer name') + parser.add_argument('-d', '--directory', type=str, help='Directory path') + parser.add_argument('-s', '--subject', type=str, help='Subject') + + # Parse arguments + args = parser.parse_args() + + # Access arguments + resource_group = args.resource_group + issuer = args.issuer + directory = args.directory + subject = args.subject + + if not subject: + subject = "system:serviceaccount:azureserviceoperator-system:azureserviceoperator-default" + + if not resource_group or not issuer or not directory: + parser.print_help() + exit(1) + + subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID") + + # TODO: Might almost be easier to use the Azure SDK here... + identities = subprocess.check_output( + ["az", "identity", "list", "--resource-group", resource_group, "--output", "table"]) + identities = identities.rstrip() + if identities: + print("An identity already exists, not creating another one") + exit(0) + + identity_name = os.getenv("AZURE_IDENTITY_NAME") + existing_identity = False + if identity_name: + existing_identity = True + resource_group = os.getenv("AZURE_IDENTITY_RG") + else: + identity_name = f"mi{generate_random_string()}" + print(f"Generated new identity name: {identity_name}") + + if not existing_identity: + subprocess.run( + ["az", "identity", "create", "--name", identity_name, "--resource-group", resource_group], + check=True) + + subprocess.run([ + "az", + "identity", + "federated-credential", + "create", + "--identity-name", + identity_name, + "--name", + "fic", + "--resource-group", + resource_group, + "--issuer", + issuer, + "--subject", + subject, + "--audiences", + "api://AzureADTokenExchange" + ], check=True) + + + # need shell=True for the --query features + user_assigned_client_id = subprocess.check_output( + "az identity show --resource-group {} --name {} --query 'clientId' -otsv".format( + resource_group, + identity_name), + text=True, + shell=True) + + # need shell=True for the --query features + user_assigned_object_id = subprocess.check_output( + "az identity show --resource-group {} --name {} --query 'principalId' -otsv".format( + resource_group, + identity_name), + text=True, + shell=True) + + if not existing_identity: + role_assignment_id = retry_create_role_assignment(subscription_id, user_assigned_object_id) + with open(os.path.join(directory, "azure/roleassignmentid.txt"), "w") as f: + f.write(role_assignment_id) + + with open(os.path.join(directory, "azure/miclientid.txt"), "w") as f: + f.write(user_assigned_client_id) + + if existing_identity: + with open(os.path.join(directory, "azure/fic.txt"), "w") as f: + f.write("fic") + + +if __name__ == "__main__": + main() diff --git a/scripts/v2/make-mi-fic.sh b/scripts/v2/make-mi-fic.sh deleted file mode 100755 index 8039243a469..00000000000 --- a/scripts/v2/make-mi-fic.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -function create_role_assignment() { - az role assignment create --assignee "${USER_ASSIGNED_OBJECT_ID}" \ - --role "Owner" \ - --subscription "${AZURE_SUBSCRIPTION_ID}" \ - --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" -} - -function retry_create_role_assignment() { - until create_role_assignment; do - sleep 5 - done -} - -print_usage() { - echo "Usage: make-mi-fic.sh -g -i -d " - echo " To bring your own identity, set the AZURE_IDENTITY_NAME and AZURE_IDENTITY_RG env variables" -} - -RESOURCE_GROUP= -ISSUER= -DIR= - -while getopts 'g:i:d:' flag; do - case "${flag}" in - g) RESOURCE_GROUP="${OPTARG}" ;; - i) ISSUER="${OPTARG}" ;; - d) DIR="${OPTARG}" ;; - *) print_usage - exit 1 ;; - esac -done - - -if [[ -z "$RESOURCE_GROUP" ]] || [[ -z "$ISSUER" ]] || [[ -z "$DIR" ]]; then - print_usage - exit 1 -fi - -IDENTITIES=$(az identity list --resource-group ${RESOURCE_GROUP} -o table) - -if [[ ! -z "$IDENTITIES" ]]; then - echo "An identity already exists, not creating another one" - exit 0 -fi - -EXISTING_IDENTITY=false -if [[ ! -z "${AZURE_IDENTITY_NAME:-}" ]]; then - IDENTITY_NAME="$AZURE_IDENTITY_NAME" - RESOURCE_GROUP="$AZURE_IDENTITY_RG" - EXISTING_IDENTITY=true -else - IDENTITY_NAME="mi$(openssl rand -hex 6)" -fi - -SUBJECT="system:serviceaccount:azureserviceoperator-system:azureserviceoperator-default" - -if [ "$EXISTING_IDENTITY" = false ]; then - az identity create --name ${IDENTITY_NAME} --resource-group ${RESOURCE_GROUP} -fi - -az identity federated-credential create \ - --identity-name ${IDENTITY_NAME} \ - --name fic \ - --resource-group ${RESOURCE_GROUP} \ - --issuer ${ISSUER} \ - --subject ${SUBJECT} \ - --audiences "api://AzureADTokenExchange" - -export USER_ASSIGNED_CLIENT_ID="$(az identity show --resource-group "${RESOURCE_GROUP}" --name "${IDENTITY_NAME}" --query 'clientId' -otsv)" -export USER_ASSIGNED_OBJECT_ID="$(az identity show --resource-group "${RESOURCE_GROUP}" --name "${IDENTITY_NAME}" --query 'principalId' -otsv)" - -# Assumption is if the user brought their own identity that it already has the permissions it needs -if [ "$EXISTING_IDENTITY" = false ]; then - export -f create_role_assignment - export -f retry_create_role_assignment - timeout 1m bash -c retry_create_role_assignment - - # Save the ID of the role assignment we created as well - ROLE_ASSIGNMENT_ID=$(az role assignment list --assignee "${USER_ASSIGNED_OBJECT_ID}" --query "[0].id" | tr -d '"') - echo ${ROLE_ASSIGNMENT_ID} > "${DIR}/azure/roleassignmentid.txt" -fi - -echo ${USER_ASSIGNED_CLIENT_ID} > "${DIR}/azure/miclientid.txt" -if [ "$EXISTING_IDENTITY" = true ]; then - echo "fic" > "${DIR}/azure/fic.txt" -fi diff --git a/v2/cmd/controller/app/setup.go b/v2/cmd/controller/app/setup.go index 13f8e2eaf0d..995ec4f5f13 100644 --- a/v2/cmd/controller/app/setup.go +++ b/v2/cmd/controller/app/setup.go @@ -356,7 +356,7 @@ func initializeClients(cfg config.Values, mgr ctrl.Manager) (*clients, error) { } kubeClient := kubeclient.NewClient(mgr.GetClient()) - credentialProvider := identity.NewCredentialProvider(credential, kubeClient) + credentialProvider := identity.NewCredentialProvider(credential, kubeClient, nil) armClientCache := armreconciler.NewARMClientCache( credentialProvider, diff --git a/v2/internal/identity/credential_provider.go b/v2/internal/identity/credential_provider.go index 1346efb9a4b..de562c511e4 100644 --- a/v2/internal/identity/credential_provider.go +++ b/v2/internal/identity/credential_provider.go @@ -72,18 +72,33 @@ type CredentialProvider interface { GetCredential(ctx context.Context, obj genruntime.MetaObject) (*Credential, error) } +type CredentialProviderOptions struct { + TokenProvider TokenCredentialProvider +} + type credentialProvider struct { - globalCredential *Credential - kubeClient kubeclient.Client + globalCredential *Credential + kubeClient kubeclient.Client + tokenCredentialProvider TokenCredentialProvider } func NewCredentialProvider( globalCredential *Credential, kubeClient kubeclient.Client, + opts *CredentialProviderOptions, ) CredentialProvider { + if opts == nil { + opts = &CredentialProviderOptions{} + } + + if opts.TokenProvider == nil { + opts.TokenProvider = DefaultTokenCredentialProvider() + } + return &credentialProvider{ - kubeClient: kubeClient, - globalCredential: globalCredential, + kubeClient: kubeClient, + globalCredential: globalCredential, + tokenCredentialProvider: opts.TokenProvider, } } @@ -226,7 +241,7 @@ func (c *credentialProvider) newCredentialFromSecret(secret *v1.Secret) (*Creden } if clientSecret, hasClientSecret := secret.Data[config.AzureClientSecret]; hasClientSecret { - tokenCredential, err := azidentity.NewClientSecretCredential(string(tenantID), string(clientID), string(clientSecret), nil) + tokenCredential, err := c.tokenCredentialProvider.NewClientSecretCredential(string(tenantID), string(clientID), string(clientSecret), nil) if err != nil { return nil, errors.Wrap(err, errors.Errorf("invalid Client Secret Credential for %q encountered", nsName).Error()) } @@ -245,7 +260,7 @@ func (c *credentialProvider) newCredentialFromSecret(secret *v1.Secret) (*Creden clientCertPassword = p } - tokenCredential, err := NewClientCertificateCredential(string(tenantID), string(clientID), clientCert, clientCertPassword) + tokenCredential, err := c.tokenCredentialProvider.NewClientCertificateCredential(string(tenantID), string(clientID), clientCert, clientCertPassword) if err != nil { return nil, errors.Wrap(err, errors.Errorf("invalid Client Certificate Credential for %q encountered", nsName).Error()) } @@ -265,7 +280,7 @@ func (c *credentialProvider) newCredentialFromSecret(secret *v1.Secret) (*Creden } if authMode == config.PodIdentityAuthMode { - tokenCredential, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + tokenCredential, err := c.tokenCredentialProvider.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ ClientOptions: azcore.ClientOptions{}, ID: azidentity.ClientID(clientID), }) @@ -283,7 +298,7 @@ func (c *credentialProvider) newCredentialFromSecret(secret *v1.Secret) (*Creden } // Default to Workload Identity - tokenCredential, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + tokenCredential, err := c.tokenCredentialProvider.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ ClientID: string(clientID), TenantID: string(tenantID), TokenFilePath: FederatedTokenFilePath, diff --git a/v2/internal/identity/credential_provider_test.go b/v2/internal/identity/credential_provider_test.go index ccec4c86269..fe92dbd51e6 100644 --- a/v2/internal/identity/credential_provider_test.go +++ b/v2/internal/identity/credential_provider_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/google/uuid" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,8 +33,10 @@ const ( ) type testCredentialProviderResources struct { - Provider CredentialProvider - kubeClient kubeclient.Client + Provider CredentialProvider + kubeClient kubeclient.Client + fakeProvider CredentialProvider + fakeTokenCredentialProvider *mockTokenCredentialProvider } func NewTestCredentialProvider(client kubeclient.Client) (CredentialProvider, error) { @@ -43,7 +46,7 @@ func NewTestCredentialProvider(client kubeclient.Client) (CredentialProvider, er } creds := NewDefaultCredential(tokenCreds, testPodNamespace, testSubscriptionID) - return NewCredentialProvider(creds, client), nil + return NewCredentialProvider(creds, client, nil), nil } func testCredentialProviderSetup() (*testCredentialProviderResources, error) { @@ -55,9 +58,17 @@ func testCredentialProviderSetup() (*testCredentialProviderResources, error) { return nil, err } + fakeTokenCredentialProvider := &mockTokenCredentialProvider{} + fakeProvider := NewCredentialProvider( + nil, + client, + &CredentialProviderOptions{TokenProvider: fakeTokenCredentialProvider}) + return &testCredentialProviderResources{ - Provider: provider, - kubeClient: client, + Provider: provider, + kubeClient: client, + fakeTokenCredentialProvider: fakeTokenCredentialProvider, + fakeProvider: fakeProvider, }, nil } @@ -69,7 +80,7 @@ func TestCredentialProvider_DefaultCredentialNotSet_ReturnsErrorWhenTryToUseGlob s := createTestScheme() kubeClient := NewFakeKubeClient(s) - providerWithNoDefaultCred := NewCredentialProvider(nil, kubeClient) + providerWithNoDefaultCred := NewCredentialProvider(nil, kubeClient, nil) rg := newResourceGroup("") _, err := providerWithNoDefaultCred.GetCredential(ctx, rg) @@ -187,6 +198,129 @@ func TestCredentialProvider_GlobalCredential_IsReturned(t *testing.T) { g.Expect(cred.CredentialFrom()).To(BeEquivalentTo(types.NamespacedName{Namespace: testPodNamespace, Name: globalCredentialSecretName})) } +func TestCredentialProvider_ServicePrincipalCredential_IsConfiguredCorrectly(t *testing.T) { + t.Parallel() + g := NewGomegaWithT(t) + ctx := context.TODO() + + res, err := testCredentialProviderSetup() + g.Expect(err).ToNot(HaveOccurred()) + + clientID := uuid.New().String() + tenantID := uuid.New().String() + clientSecret := uuid.New().String() + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: NamespacedSecretName, + }, + Data: map[string][]byte{ + config.AzureSubscriptionID: []byte(testSubscriptionID), + config.AzureClientID: []byte(clientID), + config.AzureTenantID: []byte(tenantID), + config.AzureClientSecret: []byte(clientSecret), + }, + } + err = res.kubeClient.Create(ctx, secret) + g.Expect(err).ToNot(HaveOccurred()) + + rg := newResourceGroup("test-namespace") + err = res.kubeClient.Create(ctx, rg) + g.Expect(err).ToNot(HaveOccurred()) + + cred, err := res.fakeProvider.GetCredential(ctx, rg) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(cred.SubscriptionID()).To(BeEquivalentTo(testSubscriptionID)) + g.Expect(res.fakeTokenCredentialProvider.ClientID).To(Equal(clientID)) + g.Expect(res.fakeTokenCredentialProvider.TenantID).To(Equal(tenantID)) + g.Expect(res.fakeTokenCredentialProvider.ClientSecret).To(Equal(clientSecret)) +} + +func TestCredentialProvider_CertificateCredential_IsConfiguredCorrectly(t *testing.T) { + t.Parallel() + g := NewGomegaWithT(t) + ctx := context.TODO() + + res, err := testCredentialProviderSetup() + g.Expect(err).ToNot(HaveOccurred()) + + clientID := uuid.New().String() + tenantID := uuid.New().String() + cert := uuid.New().String() + certPassword := uuid.New().String() + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: NamespacedSecretName, + }, + Data: map[string][]byte{ + config.AzureSubscriptionID: []byte(testSubscriptionID), + config.AzureClientID: []byte(clientID), + config.AzureTenantID: []byte(tenantID), + config.AzureClientCertificate: []byte(cert), + config.AzureClientCertificatePassword: []byte(certPassword), + }, + } + + err = res.kubeClient.Create(ctx, secret) + g.Expect(err).ToNot(HaveOccurred()) + + rg := newResourceGroup("test-namespace") + err = res.kubeClient.Create(ctx, rg) + g.Expect(err).ToNot(HaveOccurred()) + + cred, err := res.fakeProvider.GetCredential(ctx, rg) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(cred.SubscriptionID()).To(BeEquivalentTo(testSubscriptionID)) + g.Expect(res.fakeTokenCredentialProvider.ClientID).To(Equal(clientID)) + g.Expect(res.fakeTokenCredentialProvider.TenantID).To(Equal(tenantID)) + g.Expect(res.fakeTokenCredentialProvider.ClientCertificate).To(Equal([]byte(cert))) + g.Expect(res.fakeTokenCredentialProvider.Password).To(Equal([]byte(certPassword))) +} + +func TestCredentialProvider_WorkloadIdentityCredential_IsConfiguredCorrectly(t *testing.T) { + t.Parallel() + g := NewGomegaWithT(t) + ctx := context.TODO() + + res, err := testCredentialProviderSetup() + g.Expect(err).ToNot(HaveOccurred()) + + clientID := uuid.New().String() + tenantID := uuid.New().String() + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: NamespacedSecretName, + }, + Data: map[string][]byte{ + config.AzureSubscriptionID: []byte(testSubscriptionID), + config.AzureClientID: []byte(clientID), + config.AzureTenantID: []byte(tenantID), + }, + } + + err = res.kubeClient.Create(ctx, secret) + g.Expect(err).ToNot(HaveOccurred()) + + rg := newResourceGroup("test-namespace") + err = res.kubeClient.Create(ctx, rg) + g.Expect(err).ToNot(HaveOccurred()) + + cred, err := res.fakeProvider.GetCredential(ctx, rg) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(cred.SubscriptionID()).To(BeEquivalentTo(testSubscriptionID)) + g.Expect(res.fakeTokenCredentialProvider.ClientID).To(Equal(clientID)) + g.Expect(res.fakeTokenCredentialProvider.TenantID).To(Equal(tenantID)) + g.Expect(res.fakeTokenCredentialProvider.TokenFilePath).To(Equal(FederatedTokenFilePath)) +} + func newResourceGroup(namespace string) *resources.ResourceGroup { return &resources.ResourceGroup{ TypeMeta: metav1.TypeMeta{ diff --git a/v2/internal/identity/token_credential_provider.go b/v2/internal/identity/token_credential_provider.go new file mode 100644 index 00000000000..f4f031ddd70 --- /dev/null +++ b/v2/internal/identity/token_credential_provider.go @@ -0,0 +1,39 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +package identity + +import "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + +type TokenCredentialProvider interface { + NewClientSecretCredential(tenantID string, clientID string, clientSecret string, options *azidentity.ClientSecretCredentialOptions) (*azidentity.ClientSecretCredential, error) + NewClientCertificateCredential(tenantID, clientID string, clientCertificate, password []byte) (*azidentity.ClientCertificateCredential, error) + NewManagedIdentityCredential(options *azidentity.ManagedIdentityCredentialOptions) (*azidentity.ManagedIdentityCredential, error) + NewWorkloadIdentityCredential(options *azidentity.WorkloadIdentityCredentialOptions) (*azidentity.WorkloadIdentityCredential, error) +} + +var _ TokenCredentialProvider = &tokenCredentialProvider{} + +type tokenCredentialProvider struct{} + +func (t *tokenCredentialProvider) NewClientSecretCredential(tenantID string, clientID string, clientSecret string, options *azidentity.ClientSecretCredentialOptions) (*azidentity.ClientSecretCredential, error) { + return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, options) +} + +func (t *tokenCredentialProvider) NewClientCertificateCredential(tenantID, clientID string, clientCertificate, password []byte) (*azidentity.ClientCertificateCredential, error) { + return NewClientCertificateCredential(tenantID, clientID, clientCertificate, password) +} + +func (t *tokenCredentialProvider) NewManagedIdentityCredential(options *azidentity.ManagedIdentityCredentialOptions) (*azidentity.ManagedIdentityCredential, error) { + return azidentity.NewManagedIdentityCredential(options) +} + +func (t *tokenCredentialProvider) NewWorkloadIdentityCredential(options *azidentity.WorkloadIdentityCredentialOptions) (*azidentity.WorkloadIdentityCredential, error) { + return azidentity.NewWorkloadIdentityCredential(options) +} + +func DefaultTokenCredentialProvider() TokenCredentialProvider { + return &tokenCredentialProvider{} +} diff --git a/v2/internal/identity/token_credential_provider_test.go b/v2/internal/identity/token_credential_provider_test.go new file mode 100644 index 00000000000..e396bd5d0a7 --- /dev/null +++ b/v2/internal/identity/token_credential_provider_test.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +package identity + +import "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + +var _ TokenCredentialProvider = &mockTokenCredentialProvider{} + +type mockTokenCredentialProvider struct { + TenantID string + ClientID string + ClientSecret string + ClientCertificate []byte + Password []byte + TokenFilePath string +} + +func (m *mockTokenCredentialProvider) NewClientSecretCredential(tenantID string, clientID string, clientSecret string, options *azidentity.ClientSecretCredentialOptions) (*azidentity.ClientSecretCredential, error) { + m.TenantID = tenantID + m.ClientID = clientID + m.ClientSecret = clientSecret + // We're not doing anything with the returned secrets so just return a dummy + return &azidentity.ClientSecretCredential{}, nil +} + +func (m *mockTokenCredentialProvider) NewClientCertificateCredential(tenantID, clientID string, clientCertificate, password []byte) (*azidentity.ClientCertificateCredential, error) { + m.TenantID = tenantID + m.ClientID = clientID + m.ClientCertificate = clientCertificate + m.Password = password + + return &azidentity.ClientCertificateCredential{}, nil +} + +func (m *mockTokenCredentialProvider) NewManagedIdentityCredential(options *azidentity.ManagedIdentityCredentialOptions) (*azidentity.ManagedIdentityCredential, error) { + return &azidentity.ManagedIdentityCredential{}, nil +} + +func (m *mockTokenCredentialProvider) NewWorkloadIdentityCredential(options *azidentity.WorkloadIdentityCredentialOptions) (*azidentity.WorkloadIdentityCredential, error) { + m.TenantID = options.TenantID + m.ClientID = options.ClientID + m.TokenFilePath = options.TokenFilePath + + return &azidentity.WorkloadIdentityCredential{}, nil +} diff --git a/v2/internal/reconcilers/arm/arm_client_cache_test.go b/v2/internal/reconcilers/arm/arm_client_cache_test.go index e840e29fb1e..88219c3110e 100644 --- a/v2/internal/reconcilers/arm/arm_client_cache_test.go +++ b/v2/internal/reconcilers/arm/arm_client_cache_test.go @@ -49,7 +49,7 @@ func NewTestCredentialProvider(client kubeclient.Client) (identity.CredentialPro } creds := identity.NewDefaultCredential(tokenCreds, testPodNamespace, testSubscriptionID) - return identity.NewCredentialProvider(creds, client), nil + return identity.NewCredentialProvider(creds, client, nil), nil } func NewTestARMClientCache(client kubeclient.Client) (*ARMClientCache, error) { @@ -106,7 +106,7 @@ func Test_DefaultCredential_NotSet_ReturnsErrorWhenTryToUseGlobalCredential(t *t cfg, err := config.ReadFromEnvironment() g.Expect(err).To(BeNil()) - providerWithNoDefaultCred := identity.NewCredentialProvider(nil, kubeClient) + providerWithNoDefaultCred := identity.NewCredentialProvider(nil, kubeClient, nil) clientWithNoDefaultCred := NewARMClientCache(providerWithNoDefaultCred, kubeClient, cfg.Cloud(), nil, metrics.NewARMClientMetrics()) rg := newResourceGroup("") diff --git a/v2/internal/testcommon/creds/creds.go b/v2/internal/testcommon/creds/creds.go index 6b4ec53a2ec..1e4ade1317c 100644 --- a/v2/internal/testcommon/creds/creds.go +++ b/v2/internal/testcommon/creds/creds.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" "github.com/Azure/azure-service-operator/v2/pkg/common/config" ) @@ -33,14 +34,51 @@ type AzureIDs struct { BillingInvoiceID string } +// getCredentials returns the token credential authentication modes supported by +// the test framework. +// We primarily support two modes of authentication: +// - EnvironmentCredential +// - CLICredential +// We don't use NewDefaultAzureCredential because it puts CLI credentials last +// which can cause issues when trying to do CLI auth from clients such as Virtual DevBoxes (which have a UMI +// that gets preferred over the CLI credentials). +func getCredentials() (*azidentity.ChainedTokenCredential, error) { + var result []azcore.TokenCredential + var errs []error + cliCred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + errs = append(errs, err) + } else { + result = append(result, cliCred) + } + + envCred, err := azidentity.NewEnvironmentCredential(nil) + if err != nil { + errs = append(errs, err) + } else { + result = append(result, envCred) + } + + if len(result) > 0 { + var chained *azidentity.ChainedTokenCredential + chained, err = azidentity.NewChainedTokenCredential(result, nil) + if err != nil { + return nil, err + } + return chained, nil + } else { + return nil, kerrors.NewAggregate(errs) + } +} + func GetCreds() (azcore.TokenCredential, AzureIDs, error) { if cachedCreds != nil { return cachedCreds, cachedIds, nil } - creds, err := azidentity.NewDefaultAzureCredential(nil) + creds, err := getCredentials() if err != nil { - return nil, AzureIDs{}, errors.Wrapf(err, "creating default credential") + return nil, AzureIDs{}, errors.Wrapf(err, "creating credentials") } subscriptionID := os.Getenv(config.AzureSubscriptionID) @@ -121,19 +159,3 @@ func NewScopedManagedIdentitySecret( return secret } - -func NewScopedServicePrincipalCertificateSecret( - subscriptionID string, - tenantID string, - clientID string, - clientCert string, - name string, - namespace string, -) *v1.Secret { - secret := newScopedCredentialSecret(subscriptionID, tenantID, name, namespace) - - secret.Data[config.AzureClientID] = []byte(clientID) - secret.Data[config.AzureClientCertificate] = []byte(clientCert) - - return secret -} diff --git a/v2/internal/testcommon/kube_test_context_envtest.go b/v2/internal/testcommon/kube_test_context_envtest.go index 8969d4fa2e9..27b0e2b7671 100644 --- a/v2/internal/testcommon/kube_test_context_envtest.go +++ b/v2/internal/testcommon/kube_test_context_envtest.go @@ -489,7 +489,7 @@ func createEnvtestContext() (BaseTestContextFactory, context.CancelFunc) { cfg.PodNamespace, perTestContext.AzureSubscription) - credentialProvider := identity.NewCredentialProvider(defaultCred, envtest.KubeClient) + credentialProvider := identity.NewCredentialProvider(defaultCred, envtest.KubeClient, nil) // register resources needed by controller for namespace armClientCache := arm.NewARMClientCache( credentialProvider, diff --git a/v2/test/single_operator_multitenant_test.go b/v2/test/single_operator_multitenant_test.go index 7c8842a0ce4..dfcab7a2c53 100644 --- a/v2/test/single_operator_multitenant_test.go +++ b/v2/test/single_operator_multitenant_test.go @@ -6,93 +6,95 @@ package test import ( "os" + "strings" "testing" + "time" "github.com/google/uuid" - "github.com/pkg/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - . "github.com/onsi/gomega" - + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + managedidentity "github.com/Azure/azure-service-operator/v2/api/managedidentity/v1api20230131" resources "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" storage "github.com/Azure/azure-service-operator/v2/api/storage/v1api20210401" "github.com/Azure/azure-service-operator/v2/internal/identity" "github.com/Azure/azure-service-operator/v2/internal/testcommon" "github.com/Azure/azure-service-operator/v2/internal/testcommon/creds" + "github.com/Azure/azure-service-operator/v2/internal/util/to" "github.com/Azure/azure-service-operator/v2/pkg/common/annotations" "github.com/Azure/azure-service-operator/v2/pkg/genruntime" "github.com/Azure/azure-service-operator/v2/pkg/genruntime/conditions" ) const ( - AzureClientIDMultitenantVar = "AZURE_CLIENT_ID_MULTITENANT" - AzureClientIDMultitenantCertAuthVar = "AZURE_CLIENT_ID_CERT_AUTH" - // #nosec - AzureClientSecretMultitenantVar = "AZURE_CLIENT_SECRET_MULTITENANT" - // #nosec - AzureClientCertificateMultitenantVar = "AZURE_CLIENT_SECRET_CERT_AUTH" + // TODO: This assumes the test is running at a particular path... I _think_ this is safe? + KindOIDCIssuerPath = "../out/kind-identity/azure/saissuer.txt" + CLusterOIDCIssuerVar = "CLUSTER_OIDC_ISSUER" + FICSubject = "system:serviceaccount:azureserviceoperator-system:azureserviceoperator-default" ) -func Test_Multitenant_SingleOperator_CertificateAuth(t *testing.T) { +// These tests are focused on multitenancy in a real deployment, where different namespaces use different credentials + +func Test_Multitenant_SingleOperator_NamespacedCredential(t *testing.T) { t.Parallel() tc := globalTestContext.ForTest(t) - secret, err := newClientCertificateCredential(tc.AzureSubscription, tc.AzureTenant, identity.NamespacedSecretName, tc.Namespace) - tc.Expect(err).To(BeNil()) - - tc.CreateResource(secret) - rg := tc.NewTestResourceGroup() - - tc.CreateResourcesAndWait(rg) - - resID := genruntime.GetResourceIDOrDefault(rg) - - // Make sure the ResourceGroup is created successfully in Azure. - exists, _, err := tc.AzureClient.CheckExistenceWithGetByID(tc.Ctx, resID, string(storage.APIVersion_Value)) + // Get the issuer we're going to use for Workload Identity + issuer, err := getOIDCIssuer() tc.Expect(err).ToNot(HaveOccurred()) - tc.Expect(exists).To(BeTrue()) - - tc.DeleteResourceAndWait(rg) -} -func Test_Multitenant_SingleOperator_NamespacedCredential(t *testing.T) { - t.Parallel() + namespaceName := "multitenant-namespaced-cred" + tc.Expect(tc.CreateTestNamespace(namespaceName)).To(Succeed()) + defer tc.DeleteResource(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + }) - tc := globalTestContext.ForTest(t) + rg := tc.CreateTestResourceGroupAndWait() - secret, err := newClientSecretCredential(tc.AzureSubscription, tc.AzureTenant, identity.NamespacedSecretName, tc.Namespace) - tc.Expect(err).To(BeNil()) + mi := newManagedIdentity(tc, rg) + fic := newFederatedIdentityCredential(tc, mi, issuer) + tc.CreateResourcesAndWait(mi, fic) + // TODO: See https://github.com/Azure/azure-service-operator/issues/3439 for the issue tracking us enabling this more + // TODO: easily in the operator itself (rather than reading from .Status in code) + secret := creds.NewScopedManagedIdentitySecret(tc.AzureSubscription, tc.AzureTenant, to.Value(mi.Status.ClientId), identity.NamespacedSecretName, namespaceName) tc.CreateResource(secret) - rg := tc.NewTestResourceGroup() - - tc.CreateResourcesAndWait(rg) - acct := newStorageAccount(tc, rg) + testResourceGroup := tc.NewTestResourceGroup() + testResourceGroup.Namespace = namespaceName - // Creating new storage account with restricted permissions namespaced secret should fail. - tc.CreateResourceAndWaitForState(acct, metav1.ConditionFalse, conditions.ConditionSeverityWarning) + // Creating new rg with restricted permissions namespaced secret should fail. + tc.CreateResourceAndWaitForState(testResourceGroup, metav1.ConditionFalse, conditions.ConditionSeverityWarning) + // We should eventually see an authorization error. There may be some initial errors related to the FederatedIdentityCredential + // not yet existing (AAD propagation can take a few seconds) + tc.Eventually(func() string { + tc.GetResource(types.NamespacedName{Namespace: testResourceGroup.Namespace, Name: testResourceGroup.Name}, testResourceGroup) + return testResourceGroup.Status.Conditions[0].Message + }).WithTimeout(1 * time.Minute).Should(ContainSubstring("does not have authorization to perform action")) // Deleting the credential would default to applying the global credential with all permissions tc.DeleteResource(secret) - tc.Eventually(acct).Should(tc.Match.BeProvisioned(0)) + tc.Eventually(testResourceGroup).Should(tc.Match.BeProvisioned(0)) - objKey := client.ObjectKeyFromObject(acct) - tc.GetResource(objKey, acct) + objKey := client.ObjectKeyFromObject(testResourceGroup) + tc.GetResource(objKey, testResourceGroup) - resID := genruntime.GetResourceIDOrDefault(acct) + resID := genruntime.GetResourceIDOrDefault(testResourceGroup) - // Make sure the StorageAccount is created successfully in Azure. - exists, _, err := tc.AzureClient.CheckExistenceWithGetByID(tc.Ctx, resID, string(storage.APIVersion_Value)) + // Make sure the ResourceGroup is created successfully in Azure. + exists, _, err := tc.AzureClient.CheckExistenceWithGetByID(tc.Ctx, resID, string(resources.APIVersion_Value)) tc.Expect(err).ToNot(HaveOccurred()) tc.Expect(exists).To(BeTrue()) - tc.DeleteResourcesAndWait(acct, rg) + tc.DeleteResourcesAndWait(testResourceGroup) } func Test_Multitenant_SingleOperator_PerResourceCredential(t *testing.T) { @@ -100,19 +102,32 @@ func Test_Multitenant_SingleOperator_PerResourceCredential(t *testing.T) { tc := globalTestContext.ForTest(t) - secret, err := newClientSecretCredential(tc.AzureSubscription, tc.AzureTenant, "credential", tc.Namespace) - tc.Expect(err).To(BeNil()) - - tc.CreateResource(secret) + // Get the issuer we're going to use for Workload Identity + issuer, err := getOIDCIssuer() + tc.Expect(err).ToNot(HaveOccurred()) rg := tc.CreateTestResourceGroupAndWait() + mi := newManagedIdentity(tc, rg) + fic := newFederatedIdentityCredential(tc, mi, issuer) + tc.CreateResourcesAndWait(mi, fic) + + // TODO: See https://github.com/Azure/azure-service-operator/issues/3439 for the issue tracking us enabling this more + // TODO: easily in the operator itself (rather than reading from .Status in code) + secret := creds.NewScopedManagedIdentitySecret(tc.AzureSubscription, tc.AzureTenant, to.Value(mi.Status.ClientId), "credential", tc.Namespace) + tc.CreateResource(secret) + acct := newStorageAccount(tc, rg) acct.Annotations = map[string]string{annotations.PerResourceSecret: secret.Name} // Creating new storage account in with restricted permissions per resource secret should fail. tc.CreateResourceAndWaitForState(acct, metav1.ConditionFalse, conditions.ConditionSeverityWarning) - tc.Expect(acct.Status.Conditions[0].Message).To(ContainSubstring("does not have authorization to perform action")) + // We should eventually see an authorization error. There may be some initial errors related to the FederatedIdentityCredential + // not yet existing (AAD propagation can take a few seconds) + tc.Eventually(func() string { + tc.GetResource(types.NamespacedName{Namespace: acct.Namespace, Name: acct.Name}, acct) + return acct.Status.Conditions[0].Message + }).WithTimeout(1 * time.Minute).Should(ContainSubstring("does not have authorization to perform action")) // Deleting the per-resource credential annotation would default to applying the global credential with all permissions old := acct.DeepCopy() @@ -140,13 +155,21 @@ func Test_Multitenant_SingleOperator_PerResourceCredential_MatchSubscriptionWith tc := globalTestContext.ForTest(t) - secret, err := newClientSecretCredential(uuid.New().String(), tc.AzureTenant, "credential", tc.Namespace) - tc.Expect(err).To(BeNil()) - - tc.CreateResource(secret) + // Get the issuer we're going to use for Workload Identity + issuer, err := getOIDCIssuer() + tc.Expect(err).ToNot(HaveOccurred()) rg := tc.CreateTestResourceGroupAndWait() + mi := newManagedIdentity(tc, rg) + fic := newFederatedIdentityCredential(tc, mi, issuer) + tc.CreateResourcesAndWait(mi, fic) + + // TODO: See https://github.com/Azure/azure-service-operator/issues/3439 for the issue tracking us enabling this more + // TODO: easily in the operator itself (rather than reading from .Status in code) + secret := creds.NewScopedManagedIdentitySecret(uuid.New().String(), tc.AzureTenant, to.Value(mi.Status.ClientId), "credential", tc.Namespace) + tc.CreateResource(secret) + acct := newStorageAccount(tc, rg) acct.Annotations = map[string]string{annotations.PerResourceSecret: secret.Name} @@ -156,32 +179,56 @@ func Test_Multitenant_SingleOperator_PerResourceCredential_MatchSubscriptionWith tc.Expect(acct.Status.Conditions[0].Severity).To(Equal(conditions.ConditionSeverityError)) } -func newClientSecretCredential(subscriptionID, tenantID, name, namespace string) (*v1.Secret, error) { - clientSecret := os.Getenv(AzureClientSecretMultitenantVar) - if clientSecret == "" { - return nil, errors.Errorf("required environment variable %q was not supplied", AzureClientSecretMultitenantVar) +// getOIDCIssuer gets the OIDC issuer for the cluster. +// There are two supported ways of informing the tests about the OIDC issuer. +// - Set the CLUSTER_OIDC_ISSUER environment variable +// - Run in a kind cluster created by ASO's automation, which writes the OIDC issuer to {{.KIND_WORKLOAD_IDENTITY_PATH}}/azure/saissuer.txt +func getOIDCIssuer() (string, error) { + // If we wanted this to work more generically for any OIDC issuer, we could do something like so: + // https://azure.github.io/azure-workload-identity/docs/installation/managed-clusters.html#steps-to-get-the-oidc-issuer-url-from-a-generic-managed-cluster + // but for now we don't really need that flexibility. + + // if the KIND OIDC issuer file exists, it takes precedence: + content, err := os.ReadFile(KindOIDCIssuerPath) + if err != nil { + if !os.IsNotExist(err) { + return "", err + } + } else { + return strings.TrimSpace(string(content)), nil } - clientID := os.Getenv(AzureClientIDMultitenantVar) - if clientID == "" { - return nil, errors.Errorf("required environment variable %q was not supplied", AzureClientIDMultitenantVar) + issuer := os.Getenv(CLusterOIDCIssuerVar) + if issuer != "" { + return issuer, nil } - return creds.NewScopedServicePrincipalSecret(subscriptionID, tenantID, clientID, clientSecret, name, namespace), nil + return "", errors.Errorf("could not determine cluster OIDC issuer either from %s or %s", KindOIDCIssuerPath, CLusterOIDCIssuerVar) } -func newClientCertificateCredential(subscriptionID, tenantID, name, namespace string) (*v1.Secret, error) { - clientCert := os.Getenv(AzureClientCertificateMultitenantVar) - if clientCert == "" { - return nil, errors.Errorf("required environment variable %q was not supplied", AzureClientCertificateMultitenantVar) +func newManagedIdentity(tc *testcommon.KubePerTestContext, rg *resources.ResourceGroup) *managedidentity.UserAssignedIdentity { + return &managedidentity.UserAssignedIdentity{ + ObjectMeta: tc.MakeObjectMetaWithName(tc.NoSpaceNamer.GenerateName("mi")), + Spec: managedidentity.UserAssignedIdentity_Spec{ + Location: tc.AzureRegion, + Owner: testcommon.AsOwner(rg), + }, } +} - clientID := os.Getenv(AzureClientIDMultitenantCertAuthVar) - if clientID == "" { - return nil, errors.Errorf("required environment variable %q was not supplied", AzureClientIDMultitenantCertAuthVar) +func newFederatedIdentityCredential(tc *testcommon.KubePerTestContext, umi *managedidentity.UserAssignedIdentity, issuer string) *managedidentity.FederatedIdentityCredential { + return &managedidentity.FederatedIdentityCredential{ + ObjectMeta: tc.MakeObjectMetaWithName("fic"), // Safe to always use this name as it's per-mi + Spec: managedidentity.UserAssignedIdentities_FederatedIdentityCredential_Spec{ + Owner: testcommon.AsOwner(umi), + // For Workload Identity, Audiences should always be "api://AzureADTokenExchange" + Audiences: []string{ + "api://AzureADTokenExchange", + }, + Issuer: to.Ptr(issuer), + Subject: to.Ptr(FICSubject), + }, } - - return creds.NewScopedServicePrincipalCertificateSecret(subscriptionID, tenantID, clientID, clientCert, name, namespace), nil } func newStorageAccount(tc *testcommon.KubePerTestContext, rg *resources.ResourceGroup) *storage.StorageAccount {