Integrating akvs to pipeline config (Azure#4770)

* akvs integration to pipeline config

* cspell fix

* formalize docs

* spell out akvs
remove extra space from pipeline template

* fix indentation for azdo template

* unit test for vars and secret gen

.vscode/cspell-github-user-aliases.txt

@@ -26,13 +26,14 @@ nicksnyder
 otiai10
 pamelafox
 pbnj
+rujche
 sebastianmattar
 sergi
 sethvargo
 stretchr
 theckman
 TheEskhaton
 tonybaloney
+vivazqu
 weilim
-Yionse
-rujche
+Yionse

cli/azd/cmd/env.go

@@ -426,7 +426,7 @@ func (e *envSetSecretAction) Run(ctx context.Context) (*actions.ActionResult, er
 	}
 
 	// akvs -> Azure Key Vault Secret (akvs://<subId>/<keyvault-name>/<secret-name>)
-	envValue := keyvault.NewAkvs(subId, kvAccount.Name, kvSecretName)
+	envValue := keyvault.NewAzureKeyVaultSecret(subId, kvAccount.Name, kvSecretName)
 	e.env.DotenvSet(secretName, envValue)
 	if err := e.envManager.Save(ctx, e.env); err != nil {
 		return nil, fmt.Errorf("saving environment: %w", err)

cli/azd/docs/using-environment-secrets.md

@@ -79,3 +79,79 @@ hooks:
 ```
 
 The next time you execute `azd provision`, the `preprovision` hook will run and resolve `MY_SECRET` into `SECRET_RESOLVE`.
+
+
+## Pipeline config
+
+The Azure Developer CLI simplifies the process of setting up continuous integration (CI) for your application. Whether you are using GitHub or Azure DevOps, you can run the command `azd pipeline config` and follow the guided steps provided by AZD to configure CI/CD.
+
+As part of the automatic configuration, AZD creates secrets and variables for your CI/CD deployment workflow. For example, the Azure Subscription ID and location are set as variables. Additionally, you can define a list of variables and secrets by using the `pipeline` configuration in the `azure.yaml` file within your project. The list of variables or secrets you define corresponds to the names of the keys in your AZD environment (.env). If the name of the key holds a secret reference (akvs), AZD will apply the following rules to set the value in your CI/CD settings.
+
+### Variables
+
+If the secret is added to the `variables` section of the pipeline configuration, the Azure Developer CLI (AZD) will use the value from the environment without retrieving the actual secret value. This approach is beneficial when you prefer to maintain Azure Key Vault references within your CI/CD settings. By doing so, you can rotate your secrets in the Key Vault, ensuring that your CI/CD pipeline uses the latest secret values without the need to update your workflow variables or secrets.
+
+#### Example
+
+- From an initialized AZD project, you have already set a secret. For example, you ran:
+
+```sh
+azd env set-secret SECURE_KEY
+```
+
+If you run `azd env get-values`, you would see an output like:
+
+```
+AZURE_ENV_NAME="your-env-name"
+AZURE_LOCATION="location"
+AZURE_SUBSCRIPTION_ID="faa080af-c1d8-40ad-9cce-000000000000"
+SECURE_KEY="akvs://faa080af-c1d8-40ad-9cce-000000000000/vivazqu-kv/SECURE-KEY-kv-secret"
+```
+
+- Then, you can set `SECURE_KEY` as a `variable` for your CI/CD pipeline by adding the `pipeline` field to the `azure.yaml` like:
+
+```yaml
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json
+name: your-project-name
+pipeline:
+  variables:
+    - SECURE_KEY
+```
+When you run `azd pipeline config`, the `SECURE_KEY` will be set as a variable in your CI/CD workflow, and its value will be the Azure Key Vault reference. For the example above, it would be: `akvs://faa080af-c1d8-40ad-9cce-000000000000/vivazqu-kv/SECURE-KEY-kv-secret`.
+
+> Note: AZD will attempt to assign a read-access role to the service principal used for logging into AZD from the CI/CD workflow. If you do not have sufficient permissions to assign the read role for the Key Vault, the operation will fail.
+
+You can reference the `SECURE_KEY` variable as a system environment variable. If `SECURE_KEY` is mapped to a Bicep input parameter or if it is mapped from a hook definition (refer to the earlier sections of this document for more details), AZD will automatically retrieve the value for the secret.
+
+### Secrets
+
+If the secret is added to the `secrets` section of the pipeline configuration, the Azure Developer CLI (AZD) will retrieve the actual value of the secret from the Azure Key Vault. This method is useful when it is not possible to assign a read-access role to the service principal used in your CI/CD workflow. However, it is important to note that you will need to update the secret value manually each time the secret is rotated. To re-apply the updated value, you can run the `azd pipeline config` command again.
+
+#### Example
+
+- From an initialized AZD project, you have already set a secret. For example, you ran:
+
+```sh
+azd env set-secret SECURE_KEY
+```
+
+If you run `azd env get-values`, you would see an output like:
+
+```
+AZURE_ENV_NAME="your-env-name"
+AZURE_LOCATION="location"
+AZURE_SUBSCRIPTION_ID="faa080af-c1d8-40ad-9cce-000000000000"
+SECURE_KEY="akvs://faa080af-c1d8-40ad-9cce-000000000000/vivazqu-kv/SECURE-KEY-kv-secret"
+```
+
+- Then, you can set `SECURE_KEY` as a `secret` for your CI/CD pipeline by adding the `pipeline` field to the `azure.yaml` like:
+
+```yaml
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json
+name: your-project-name
+pipeline:
+  secrets:
+    - SECURE_KEY
+```
+
+- When you run `azd pipeline config`, the `SECURE_KEY` will be set as a secret in your CI/CD workflow and its value will be the Azure Key Vault value.

cli/azd/pkg/ext/hooks_runner.go

@@ -148,7 +148,7 @@ func (h *HooksRunner) execHook(ctx context.Context, hookConfig *HookConfig, opti
 			for key, value := range hookConfig.Secrets {
 				setValue := value
 				if valueFromEnv, exists := h.env.LookupEnv(value); exists {
-					if keyvault.IsAkvs(valueFromEnv) {
+					if keyvault.IsAzureKeyVaultSecret(valueFromEnv) {
 						secretValue, err := keyvaultService.SecretFromAkvs(ctx, valueFromEnv)
 						if err != nil {
 							return err

cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go

@@ -1882,7 +1882,7 @@ func (p *BicepProvider) ensureParameters(
 				if stringValue, isString := paramValue.(string); isString && param.Secure() {
 					// For secure parameters using a string value, azd checks if the string is an Azure Key Vault Secret
 					// and if yes, it fetches the secret value from the Key Vault.
-					if keyvault.IsAkvs(stringValue) {
+					if keyvault.IsAzureKeyVaultSecret(stringValue) {
 						paramValue, err = p.keyvaultService.SecretFromAkvs(ctx, stringValue)
 						if err != nil {
 							return nil, err

cli/azd/pkg/keyvault/keyvault.go

@@ -341,7 +341,7 @@ const (
 	RoleIdKeyVaultSecretsUser   string = resourceIdPathPrefix + "4633458b-17de-408a-b874-0445c86b69e6"
 )
 
-func IsAkvs(id string) bool {
+func IsAzureKeyVaultSecret(id string) bool {
 	return strings.HasPrefix(id, vaultSchemaAkvs)
 }
 
@@ -351,32 +351,70 @@ func IsValidSecretName(kvSecretName string) bool {
 	}) == -1
 }
 
-func NewAkvs(subId, vaultId, secretName string) string {
+func NewAzureKeyVaultSecret(subId, vaultId, secretName string) string {
 	return vaultSchemaAkvs + subId + "/" + vaultId + "/" + secretName
 }
 
 func (kvs *keyVaultService) SecretFromAkvs(ctx context.Context, akvs string) (string, error) {
-	if !IsAkvs(akvs) {
-		return "", fmt.Errorf("invalid Azure Key Vault Secret reference: %s", akvs)
+	parseAkvs, err := ParseAzureKeyVaultSecret(akvs)
+	if err != nil {
+		return "", err
 	}
 
-	noSchema := strings.TrimPrefix(akvs, vaultSchemaAkvs)
-	vaultParts := strings.Split(noSchema, "/")
-	if len(vaultParts) != 3 {
-		return "", fmt.Errorf(
-			"invalid Azure Key Vault Secret reference: %s. Expected format: %s",
-			akvs,
-			vaultSchemaAkvs+"<subscription-id>/<vault-name>/<secret-name>",
-		)
-	}
-	subscriptionId, vaultName, secretName := vaultParts[0], vaultParts[1], vaultParts[2]
 	// subscriptionId is required by the Key Vault service to figure the TenantId for the
 	// tokenCredential. The assumption here is that the user has access to the Tenant
 	// used to deploy the app and to whatever Tenant the Key Vault is in. And the tokenCredential
 	// can use any of the Tenant ids.
-	secretValue, err := kvs.GetKeyVaultSecret(ctx, subscriptionId, vaultName, secretName)
+	secretValue, err := kvs.GetKeyVaultSecret(
+		ctx, parseAkvs.SubscriptionId, parseAkvs.VaultName, parseAkvs.SecretName)
 	if err != nil {
 		return "", fmt.Errorf("fetching secret value from key vault: %w", err)
 	}
 	return secretValue.Value, nil
 }
+
+// AzureKeyVaultSecret represents a secret stored in an Azure Key Vault.
+// It contains the necessary information to identify and access the secret.
+//
+// Fields:
+// - SubscriptionId: The ID of the Azure subscription that contains the Key Vault.
+// - VaultName: The name of the Key Vault where the secret is stored.
+// - SecretName: The name of the secret within the Key Vault.
+type AzureKeyVaultSecret struct {
+	SubscriptionId string
+	VaultName      string
+	SecretName     string
+}
+
+// ParseAzureKeyVaultSecret parses a string representing an Azure Key Vault Secret reference
+// and returns an AzureKeyVaultSecret struct if the reference is valid.
+//
+// The expected format for the Azure Key Vault Secret reference is:
+// "akvs://<subscription-id>/<vault-name>/<secret-name>"
+//
+// Parameters:
+//   - akvs: A string representing the Azure Key Vault Secret reference.
+//
+// Returns:
+//   - AzureKeyVaultSecret: A struct containing the subscription ID, vault name, and secret name.
+//   - error: An error if the Azure Key Vault Secret reference is invalid.
+func ParseAzureKeyVaultSecret(akvs string) (AzureKeyVaultSecret, error) {
+	if !IsAzureKeyVaultSecret(akvs) {
+		return AzureKeyVaultSecret{}, fmt.Errorf("invalid Azure Key Vault Secret reference: %s", akvs)
+	}
+
+	noSchema := strings.TrimPrefix(akvs, vaultSchemaAkvs)
+	vaultParts := strings.Split(noSchema, "/")
+	if len(vaultParts) != 3 {
+		return AzureKeyVaultSecret{}, fmt.Errorf(
+			"invalid Azure Key Vault Secret reference: %s. Expected format: %s",
+			akvs,
+			vaultSchemaAkvs+"<subscription-id>/<vault-name>/<secret-name>",
+		)
+	}
+	return AzureKeyVaultSecret{
+		SubscriptionId: vaultParts[0],
+		VaultName:      vaultParts[1],
+		SecretName:     vaultParts[2],
+	}, nil
+}

cli/azd/pkg/pipeline/pipeline.go

@@ -251,4 +251,6 @@ type projectProperties struct {
 	HasAppHost    bool
 	BranchName    string
 	AuthType      PipelineAuthType
+	Variables     []string
+	Secrets       []string
 }

cli/azd/pkg/pipeline/pipeline_manager.go

@@ -24,6 +24,7 @@ import (
 	"github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning"
 	"github.com/azure/azure-dev/cli/azd/pkg/input"
 	"github.com/azure/azure-dev/cli/azd/pkg/ioc"
+	"github.com/azure/azure-dev/cli/azd/pkg/keyvault"
 	"github.com/azure/azure-dev/cli/azd/pkg/osutil"
 	"github.com/azure/azure-dev/cli/azd/pkg/output"
 	"github.com/azure/azure-dev/cli/azd/pkg/output/ux"
@@ -94,6 +95,7 @@ type PipelineManager struct {
 	configOptions     *configurePipelineOptions
 	infra             *project.Infra
 	userConfigManager config.UserConfigManager
+	keyVaultService   keyvault.KeyVaultService
 }
 
 func NewPipelineManager(
@@ -108,6 +110,7 @@ func NewPipelineManager(
 	serviceLocator ioc.ServiceLocator,
 	importManager *project.ImportManager,
 	userConfigManager config.UserConfigManager,
+	keyVaultService keyvault.KeyVaultService,
 ) (*PipelineManager, error) {
 	pipelineProvider := &PipelineManager{
 		azdCtx:            azdCtx,
@@ -120,6 +123,7 @@ func NewPipelineManager(
 		serviceLocator:    serviceLocator,
 		importManager:     importManager,
 		userConfigManager: userConfigManager,
+		keyVaultService:   keyVaultService,
 	}
 
 	// check that scm and ci providers are set
@@ -408,6 +412,72 @@ func (pm *PipelineManager) Configure(ctx context.Context, projectName string) (r
 		pm.configOptions.projectVariables, pm.configOptions.projectSecrets,
 		defaultAzdVariables, defaultAzdSecrets, pm.env.Dotenv())
 
+	// resolve akvs secrets
+	// For each akvs in the secrets array:
+	// azd gets the value from Azure Key Vault and use it as a secret in the pipeline
+	for key, value := range pm.configOptions.secrets {
+		if !strings.HasPrefix(value, "akvs://") {
+			continue
+		}
+		kvSecret, err := pm.keyVaultService.SecretFromAkvs(ctx, value)
+		if err != nil {
+			return result, fmt.Errorf("failed to resolve akvs '%s': %w", key, err)
+		}
+		pm.configOptions.secrets[key] = kvSecret
+	}
+	// For each akvs in the variables array:
+	// azd must grant read access role to the pipelines's identity to read the akvs
+	displayMsg = "Assigning read access role for Key Vault to service principal"
+	pm.console.ShowSpinner(ctx, displayMsg, input.Step)
+	kvAccounts := make(map[string]struct{})
+	for key, value := range pm.configOptions.variables {
+		if !strings.HasPrefix(value, "akvs://") {
+			continue
+		}
+
+		akvs, err := keyvault.ParseAzureKeyVaultSecret(value)
+		if err != nil {
+			return result, fmt.Errorf("failed to parse akvs '%s': %w", key, err)
+		}
+		kvId := akvs.SubscriptionId + akvs.VaultName
+		if _, ok := kvAccounts[kvId]; ok {
+			// skip if already assigned role for this key vault
+			continue
+		}
+
+		// can't use keyvaultService.Get() because it requires the resource group name and we don't save it for akvs
+		allKvFromSub, err := pm.keyVaultService.ListSubscriptionVaults(ctx, akvs.SubscriptionId)
+		if err != nil {
+			return result, fmt.Errorf(
+				"assigning read access role for Key Vault to service principal: %w", err)
+		}
+		var vaultResourceId string
+		foundKeyVault := slices.ContainsFunc(allKvFromSub, func(kv keyvault.Vault) bool {
+			if kv.Name == akvs.VaultName {
+				vaultResourceId = kv.Id
+				return true
+			}
+			return false
+		})
+		if !foundKeyVault {
+			return result, fmt.Errorf(
+				"assigning read access role for Key Vault to service principal: "+
+					"key vault '%s' not found in subscription '%s'", akvs.VaultName, akvs.SubscriptionId)
+		}
+
+		// CreateRbac uses the azure-sdk RoleAssignmentsClient.Create() which creates or updates the role assignment
+		// We don't need to check if the role assignment already exists, the method will handle it.
+		err = pm.entraIdService.CreateRbac(
+			ctx, akvs.SubscriptionId, vaultResourceId, keyvault.RoleIdKeyVaultSecretsUser, *servicePrincipal.Id)
+		if err != nil {
+			return result, fmt.Errorf(
+				"assigning read access role for Key Vault to service principal: %w", err)
+		}
+		// save the kvId to avoid assigning the role multiple times for the same key vault
+		kvAccounts[kvId] = struct{}{}
+	}
+	pm.console.StopSpinner(ctx, displayMsg, input.GetStepResultFormat(err))
+
 	// config pipeline handles setting or creating the provider pipeline to be used
 	ciPipeline, err := pm.ciProvider.configurePipeline(ctx, gitRepoInfo, pm.configOptions)
 	if err != nil {
@@ -766,6 +836,8 @@ func (pm *PipelineManager) initialize(ctx context.Context, override string) erro
 			HasAppHost:    hasAppHost,
 			BranchName:    branchName,
 			AuthType:      authType,
+			Variables:     prjConfig.Pipeline.Variables,
+			Secrets:       prjConfig.Pipeline.Secrets,
 		}); err != nil {
 		return err
 	}
@@ -975,10 +1047,14 @@ func generatePipelineDefinition(path string, props projectProperties) error {
 		BranchName             string
 		FedCredLogIn           bool
 		InstallDotNetForAspire bool
+		Variables              []string
+		Secrets                []string
 	}{
 		BranchName:             props.BranchName,
 		FedCredLogIn:           props.AuthType == AuthTypeFederated,
 		InstallDotNetForAspire: props.HasAppHost,
+		Variables:              props.Variables,
+		Secrets:                props.Secrets,
 	})
 	if err != nil {
 		return fmt.Errorf("executing template: %w", err)