From 4d445ce2f184b961b361342bb8fa42f623c3f4aa Mon Sep 17 00:00:00 2001 From: Xiaolu Dai Date: Mon, 23 Dec 2024 17:01:33 +0800 Subject: [PATCH] add azure event hubs --- cli/azd/internal/appdetect/appdetect.go | 28 ++ cli/azd/internal/auth_type.go | 29 ++ cli/azd/internal/repository/app_init.go | 180 ++++++++- cli/azd/internal/repository/detect_confirm.go | 12 + cli/azd/internal/repository/infra_confirm.go | 23 +- .../internal/repository/infra_confirm_test.go | 4 +- cli/azd/internal/scaffold/bicep_env.go | 192 +++++++++ cli/azd/internal/scaffold/bicep_env_test.go | 114 ++++++ cli/azd/internal/scaffold/scaffold.go | 13 +- cli/azd/internal/scaffold/scaffold_test.go | 20 +- cli/azd/internal/scaffold/spec.go | 42 +- .../internal/scaffold/spec_service_binding.go | 380 ++++++++++++++++++ .../scaffold/spec_service_binding_test.go | 168 ++++++++ cli/azd/pkg/project/importer.go | 4 +- cli/azd/pkg/project/importer_test.go | 8 + cli/azd/pkg/project/resources.go | 50 ++- cli/azd/pkg/project/scaffold_gen.go | 195 ++++++--- cli/azd/pkg/project/service_config.go | 2 + ...ent-hubs-namespace-connection-string.bicep | 21 + .../scaffold/templates/resources.bicept | 106 +++-- 20 files changed, 1464 insertions(+), 127 deletions(-) create mode 100644 cli/azd/internal/auth_type.go create mode 100644 cli/azd/internal/scaffold/bicep_env.go create mode 100644 cli/azd/internal/scaffold/bicep_env_test.go create mode 100644 cli/azd/internal/scaffold/spec_service_binding.go create mode 100644 cli/azd/internal/scaffold/spec_service_binding_test.go create mode 100644 cli/azd/resources/scaffold/base/modules/set-event-hubs-namespace-connection-string.bicep diff --git a/cli/azd/internal/appdetect/appdetect.go b/cli/azd/internal/appdetect/appdetect.go index e6103d3cbd7..c0314e941a8 100644 --- a/cli/azd/internal/appdetect/appdetect.go +++ b/cli/azd/internal/appdetect/appdetect.go @@ -141,6 +141,12 @@ type Project struct { // Experimental: Database dependencies inferred through heuristics while scanning dependencies in the project. DatabaseDeps []DatabaseDep + // Experimental: Azure dependencies inferred through heuristics while scanning dependencies in the project. + AzureDeps []AzureDep + + // Experimental: Metadata inferred through heuristics while scanning the project. + Metadata Metadata + // The path to the project directory. Path string @@ -151,6 +157,28 @@ type Project struct { Docker *Docker } +//type AzureDep string + +type AzureDep interface { + ResourceDisplay() string +} + +type AzureDepEventHubs struct { + EventHubsNamePropertyMap map[string]string + UseKafka bool + SpringBootVersion string +} + +func (a AzureDepEventHubs) ResourceDisplay() string { + return "Azure Event Hubs" +} + +type Metadata struct { + ContainsDependencySpringCloudAzureStarter bool +} + +const UnknownSpringBootVersion string = "unknownSpringBootVersion" + func (p *Project) HasWebUIFramework() bool { for _, f := range p.Dependencies { if f.IsWebUIFramework() { diff --git a/cli/azd/internal/auth_type.go b/cli/azd/internal/auth_type.go new file mode 100644 index 00000000000..0399d327079 --- /dev/null +++ b/cli/azd/internal/auth_type.go @@ -0,0 +1,29 @@ +package internal + +// AuthType defines different authentication types. +type AuthType string + +const ( + AuthTypeUnspecified AuthType = "unspecified" + // Username and password, or key based authentication + AuthTypePassword AuthType = "password" + // Connection string authentication + AuthTypeConnectionString AuthType = "connectionString" + // Microsoft EntraID token credential + AuthTypeUserAssignedManagedIdentity AuthType = "userAssignedManagedIdentity" +) + +func GetAuthTypeDescription(authType AuthType) string { + switch authType { + case AuthTypeUnspecified: + return "Unspecified" + case AuthTypePassword: + return "Username and password" + case AuthTypeConnectionString: + return "Connection string" + case AuthTypeUserAssignedManagedIdentity: + return "User assigned managed identity" + default: + return "Unspecified" + } +} diff --git a/cli/azd/internal/repository/app_init.go b/cli/azd/internal/repository/app_init.go index cb211bc14b1..d937e58bdd6 100644 --- a/cli/azd/internal/repository/app_init.go +++ b/cli/azd/internal/repository/app_init.go @@ -44,6 +44,10 @@ var dbMap = map[appdetect.DatabaseDep]struct{}{ var featureCompose = alpha.MustFeatureKey("compose") +var azureDepMap = map[string]struct{}{ + appdetect.AzureDepEventHubs{}.ResourceDisplay(): {}, +} + // InitFromApp initializes the infra directory and project file from the current existing app. func (i *Initializer) InitFromApp( ctx context.Context, @@ -120,10 +124,44 @@ func (i *Initializer) InitFromApp( i.console.StopSpinner(ctx, title, input.StepDone) var prjAppHost []appdetect.Project - for _, prj := range projects { + for index, prj := range projects { if prj.Language == appdetect.DotNetAppHost { prjAppHost = append(prjAppHost, prj) } + + if prj.Language == appdetect.Java { + var hasKafkaDep bool + for depIndex, dep := range prj.AzureDeps { + if eventHubs, ok := dep.(appdetect.AzureDepEventHubs); ok { + // prompt spring boot version if not detected for kafka + if eventHubs.UseKafka { + hasKafkaDep = true + springBootVersion := eventHubs.SpringBootVersion + if springBootVersion == appdetect.UnknownSpringBootVersion { + springBootVersionInput, err := promptSpringBootVersion(i.console, ctx) + if err != nil { + return err + } + eventHubs.SpringBootVersion = springBootVersionInput + prj.AzureDeps[depIndex] = eventHubs + } + } + // prompt event hubs name if not detected + for property, eventHubsName := range eventHubs.EventHubsNamePropertyMap { + if eventHubsName == "" { + promptMissingPropertyAndExit(i.console, ctx, property) + } + } + } + } + + if hasKafkaDep && !prj.Metadata.ContainsDependencySpringCloudAzureStarter { + err := processSpringCloudAzureDepByPrompt(i.console, ctx, &projects[index]) + if err != nil { + return err + } + } + } } if len(prjAppHost) > 1 { @@ -455,6 +493,39 @@ func (i *Initializer) prjConfigFromDetect( dbNames[database] = db.Name } + for _, azureDepPair := range detect.AzureDeps { + azureDep := azureDepPair.first + authType, err := chooseAuthTypeByPrompt( + azureDep.ResourceDisplay(), + []internal.AuthType{internal.AuthTypeUserAssignedManagedIdentity, internal.AuthTypeConnectionString}, + ctx, + i.console) + if err != nil { + return config, err + } + switch azureDep := azureDep.(type) { + case appdetect.AzureDepEventHubs: + if azureDep.UseKafka { + config.Resources["kafka"] = &project.ResourceConfig{ + Type: project.ResourceTypeMessagingKafka, + Props: project.KafkaProps{ + Topics: distinctValues(azureDep.EventHubsNamePropertyMap), + AuthType: authType, + SpringBootVersion: azureDep.SpringBootVersion, + }, + } + } else { + config.Resources["eventhubs"] = &project.ResourceConfig{ + Type: project.ResourceTypeMessagingEventHubs, + Props: project.EventHubsProps{ + EventHubNames: distinctValues(azureDep.EventHubsNamePropertyMap), + AuthType: authType, + }, + } + } + } + } + backends := []*project.ResourceConfig{} frontends := []*project.ResourceConfig{} @@ -483,6 +554,17 @@ func (i *Initializer) prjConfigFromDetect( resSpec.Uses = append(resSpec.Uses, dbNames[db]) } + for _, azureDep := range svc.AzureDeps { + switch azureDep.(type) { + case appdetect.AzureDepEventHubs: + if azureDep.(appdetect.AzureDepEventHubs).UseKafka { + resSpec.Uses = append(resSpec.Uses, "kafka") + } else { + resSpec.Uses = append(resSpec.Uses, "eventhubs") + } + } + } + resSpec.Name = name resSpec.Props = props config.Resources[name] = &resSpec @@ -578,3 +660,99 @@ func ServiceFromDetect( return svc, nil } + +func chooseAuthTypeByPrompt( + name string, + authOptions []internal.AuthType, + ctx context.Context, + console input.Console) (internal.AuthType, error) { + var options []string + for _, option := range authOptions { + options = append(options, internal.GetAuthTypeDescription(option)) + } + selection, err := console.Select(ctx, input.ConsoleOptions{ + Message: "Choose auth type for " + name + ":", + Options: options, + }) + if err != nil { + return internal.AuthTypeUnspecified, err + } + return authOptions[selection], nil +} + +func promptMissingPropertyAndExit(console input.Console, ctx context.Context, key string) { + console.Message(ctx, fmt.Sprintf("No value was provided for %s. Please update the configuration file "+ + "(like application.properties or application.yaml) with a valid value.", key)) + os.Exit(0) +} + +func distinctValues(input map[string]string) []string { + valueSet := make(map[string]struct{}) + for _, value := range input { + valueSet[value] = struct{}{} + } + + var result []string + for value := range valueSet { + result = append(result, value) + } + + return result +} + +func processSpringCloudAzureDepByPrompt(console input.Console, ctx context.Context, project *appdetect.Project) error { + continueOption, err := console.Select(ctx, input.ConsoleOptions{ + Message: "Detected Kafka dependency but no spring-cloud-azure-starter found. Select an option", + Options: []string{ + "Exit then I will manually add this dependency", + "Continue without this dependency, and provision Azure Event Hubs for Kafka", + "Continue without this dependency, and not provision Azure Event Hubs for Kafka", + }, + }) + if err != nil { + return err + } + + switch continueOption { + case 0: + console.Message(ctx, "you have to manually add dependency com.azure.spring:spring-cloud-azure-starter. "+ + "And use right version according to this page: "+ + "https://github.com/Azure/azure-sdk-for-java/wiki/Spring-Versions-Mapping") + os.Exit(0) + case 1: + return nil + case 2: + // remove Kafka Azure Dep + var result []appdetect.AzureDep + for _, dep := range project.AzureDeps { + if eventHubs, ok := dep.(appdetect.AzureDepEventHubs); !(ok && eventHubs.UseKafka) { + result = append(result, dep) + } + } + project.AzureDeps = result + return nil + } + return nil +} + +func promptSpringBootVersion(console input.Console, ctx context.Context) (string, error) { + selection, err := console.Select(ctx, input.ConsoleOptions{ + Message: "No spring boot version detected, what is your spring boot version?", + Options: []string{ + "Spring Boot 2.x", + "Spring Boot 3.x", + }, + }) + if err != nil { + return "", err + } + + switch selection { + case 0: + return "2.x", nil + case 1: + return "3.x", nil + default: + return appdetect.UnknownSpringBootVersion, nil + } +} diff --git a/cli/azd/internal/repository/detect_confirm.go b/cli/azd/internal/repository/detect_confirm.go index e7191d271ae..02a841c6ca9 100644 --- a/cli/azd/internal/repository/detect_confirm.go +++ b/cli/azd/internal/repository/detect_confirm.go @@ -42,11 +42,17 @@ const ( EntryKindModified EntryKind = "modified" ) +type Pair struct { + first appdetect.AzureDep + second EntryKind +} + // detectConfirm handles prompting for confirming the detected services and databases type detectConfirm struct { // detected services and databases Services []appdetect.Project Databases map[appdetect.DatabaseDep]EntryKind + AzureDeps map[string]Pair // the root directory of the project root string @@ -73,6 +79,12 @@ func (d *detectConfirm) Init(projects []appdetect.Project, root string) { d.Databases[dbType] = EntryKindDetected } } + + for _, azureDep := range project.AzureDeps { + if _, supported := azureDepMap[azureDep.ResourceDisplay()]; supported { + d.AzureDeps[azureDep.ResourceDisplay()] = Pair{azureDep, EntryKindDetected} + } + } } d.captureUsage( diff --git a/cli/azd/internal/repository/infra_confirm.go b/cli/azd/internal/repository/infra_confirm.go index ee5a50f716b..1abe81938c6 100644 --- a/cli/azd/internal/repository/infra_confirm.go +++ b/cli/azd/internal/repository/infra_confirm.go @@ -85,19 +85,24 @@ func (i *Initializer) infraSpecFromDetect( switch db { case appdetect.DbMongo: - serviceSpec.DbCosmosMongo = &scaffold.DatabaseReference{ - DatabaseName: spec.DbCosmosMongo.DatabaseName, - } + err = scaffold.BindToMongoDb(&serviceSpec, spec.DbCosmosMongo) case appdetect.DbPostgres: - serviceSpec.DbPostgres = &scaffold.DatabaseReference{ - DatabaseName: spec.DbPostgres.DatabaseName, - } + err = scaffold.BindToPostgres(&serviceSpec, spec.DbPostgres) case appdetect.DbRedis: - serviceSpec.DbRedis = &scaffold.DatabaseReference{ - DatabaseName: "redis", - } + err = scaffold.BindToRedis(&serviceSpec, spec.DbRedis) + } + } + + for _, azureDep := range svc.AzureDeps { + switch azureDep.(type) { + case appdetect.AzureDepEventHubs: + err = scaffold.BindToEventHubs(&serviceSpec, spec.AzureEventHubs) } } + + if err != nil { + return scaffold.InfraSpec{}, err + } spec.Services = append(spec.Services, serviceSpec) } diff --git a/cli/azd/internal/repository/infra_confirm_test.go b/cli/azd/internal/repository/infra_confirm_test.go index 7ccfdfeab25..a9bda2bac4c 100644 --- a/cli/azd/internal/repository/infra_confirm_test.go +++ b/cli/azd/internal/repository/infra_confirm_test.go @@ -14,6 +14,7 @@ import ( ) func TestInitializer_infraSpecFromDetect(t *testing.T) { + envs, _ := scaffold.GetServiceBindingEnvsForPostgres() tests := []struct { name string detect detectConfirm @@ -183,9 +184,10 @@ func TestInitializer_infraSpecFromDetect(t *testing.T) { }, }, }, - DbPostgres: &scaffold.DatabaseReference{ + DbPostgres: &scaffold.DatabasePostgres{ DatabaseName: "myappdb", }, + Envs: envs, }, { Name: "js", diff --git a/cli/azd/internal/scaffold/bicep_env.go b/cli/azd/internal/scaffold/bicep_env.go new file mode 100644 index 00000000000..4045427644e --- /dev/null +++ b/cli/azd/internal/scaffold/bicep_env.go @@ -0,0 +1,192 @@ +package scaffold + +import ( + "fmt" + "strings" +) + +func ToBicepEnv(env Env) BicepEnv { + if isServiceBindingEnvValue(env.Value) { + serviceType, infoType := toServiceTypeAndServiceBindingInfoType(env.Value) + value, ok := bicepEnv[serviceType][infoType] + if !ok { + panic(unsupportedType(env)) + } + if isSecret(infoType) { + if isKeyVaultSecret(value) { + return BicepEnv{ + BicepEnvType: BicepEnvTypeKeyVaultSecret, + Name: env.Name, + SecretName: secretName(env), + SecretValue: unwrapKeyVaultSecretValue(value), + } + } else { + return BicepEnv{ + BicepEnvType: BicepEnvTypeSecret, + Name: env.Name, + SecretName: secretName(env), + SecretValue: value, + } + } + } else { + return BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: env.Name, + PlainTextValue: value, + } + } + } else { + return BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: env.Name, + PlainTextValue: toBicepEnvPlainTextValue(env.Value), + } + } +} + +// inputStringExample -> 'inputStringExample' +func addQuotation(input string) string { + return fmt.Sprintf("'%s'", input) +} + +// 'inputStringExample' -> 'inputStringExample' +// '${inputSingleVariableExample}' -> inputSingleVariableExample +// '${HOST}:${PORT}' -> '${HOST}:${PORT}' +func removeQuotationIfItIsASingleVariable(input string) string { + prefix := "'${" + suffix := "}'" + if strings.HasPrefix(input, prefix) && strings.HasSuffix(input, suffix) { + prefixTrimmed := strings.TrimPrefix(input, prefix) + trimmed := strings.TrimSuffix(prefixTrimmed, suffix) + if !strings.ContainsAny(trimmed, "}") { + return trimmed + } else { + return input + } + } else { + return input + } +} + +// The BicepEnv.PlainTextValue is handled as variable by default. +// If the value is string, it should contain ('). +// Here are some examples of input and output: +// inputStringExample -> 'inputStringExample' +// ${inputSingleVariableExample} -> inputSingleVariableExample +// ${HOST}:${PORT} -> '${HOST}:${PORT}' +func toBicepEnvPlainTextValue(input string) string { + return removeQuotationIfItIsASingleVariable(addQuotation(input)) +} + +// BicepEnv +// +// For Name and SecretName, they are handled as string by default. +// Which means quotation will be added before they are used in bicep file, because they are always string value. +// +// For PlainTextValue and SecretValue, they are handled as variable by default. +// When they are string value, quotation should be contained by themselves. +// Set variable as default is mainly to avoid this problem: +// https://learn.microsoft.com/en-us/azure/azure-resource-manager/bicep/linter-rule-simplify-interpolation +type BicepEnv struct { + BicepEnvType BicepEnvType + Name string + PlainTextValue string + SecretName string + SecretValue string +} + +type BicepEnvType string + +const ( + BicepEnvTypePlainText BicepEnvType = "plainText" + BicepEnvTypeSecret BicepEnvType = "secret" + BicepEnvTypeKeyVaultSecret BicepEnvType = "keyVaultSecret" +) + +// Note: The value is handled as variable. +// If the value is string, it should contain quotation inside itself. +var bicepEnv = map[ServiceType]map[ServiceBindingInfoType]string{ + ServiceTypeDbPostgres: { + ServiceBindingInfoTypeHost: "postgreServer.outputs.fqdn", + ServiceBindingInfoTypePort: "'5432'", + ServiceBindingInfoTypeDatabaseName: "postgreSqlDatabaseName", + ServiceBindingInfoTypeUsername: "postgreSqlDatabaseUser", + ServiceBindingInfoTypePassword: "postgreSqlDatabasePassword", + ServiceBindingInfoTypeUrl: "'postgresql://${postgreSqlDatabaseUser}:${postgreSqlDatabasePassword}@" + + "${postgreServer.outputs.fqdn}:5432/${postgreSqlDatabaseName}'", + ServiceBindingInfoTypeJdbcUrl: "'jdbc:postgresql://${postgreServer.outputs.fqdn}:5432/" + + "${postgreSqlDatabaseName}'", + }, + ServiceTypeDbRedis: { + ServiceBindingInfoTypeHost: "redis.outputs.hostName", + ServiceBindingInfoTypePort: "string(redis.outputs.sslPort)", + ServiceBindingInfoTypeEndpoint: "'${redis.outputs.hostName}:${redis.outputs.sslPort}'", + ServiceBindingInfoTypePassword: wrapToKeyVaultSecretValue("redisConn.outputs.keyVaultUrlForPass"), + ServiceBindingInfoTypeUrl: wrapToKeyVaultSecretValue("redisConn.outputs.keyVaultUrlForUrl"), + }, + ServiceTypeDbMongo: { + ServiceBindingInfoTypeDatabaseName: "mongoDatabaseName", + ServiceBindingInfoTypeUrl: wrapToKeyVaultSecretValue( + "cosmos.outputs.exportedSecrets['MONGODB-URL'].secretUri", + ), + }, + ServiceTypeMessagingEventHubs: { + ServiceBindingInfoTypeNamespace: "eventHubNamespace.outputs.name", + ServiceBindingInfoTypeEndpoint: "'${eventHubNamespace.outputs.name}.servicebus.windows.net:9093'", + ServiceBindingInfoTypeConnectionString: wrapToKeyVaultSecretValue( + "eventHubsConnectionString.outputs.keyVaultUrl", + ), + }, + ServiceTypeOpenAiModel: { + ServiceBindingInfoTypeEndpoint: "account.outputs.endpoint", + }, + ServiceTypeHostContainerApp: { + ServiceBindingInfoTypeHost: "https://{{BackendName}}.${containerAppsEnvironment.outputs.defaultDomain}", + }, +} + +func GetContainerAppHost(name string) string { + return strings.ReplaceAll( + bicepEnv[ServiceTypeHostContainerApp][ServiceBindingInfoTypeHost], + "{{BackendName}}", + name, + ) +} + +func unsupportedType(env Env) string { + return fmt.Sprintf( + "unsupported connection info type for resource type. value = %s", env.Value, + ) +} + +func PlaceHolderForServiceIdentityClientId() string { + return "__PlaceHolderForServiceIdentityClientId" +} + +func isSecret(info ServiceBindingInfoType) bool { + return info == ServiceBindingInfoTypePassword || info == ServiceBindingInfoTypeUrl || + info == ServiceBindingInfoTypeConnectionString +} + +func secretName(env Env) string { + resourceType, resourceInfoType := toServiceTypeAndServiceBindingInfoType(env.Value) + name := fmt.Sprintf("%s-%s", resourceType, resourceInfoType) + lowerCaseName := strings.ToLower(name) + noDotName := strings.Replace(lowerCaseName, ".", "-", -1) + noUnderscoreName := strings.Replace(noDotName, "_", "-", -1) + return noUnderscoreName +} + +var keyVaultSecretPrefix = "keyvault:" + +func isKeyVaultSecret(value string) bool { + return strings.HasPrefix(value, keyVaultSecretPrefix) +} + +func wrapToKeyVaultSecretValue(value string) string { + return fmt.Sprintf("%s%s", keyVaultSecretPrefix, value) +} + +func unwrapKeyVaultSecretValue(value string) string { + return strings.TrimPrefix(value, keyVaultSecretPrefix) +} diff --git a/cli/azd/internal/scaffold/bicep_env_test.go b/cli/azd/internal/scaffold/bicep_env_test.go new file mode 100644 index 00000000000..bafefc99e64 --- /dev/null +++ b/cli/azd/internal/scaffold/bicep_env_test.go @@ -0,0 +1,114 @@ +package scaffold + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToBicepEnv(t *testing.T) { + tests := []struct { + name string + in Env + want BicepEnv + }{ + { + name: "Plain text", + in: Env{ + Name: "enable-customer-related-feature", + Value: "true", + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: "enable-customer-related-feature", + PlainTextValue: "'true'", // Note: Quotation add automatically + }, + }, + { + name: "Plain text from EnvTypeResourceConnectionPlainText", + in: Env{ + Name: "spring.jms.servicebus.pricing-tier", + Value: "premium", + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: "spring.jms.servicebus.pricing-tier", + PlainTextValue: "'premium'", // Note: Quotation add automatically + }, + }, + { + name: "Plain text from EnvTypeResourceConnectionResourceInfo", + in: Env{ + Name: "POSTGRES_PORT", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePort), + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: "POSTGRES_PORT", + PlainTextValue: "'5432'", + }, + }, + { + name: "Secret", + in: Env{ + Name: "POSTGRES_PASSWORD", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePassword), + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypeSecret, + Name: "POSTGRES_PASSWORD", + SecretName: "db-postgres-password", + SecretValue: "postgreSqlDatabasePassword", + }, + }, + { + name: "KeuVault Secret", + in: Env{ + Name: "REDIS_PASSWORD", + Value: ToServiceBindingEnvValue(ServiceTypeDbRedis, ServiceBindingInfoTypePassword), + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypeKeyVaultSecret, + Name: "REDIS_PASSWORD", + SecretName: "db-redis-password", + SecretValue: "redisConn.outputs.keyVaultUrlForPass", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ToBicepEnv(tt.in) + assert.Equal(t, tt.want, actual) + }) + } +} + +func TestToBicepEnvPlainTextValue(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + name: "string", + in: "inputStringExample", + want: "'inputStringExample'", + }, + { + name: "single variable", + in: "${inputSingleVariableExample}", + want: "inputSingleVariableExample", + }, + { + name: "multiple variable", + in: "${HOST}:${PORT}", + want: "'${HOST}:${PORT}'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := toBicepEnvPlainTextValue(tt.in) + assert.Equal(t, tt.want, actual) + }) + } +} diff --git a/cli/azd/internal/scaffold/scaffold.go b/cli/azd/internal/scaffold/scaffold.go index f9ce4752ea9..d2b6b7db3ec 100644 --- a/cli/azd/internal/scaffold/scaffold.go +++ b/cli/azd/internal/scaffold/scaffold.go @@ -3,6 +3,7 @@ package scaffold import ( "bytes" "fmt" + "github.com/azure/azure-dev/cli/azd/internal" "io/fs" "os" "path" @@ -30,6 +31,8 @@ func Load() (*template.Template, error) { "lower": strings.ToLower, "alphaSnakeUpper": AlphaSnakeUpper, "formatParam": FormatParameter, + "hasPrefix": strings.HasPrefix, + "toBicepEnv": ToBicepEnv, } t, err := template.New("templates"). @@ -76,6 +79,10 @@ func supportingFiles(spec InfraSpec) []string { files = append(files, "/modules/fetch-container-image.bicep") } + if spec.AzureEventHubs != nil && spec.AzureEventHubs.AuthType == internal.AuthTypeConnectionString { + files = append(files, "/modules/set-event-hubs-namespace-connection-string.bicep") + } + return files } @@ -201,12 +208,12 @@ func executeToFS(targetFS *memfs.FS, tmpl *template.Template, name string, path } func preExecExpand(spec *InfraSpec) { - // postgres requires specific password seeding parameters + // postgres and mysql requires specific password seeding parameters if spec.DbPostgres != nil { spec.Parameters = append(spec.Parameters, Parameter{ - Name: "databasePassword", - Value: "$(secretOrRandomPassword ${AZURE_KEY_VAULT_NAME} databasePassword)", + Name: "postgreSqlDatabasePassword", + Value: "$(secretOrRandomPassword ${AZURE_KEY_VAULT_NAME} postgreSqlDatabasePassword)", Type: "string", Secret: true, }) diff --git a/cli/azd/internal/scaffold/scaffold_test.go b/cli/azd/internal/scaffold/scaffold_test.go index 238043c3673..d5a7dc212fb 100644 --- a/cli/azd/internal/scaffold/scaffold_test.go +++ b/cli/azd/internal/scaffold/scaffold_test.go @@ -98,13 +98,11 @@ func TestExecInfra(t *testing.T) { }, }, }, - DbCosmosMongo: &DatabaseReference{ + DbCosmosMongo: &DatabaseCosmosMongo{ DatabaseName: "appdb", }, - DbRedis: &DatabaseReference{ - DatabaseName: "redis", - }, - DbPostgres: &DatabaseReference{ + DbRedis: &DatabaseRedis{}, + DbPostgres: &DatabasePostgres{ DatabaseName: "appdb", }, }, @@ -133,7 +131,7 @@ func TestExecInfra(t *testing.T) { { Name: "api", Port: 3100, - DbPostgres: &DatabaseReference{ + DbPostgres: &DatabasePostgres{ DatabaseName: "appdb", }, }, @@ -150,7 +148,7 @@ func TestExecInfra(t *testing.T) { { Name: "api", Port: 3100, - DbCosmosMongo: &DatabaseReference{ + DbCosmosMongo: &DatabaseCosmosMongo{ DatabaseName: "appdb", }, }, @@ -163,11 +161,9 @@ func TestExecInfra(t *testing.T) { DbRedis: &DatabaseRedis{}, Services: []ServiceSpec{ { - Name: "api", - Port: 3100, - DbRedis: &DatabaseReference{ - DatabaseName: "redis", - }, + Name: "api", + Port: 3100, + DbRedis: &DatabaseRedis{}, }, }, }, diff --git a/cli/azd/internal/scaffold/spec.go b/cli/azd/internal/scaffold/spec.go index 763b83c322e..661b0456d5b 100644 --- a/cli/azd/internal/scaffold/spec.go +++ b/cli/azd/internal/scaffold/spec.go @@ -3,6 +3,8 @@ package scaffold import ( "fmt" "strings" + + "github.com/azure/azure-dev/cli/azd/internal" ) type InfraSpec struct { @@ -16,6 +18,8 @@ type InfraSpec struct { // ai models AIModels []AIModel + + AzureEventHubs *AzureDepEventHubs } type Parameter struct { @@ -51,11 +55,18 @@ type AIModelModel struct { Version string } +type AzureDepEventHubs struct { + EventHubNames []string + AuthType internal.AuthType + UseKafka bool + SpringBootVersion string +} + type ServiceSpec struct { Name string Port int - Env map[string]string + Envs []Env // Front-end properties. Frontend *Frontend @@ -64,12 +75,19 @@ type ServiceSpec struct { Backend *Backend // Connection to a database - DbPostgres *DatabaseReference - DbCosmosMongo *DatabaseReference - DbRedis *DatabaseReference + DbPostgres *DatabasePostgres + DbCosmosMongo *DatabaseCosmosMongo + DbRedis *DatabaseRedis // AI model connections AIModels []AIModelReference + + AzureEventHubs *AzureDepEventHubs +} + +type Env struct { + Name string + Value string } type Frontend struct { @@ -140,3 +158,19 @@ func serviceDefPlaceholder(serviceName string) Parameter { Secret: true, } } + +func AddNewEnvironmentVariable(serviceSpec *ServiceSpec, name string, value string) error { + merged, err := mergeEnvWithDuplicationCheck(serviceSpec.Envs, + []Env{ + { + Name: name, + Value: value, + }, + }, + ) + if err != nil { + return err + } + serviceSpec.Envs = merged + return nil +} diff --git a/cli/azd/internal/scaffold/spec_service_binding.go b/cli/azd/internal/scaffold/spec_service_binding.go new file mode 100644 index 00000000000..2fd31449638 --- /dev/null +++ b/cli/azd/internal/scaffold/spec_service_binding.go @@ -0,0 +1,380 @@ +package scaffold + +import ( + "fmt" + "strings" + + "github.com/azure/azure-dev/cli/azd/internal" +) + +// todo merge ServiceType and project.ResourceType +// Not use project.ResourceType because it will cause cycle import. +// Not merge it in current PR to avoid conflict with upstream main branch. +// Solution proposal: define a ServiceType in lower level that can be used both in scaffold and project package. + +type ServiceType string + +const ( + ServiceTypeDbRedis ServiceType = "db.redis" + ServiceTypeDbPostgres ServiceType = "db.postgres" + ServiceTypeDbMongo ServiceType = "db.mongo" + ServiceTypeHostContainerApp ServiceType = "host.containerapp" + ServiceTypeOpenAiModel ServiceType = "ai.openai.model" + ServiceTypeMessagingEventHubs ServiceType = "messaging.eventhubs" +) + +type ServiceBindingInfoType string + +const ( + ServiceBindingInfoTypeHost ServiceBindingInfoType = "host" + ServiceBindingInfoTypePort ServiceBindingInfoType = "port" + ServiceBindingInfoTypeEndpoint ServiceBindingInfoType = "endpoint" + ServiceBindingInfoTypeDatabaseName ServiceBindingInfoType = "databaseName" + ServiceBindingInfoTypeNamespace ServiceBindingInfoType = "namespace" + ServiceBindingInfoTypeUsername ServiceBindingInfoType = "username" + ServiceBindingInfoTypePassword ServiceBindingInfoType = "password" + ServiceBindingInfoTypeUrl ServiceBindingInfoType = "url" + ServiceBindingInfoTypeJdbcUrl ServiceBindingInfoType = "jdbcUrl" + ServiceBindingInfoTypeConnectionString ServiceBindingInfoType = "connectionString" +) + +var serviceBindingEnvValuePrefix = "$service.binding" + +func isServiceBindingEnvValue(env string) bool { + if !strings.HasPrefix(env, serviceBindingEnvValuePrefix) { + return false + } + a := strings.Split(env, ":") + if len(a) != 3 { + return false + } + return a[0] != "" && a[1] != "" && a[2] != "" +} + +func ToServiceBindingEnvValue(resourceType ServiceType, resourceInfoType ServiceBindingInfoType) string { + return fmt.Sprintf("%s:%s:%s", serviceBindingEnvValuePrefix, resourceType, resourceInfoType) +} + +func toServiceTypeAndServiceBindingInfoType(resourceConnectionEnv string) ( + serviceType ServiceType, infoType ServiceBindingInfoType) { + if !isServiceBindingEnvValue(resourceConnectionEnv) { + return "", "" + } + a := strings.Split(resourceConnectionEnv, ":") + return ServiceType(a[1]), ServiceBindingInfoType(a[2]) +} + +func BindToMongoDb(serviceSpec *ServiceSpec, mongo *DatabaseCosmosMongo) error { + serviceSpec.DbCosmosMongo = mongo + envs := GetServiceBindingEnvsForMongo() + var err error + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToPostgres(serviceSpec *ServiceSpec, postgres *DatabasePostgres) error { + serviceSpec.DbPostgres = postgres + envs, err := GetServiceBindingEnvsForPostgres() + if err != nil { + return err + } + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToRedis(serviceSpec *ServiceSpec, redis *DatabaseRedis) error { + serviceSpec.DbRedis = redis + envs := GetServiceBindingEnvsForRedis() + var err error + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToEventHubs(serviceSpec *ServiceSpec, eventHubs *AzureDepEventHubs) error { + serviceSpec.AzureEventHubs = eventHubs + envs, err := GetServiceBindingEnvsForEventHubs(*eventHubs) + if err != nil { + return err + } + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToAIModels(serviceSpec *ServiceSpec, model string) error { + serviceSpec.AIModels = append(serviceSpec.AIModels, AIModelReference{Name: model}) + envs := GetServiceBindingEnvsForAIModel() + var err error + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +// BindToContainerApp a call b +// todo: +// 1. Add field in ServiceSpec to identify b's app type like Eureka server and Config server. +// 2. Create GetServiceBindingEnvsForContainerApp +// 3. Merge GetServiceBindingEnvsForEurekaServer and GetServiceBindingEnvsForConfigServer into +// GetServiceBindingEnvsForContainerApp. +// 4. Delete printHintsAboutUseHostContainerApp use GetServiceBindingEnvsForContainerApp instead +func BindToContainerApp(a *ServiceSpec, b *ServiceSpec) { + if a.Frontend == nil { + a.Frontend = &Frontend{} + } + a.Frontend.Backends = append(a.Frontend.Backends, ServiceReference{Name: b.Name}) + if b.Backend == nil { + b.Backend = &Backend{} + } + b.Backend.Frontends = append(b.Backend.Frontends, ServiceReference{Name: b.Name}) +} + +func GetServiceBindingEnvsForMongo() []Env { + return []Env{ + { + Name: "MONGODB_URL", + Value: ToServiceBindingEnvValue(ServiceTypeDbMongo, ServiceBindingInfoTypeUrl), + }, + { + Name: "spring.data.mongodb.uri", + Value: ToServiceBindingEnvValue(ServiceTypeDbMongo, ServiceBindingInfoTypeUrl), + }, + { + Name: "spring.data.mongodb.database", + Value: ToServiceBindingEnvValue(ServiceTypeDbMongo, ServiceBindingInfoTypeDatabaseName), + }, + } +} + +func GetServiceBindingEnvsForPostgres() ([]Env, error) { + return []Env{ + { + Name: "POSTGRES_USERNAME", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUsername), + }, + { + Name: "POSTGRES_PASSWORD", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePassword), + }, + { + Name: "POSTGRES_HOST", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeHost), + }, + { + Name: "POSTGRES_DATABASE", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeDatabaseName), + }, + { + Name: "POSTGRES_PORT", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePort), + }, + { + Name: "POSTGRES_URL", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUrl), + }, + { + Name: "spring.datasource.url", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeJdbcUrl), + }, + { + Name: "spring.datasource.username", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUsername), + }, + { + Name: "spring.datasource.password", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePassword), + }, + }, nil +} + +func GetServiceBindingEnvsForRedis() []Env { + return []Env{ + { + Name: "REDIS_HOST", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeHost), + }, + { + Name: "REDIS_PORT", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypePort), + }, + { + Name: "REDIS_ENDPOINT", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeEndpoint), + }, + { + Name: "REDIS_URL", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeUrl), + }, + { + Name: "REDIS_PASSWORD", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypePassword), + }, + { + Name: "spring.data.redis.url", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeUrl), + }, + } +} + +func GetServiceBindingEnvsForEventHubs(eventHubs AzureDepEventHubs) ([]Env, error) { + if eventHubs.UseKafka { + return GetServiceBindingEnvsForEventHubsKafka(eventHubs) + } + switch eventHubs.AuthType { + case internal.AuthTypeUserAssignedManagedIdentity: + return []Env{ + // Not add this: spring.cloud.azure.eventhubs.connection-string = "" + // because of this: https://github.com/Azure/azure-sdk-for-java/issues/42880 + { + Name: "spring.cloud.azure.eventhubs.credential.managed-identity-enabled", + Value: "true", + }, + { + Name: "spring.cloud.azure.eventhubs.credential.client-id", + Value: PlaceHolderForServiceIdentityClientId(), + }, + { + Name: "spring.cloud.azure.eventhubs.namespace", + Value: ToServiceBindingEnvValue(ServiceTypeMessagingEventHubs, ServiceBindingInfoTypeNamespace), + }, + }, nil + case internal.AuthTypeConnectionString: + return []Env{ + { + Name: "spring.cloud.azure.eventhubs.namespace", + Value: ToServiceBindingEnvValue(ServiceTypeMessagingEventHubs, ServiceBindingInfoTypeNamespace), + }, + { + Name: "spring.cloud.azure.eventhubs.connection-string", + Value: ToServiceBindingEnvValue(ServiceTypeMessagingEventHubs, ServiceBindingInfoTypeConnectionString), + }, + { + Name: "spring.cloud.azure.eventhubs.credential.managed-identity-enabled", + Value: "false", + }, + { + Name: "spring.cloud.azure.eventhubs.credential.client-id", + Value: "", + }, + }, nil + default: + return []Env{}, unsupportedAuthTypeError(ServiceTypeMessagingEventHubs, eventHubs.AuthType) + } +} + +func GetServiceBindingEnvsForEventHubsKafka(eventHubs AzureDepEventHubs) ([]Env, error) { + var springBootVersionDecidedInformation []Env + if strings.HasPrefix(eventHubs.SpringBootVersion, "2.") { + springBootVersionDecidedInformation = []Env{ + { + Name: "spring.cloud.stream.binders.kafka.environment.spring.main.sources", + Value: "com.azure.spring.cloud.autoconfigure.eventhubs.kafka.AzureEventHubsKafkaAutoConfiguration", + }, + } + } else { + springBootVersionDecidedInformation = []Env{ + { + Name: "spring.cloud.stream.binders.kafka.environment.spring.main.sources", + Value: "com.azure.spring.cloud.autoconfigure.implementation.eventhubs.kafka" + + ".AzureEventHubsKafkaAutoConfiguration", + }, + } + } + var commonInformation []Env + switch eventHubs.AuthType { + case internal.AuthTypeUserAssignedManagedIdentity: + commonInformation = []Env{ + // Not add this: spring.cloud.azure.eventhubs.connection-string = "" + // because of this: https://github.com/Azure/azure-sdk-for-java/issues/42880 + { + Name: "spring.cloud.stream.kafka.binder.brokers", + Value: ToServiceBindingEnvValue(ServiceTypeMessagingEventHubs, ServiceBindingInfoTypeEndpoint), + }, + { + Name: "spring.cloud.azure.eventhubs.credential.managed-identity-enabled", + Value: "true", + }, + { + Name: "spring.cloud.azure.eventhubs.credential.client-id", + Value: PlaceHolderForServiceIdentityClientId(), + }, + } + case internal.AuthTypeConnectionString: + commonInformation = []Env{ + { + Name: "spring.cloud.stream.kafka.binder.brokers", + Value: ToServiceBindingEnvValue(ServiceTypeMessagingEventHubs, ServiceBindingInfoTypeEndpoint), + }, + { + Name: "spring.cloud.azure.eventhubs.connection-string", + Value: ToServiceBindingEnvValue(ServiceTypeMessagingEventHubs, ServiceBindingInfoTypeConnectionString), + }, + { + Name: "spring.cloud.azure.eventhubs.credential.managed-identity-enabled", + Value: "false", + }, + { + Name: "spring.cloud.azure.eventhubs.credential.client-id", + Value: "", + }, + } + default: + return []Env{}, unsupportedAuthTypeError(ServiceTypeMessagingEventHubs, eventHubs.AuthType) + } + return mergeEnvWithDuplicationCheck(springBootVersionDecidedInformation, commonInformation) +} + +func GetServiceBindingEnvsForAIModel() []Env { + return []Env{ + { + Name: "AZURE_OPENAI_ENDPOINT", + Value: ToServiceBindingEnvValue(ServiceTypeOpenAiModel, ServiceBindingInfoTypeEndpoint), + }, + } +} + +func unsupportedAuthTypeError(serviceType ServiceType, authType internal.AuthType) error { + return fmt.Errorf("unsupported auth type, serviceType = %s, authType = %s", serviceType, authType) +} + +func mergeEnvWithDuplicationCheck(a []Env, b []Env) ([]Env, error) { + ab := append(a, b...) + var result []Env + seenName := make(map[string]Env) + for _, value := range ab { + if existingValue, exist := seenName[value.Name]; exist { + if value != existingValue { + return []Env{}, duplicatedEnvError(existingValue, value) + } + } else { + seenName[value.Name] = value + result = append(result, value) + } + } + return result, nil +} + +func duplicatedEnvError(existingValue Env, newValue Env) error { + return fmt.Errorf( + "duplicated environment variable. existingValue = %s, newValue = %s", + existingValue, newValue, + ) +} diff --git a/cli/azd/internal/scaffold/spec_service_binding_test.go b/cli/azd/internal/scaffold/spec_service_binding_test.go new file mode 100644 index 00000000000..fc47c352a1e --- /dev/null +++ b/cli/azd/internal/scaffold/spec_service_binding_test.go @@ -0,0 +1,168 @@ +package scaffold + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToServiceBindingEnvName(t *testing.T) { + tests := []struct { + name string + inputResourceType ServiceType + inputResourceInfoType ServiceBindingInfoType + want string + }{ + { + name: "postgres password", + inputResourceType: ServiceTypeDbPostgres, + inputResourceInfoType: ServiceBindingInfoTypePassword, + want: "$service.binding:db.postgres:password", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ToServiceBindingEnvValue(tt.inputResourceType, tt.inputResourceInfoType) + assert.Equal(t, tt.want, actual) + }) + } +} + +func TestIsServiceBindingEnvName(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + { + name: "valid", + input: "$service.binding:db.postgres:password", + want: true, + }, + { + name: "invalid", + input: "$service.binding:db.postgres:", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isServiceBindingEnvValue(tt.input) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestToServiceTypeAndServiceBindingInfoType(t *testing.T) { + tests := []struct { + name string + input string + wantResourceType ServiceType + wantResourceInfoType ServiceBindingInfoType + }{ + { + name: "invalid input", + input: "$service.binding:db.mysql::username", + wantResourceType: "", + wantResourceInfoType: "", + }, + { + name: "postgres password", + input: "$service.binding:db.postgres:password", + wantResourceType: ServiceTypeDbPostgres, + wantResourceInfoType: ServiceBindingInfoTypePassword, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resourceType, resourceInfoType := toServiceTypeAndServiceBindingInfoType(tt.input) + assert.Equal(t, tt.wantResourceType, resourceType) + assert.Equal(t, tt.wantResourceInfoType, resourceInfoType) + }) + } +} + +func TestMergeEnvWithDuplicationCheck(t *testing.T) { + var empty []Env + name1Value1 := []Env{ + { + Name: "name1", + Value: "value1", + }, + } + name1Value2 := []Env{ + { + Name: "name1", + Value: "value2", + }, + } + name2Value2 := []Env{ + { + Name: "name2", + Value: "value2", + }, + } + name1Value1Name2Value2 := []Env{ + { + Name: "name1", + Value: "value1", + }, + { + Name: "name2", + Value: "value2", + }, + } + + tests := []struct { + name string + a []Env + b []Env + wantEnv []Env + wantError error + }{ + { + name: "2 empty array", + a: empty, + b: empty, + wantEnv: empty, + wantError: nil, + }, + { + name: "one is empty, another is not", + a: empty, + b: name1Value1, + wantEnv: name1Value1, + wantError: nil, + }, + { + name: "no duplication", + a: name1Value1, + b: name2Value2, + wantEnv: name1Value1Name2Value2, + wantError: nil, + }, + { + name: "duplicated name but same value", + a: name1Value1, + b: name1Value1, + wantEnv: name1Value1, + wantError: nil, + }, + { + name: "duplicated name, different value", + a: name1Value1, + b: name1Value2, + wantEnv: []Env{}, + wantError: fmt.Errorf("duplicated environment variable. existingValue = %s, newValue = %s", + name1Value1[0], name1Value2[0]), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env, err := mergeEnvWithDuplicationCheck(tt.a, tt.b) + assert.Equal(t, tt.wantEnv, env) + assert.Equal(t, tt.wantError, err) + }) + } +} diff --git a/cli/azd/pkg/project/importer.go b/cli/azd/pkg/project/importer.go index 26fbde3a07e..3494d76b81c 100644 --- a/cli/azd/pkg/project/importer.go +++ b/cli/azd/pkg/project/importer.go @@ -167,7 +167,7 @@ func (im *ImportManager) ProjectInfrastructure(ctx context.Context, projectConfi composeEnabled := im.dotNetImporter.alphaFeatureManager.IsEnabled(featureCompose) if composeEnabled && len(projectConfig.Resources) > 0 { - return tempInfra(ctx, projectConfig) + return tempInfra(ctx, projectConfig, im.dotNetImporter.console) } if !composeEnabled && len(projectConfig.Resources) > 0 { @@ -209,7 +209,7 @@ func (im *ImportManager) SynthAllInfrastructure(ctx context.Context, projectConf composeEnabled := im.dotNetImporter.alphaFeatureManager.IsEnabled(featureCompose) if composeEnabled && len(projectConfig.Resources) > 0 { - return infraFsForProject(ctx, projectConfig) + return infraFsForProject(ctx, projectConfig, im.dotNetImporter.console) } if !composeEnabled && len(projectConfig.Resources) > 0 { diff --git a/cli/azd/pkg/project/importer_test.go b/cli/azd/pkg/project/importer_test.go index 168e5c93261..c85626ac50a 100644 --- a/cli/azd/pkg/project/importer_test.go +++ b/cli/azd/pkg/project/importer_test.go @@ -405,11 +405,15 @@ func Test_ImportManager_ProjectInfrastructure_FromResources(t *testing.T) { im := &ImportManager{ dotNetImporter: &DotNetImporter{ alphaFeatureManager: alpha.NewFeaturesManagerWithConfig(config.NewEmptyConfig()), + console: mocks.NewMockContext(context.Background()).Console, }, } prjConfig := &ProjectConfig{} err := yaml.Unmarshal([]byte(prjWithResources), prjConfig) + for key, res := range prjConfig.Resources { + res.Name = key + } require.NoError(t, err) infra, err := im.ProjectInfrastructure(context.Background(), prjConfig) @@ -436,11 +440,15 @@ func TestImportManager_SynthAllInfrastructure_FromResources(t *testing.T) { im := &ImportManager{ dotNetImporter: &DotNetImporter{ alphaFeatureManager: alpha.NewFeaturesManagerWithConfig(config.NewEmptyConfig()), + console: mocks.NewMockContext(context.Background()).Console, }, } prjConfig := &ProjectConfig{} err := yaml.Unmarshal([]byte(prjWithResources), prjConfig) + for key, res := range prjConfig.Resources { + res.Name = key + } require.NoError(t, err) projectFs, err := im.SynthAllInfrastructure(context.Background(), prjConfig) diff --git a/cli/azd/pkg/project/resources.go b/cli/azd/pkg/project/resources.go index 9c1494ec15e..085ad13316c 100644 --- a/cli/azd/pkg/project/resources.go +++ b/cli/azd/pkg/project/resources.go @@ -5,6 +5,7 @@ package project import ( "fmt" + "github.com/azure/azure-dev/cli/azd/internal" "github.com/braydonk/yaml" ) @@ -22,11 +23,13 @@ func AllResourceTypes() []ResourceType { } const ( - ResourceTypeDbRedis ResourceType = "db.redis" - ResourceTypeDbPostgres ResourceType = "db.postgres" - ResourceTypeDbMongo ResourceType = "db.mongo" - ResourceTypeHostContainerApp ResourceType = "host.containerapp" - ResourceTypeOpenAiModel ResourceType = "ai.openai.model" + ResourceTypeDbRedis ResourceType = "db.redis" + ResourceTypeDbPostgres ResourceType = "db.postgres" + ResourceTypeDbMongo ResourceType = "db.mongo" + ResourceTypeHostContainerApp ResourceType = "host.containerapp" + ResourceTypeOpenAiModel ResourceType = "ai.openai.model" + ResourceTypeMessagingEventHubs ResourceType = "messaging.eventhubs" + ResourceTypeMessagingKafka ResourceType = "messaging.kafka" ) func (r ResourceType) String() string { @@ -41,6 +44,10 @@ func (r ResourceType) String() string { return "Container App" case ResourceTypeOpenAiModel: return "Open AI Model" + case ResourceTypeMessagingEventHubs: + return "Event Hubs" + case ResourceTypeMessagingKafka: + return "Kafka" } return "" @@ -89,6 +96,16 @@ func (r *ResourceConfig) MarshalYAML() (interface{}, error) { if err != nil { return nil, err } + case ResourceTypeMessagingEventHubs: + err := marshalRawProps(raw.Props.(EventHubsProps)) + if err != nil { + return nil, err + } + case ResourceTypeMessagingKafka: + err := marshalRawProps(raw.Props.(KafkaProps)) + if err != nil { + return nil, err + } } return raw, nil @@ -128,6 +145,18 @@ func (r *ResourceConfig) UnmarshalYAML(value *yaml.Node) error { return err } raw.Props = cap + case ResourceTypeMessagingEventHubs: + eh := EventHubsProps{} + if err := unmarshalProps(&eh); err != nil { + return err + } + raw.Props = eh + case ResourceTypeMessagingKafka: + kp := KafkaProps{} + if err := unmarshalProps(&kp); err != nil { + return err + } + raw.Props = kp } *r = ResourceConfig(raw) @@ -155,3 +184,14 @@ type AIModelPropsModel struct { Name string `yaml:"name,omitempty"` Version string `yaml:"version,omitempty"` } + +type EventHubsProps struct { + EventHubNames []string `yaml:"eventHubNames,omitempty"` + AuthType internal.AuthType `yaml:"authType,omitempty"` +} + +type KafkaProps struct { + Topics []string `yaml:"topics,omitempty"` + AuthType internal.AuthType `yaml:"authType,omitempty"` + SpringBootVersion string `yaml:"springBootVersion,omitempty"` +} diff --git a/cli/azd/pkg/project/scaffold_gen.go b/cli/azd/pkg/project/scaffold_gen.go index 120f1c63211..3695d7f8b30 100644 --- a/cli/azd/pkg/project/scaffold_gen.go +++ b/cli/azd/pkg/project/scaffold_gen.go @@ -6,6 +6,7 @@ package project import ( "context" "fmt" + "github.com/azure/azure-dev/cli/azd/pkg/input" "io/fs" "os" "path/filepath" @@ -19,13 +20,13 @@ import ( ) // Generates the in-memory contents of an `infra` directory. -func infraFs(_ context.Context, prjConfig *ProjectConfig) (fs.FS, error) { +func infraFs(ctx context.Context, prjConfig *ProjectConfig, console input.Console) (fs.FS, error) { t, err := scaffold.Load() if err != nil { return nil, fmt.Errorf("loading scaffold templates: %w", err) } - infraSpec, err := infraSpec(prjConfig) + infraSpec, err := infraSpec(prjConfig, console, ctx) if err != nil { return nil, fmt.Errorf("generating infrastructure spec: %w", err) } @@ -41,13 +42,13 @@ func infraFs(_ context.Context, prjConfig *ProjectConfig) (fs.FS, error) { // Returns the infrastructure configuration that points to a temporary, generated `infra` directory on the filesystem. func tempInfra( ctx context.Context, - prjConfig *ProjectConfig) (*Infra, error) { + prjConfig *ProjectConfig, console input.Console) (*Infra, error) { tmpDir, err := os.MkdirTemp("", "azd-infra") if err != nil { return nil, fmt.Errorf("creating temporary directory: %w", err) } - files, err := infraFs(ctx, prjConfig) + files, err := infraFs(ctx, prjConfig, console) if err != nil { return nil, err } @@ -89,8 +90,8 @@ func tempInfra( // Generates the filesystem of all infrastructure files to be placed, rooted at the project directory. // The content only includes `./infra` currently. -func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, error) { - infraFS, err := infraFs(ctx, prjConfig) +func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig, console input.Console) (fs.FS, error) { + infraFS, err := infraFs(ctx, prjConfig, console) if err != nil { return nil, err } @@ -130,10 +131,8 @@ func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, er return generatedFS, nil } -func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { +func infraSpec(projectConfig *ProjectConfig, console input.Console, ctx context.Context) (*scaffold.InfraSpec, error) { infraSpec := scaffold.InfraSpec{} - // backends -> frontends - backendMapping := map[string]string{} for _, res := range projectConfig.Resources { switch res.Type { @@ -153,17 +152,11 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { Name: res.Name, Port: -1, } - err := mapContainerApp(res, &svcSpec, &infraSpec) if err != nil { return nil, err } - - err = mapHostUses(res, &svcSpec, backendMapping, projectConfig) - if err != nil { - return nil, err - } - + svcSpec.Envs = append(svcSpec.Envs, serviceConfigEnv(projectConfig.Services[res.Name])...) infraSpec.Services = append(infraSpec.Services, svcSpec) case ResourceTypeOpenAiModel: props := res.Props.(AIModelProps) @@ -185,18 +178,15 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { } } - // create reverse frontends -> backends mapping - for i := range infraSpec.Services { - svc := &infraSpec.Services[i] - if front, ok := backendMapping[svc.Name]; ok { - if svc.Backend == nil { - svc.Backend = &scaffold.Backend{} - } - - svc.Backend.Frontends = append(svc.Backend.Frontends, scaffold.ServiceReference{Name: front}) - } + err := mapUses(&infraSpec, projectConfig) + if err != nil { + return nil, err } + err = printEnvListAboutUses(&infraSpec, projectConfig, console, ctx) + if err != nil { + return nil, err + } slices.SortFunc(infraSpec.Services, func(a, b scaffold.ServiceSpec) int { return strings.Compare(a.Name, b.Name) }) @@ -233,7 +223,10 @@ func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec, infraSp // Here, DB_HOST is not a secret, but DB_SECRET is. And yet, DB_HOST will be marked as a secret. // This is a limitation of the current implementation, but it's safer to mark both as secrets above. evaluatedValue := genBicepParamsFromEnvSubst(value, isSecret, infraSpec) - svcSpec.Env[envVar.Name] = evaluatedValue + err := scaffold.AddNewEnvironmentVariable(svcSpec, envVar.Name, evaluatedValue) + if err != nil { + return err + } } port := props.Port @@ -245,37 +238,101 @@ func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec, infraSp return nil } -func mapHostUses( - res *ResourceConfig, - svcSpec *scaffold.ServiceSpec, - backendMapping map[string]string, - prj *ProjectConfig) error { - for _, use := range res.Uses { - useRes, ok := prj.Resources[use] +func mapUses(infraSpec *scaffold.InfraSpec, projectConfig *ProjectConfig) error { + for i := range infraSpec.Services { + userSpec := &infraSpec.Services[i] + userResourceName := userSpec.Name + userResource, ok := projectConfig.Resources[userResourceName] if !ok { - return fmt.Errorf("resource %s uses %s, which does not exist", res.Name, use) + return fmt.Errorf("service (%s) exist, but there isn't a resource with that name", + userResourceName) } - - switch useRes.Type { - case ResourceTypeDbMongo: - svcSpec.DbCosmosMongo = &scaffold.DatabaseReference{DatabaseName: useRes.Name} - case ResourceTypeDbPostgres: - svcSpec.DbPostgres = &scaffold.DatabaseReference{DatabaseName: useRes.Name} - case ResourceTypeDbRedis: - svcSpec.DbRedis = &scaffold.DatabaseReference{DatabaseName: useRes.Name} - case ResourceTypeHostContainerApp: - if svcSpec.Frontend == nil { - svcSpec.Frontend = &scaffold.Frontend{} + for _, usedResourceName := range userResource.Uses { + usedResource, ok := projectConfig.Resources[usedResourceName] + if !ok { + return fmt.Errorf("in azure.yaml, (%s) uses (%s), but (%s) doesn't", + userResourceName, usedResourceName, usedResourceName) + } + var err error + switch usedResource.Type { + case ResourceTypeDbPostgres: + err = scaffold.BindToPostgres(userSpec, infraSpec.DbPostgres) + case ResourceTypeDbMongo: + err = scaffold.BindToMongoDb(userSpec, infraSpec.DbCosmosMongo) + case ResourceTypeDbRedis: + err = scaffold.BindToRedis(userSpec, infraSpec.DbRedis) + case ResourceTypeMessagingKafka, ResourceTypeMessagingEventHubs: + err = scaffold.BindToEventHubs(userSpec, infraSpec.AzureEventHubs) + case ResourceTypeOpenAiModel: + err = scaffold.BindToAIModels(userSpec, usedResource.Name) + case ResourceTypeHostContainerApp: + usedSpec := getServiceSpecByName(infraSpec, usedResource.Name) + if usedSpec == nil { + return fmt.Errorf("'%s' uses '%s', but %s doesn't exist", userSpec.Name, usedResource.Name, + usedResource.Name) + } + scaffold.BindToContainerApp(userSpec, usedSpec) + default: + return fmt.Errorf("resource (%s) uses (%s), but the type of (%s) is (%s), which is unsupported", + userResource.Name, usedResource.Name, usedResource.Name, usedResource.Type) + } + if err != nil { + return err } - - svcSpec.Frontend.Backends = append(svcSpec.Frontend.Backends, - scaffold.ServiceReference{Name: use}) - backendMapping[use] = res.Name // record the backend -> frontend mapping - case ResourceTypeOpenAiModel: - svcSpec.AIModels = append(svcSpec.AIModels, scaffold.AIModelReference{Name: use}) } } + return nil +} +func printEnvListAboutUses(infraSpec *scaffold.InfraSpec, projectConfig *ProjectConfig, + console input.Console, ctx context.Context) error { + for i := range infraSpec.Services { + userSpec := &infraSpec.Services[i] + userResourceName := userSpec.Name + userResource, ok := projectConfig.Resources[userResourceName] + if !ok { + return fmt.Errorf("service (%s) exist, but there isn't a resource with that name", + userResourceName) + } + for _, usedResourceName := range userResource.Uses { + usedResource, ok := projectConfig.Resources[usedResourceName] + if !ok { + return fmt.Errorf("in azure.yaml, (%s) uses (%s), but (%s) doesn't", + userResourceName, usedResourceName, usedResourceName) + } + console.Message(ctx, fmt.Sprintf("\nInformation about environment variables:\n"+ + "In azure.yaml, '%s' uses '%s'. \n"+ + "The 'uses' relationship is implemented by environment variables. \n"+ + "Please make sure your application used the right environment variable. \n"+ + "Here is the list of environment variables: ", + userResourceName, usedResourceName)) + var variables []scaffold.Env + var err error + switch usedResource.Type { + case ResourceTypeDbPostgres: + variables, err = scaffold.GetServiceBindingEnvsForPostgres() + case ResourceTypeDbMongo: + variables = scaffold.GetServiceBindingEnvsForMongo() + case ResourceTypeDbRedis: + variables = scaffold.GetServiceBindingEnvsForRedis() + case ResourceTypeMessagingKafka, ResourceTypeMessagingEventHubs: + variables, err = scaffold.GetServiceBindingEnvsForEventHubs(*infraSpec.AzureEventHubs) + case ResourceTypeHostContainerApp: + printHintsAboutUseHostContainerApp(userResourceName, usedResourceName, console, ctx) + default: + return fmt.Errorf("resource (%s) uses (%s), but the type of (%s) is (%s), "+ + "which is doesn't add necessary environment variable", + userResource.Name, usedResource.Name, usedResource.Name, usedResource.Type) + } + if err != nil { + return err + } + for _, variable := range variables { + console.Message(ctx, fmt.Sprintf(" %s=xxx", variable.Name)) + } + console.Message(ctx, "\n") + } + } return nil } @@ -348,3 +405,37 @@ func genBicepParamsFromEnvSubst( return result } + +func getServiceSpecByName(infraSpec *scaffold.InfraSpec, name string) *scaffold.ServiceSpec { + for i := range infraSpec.Services { + if infraSpec.Services[i].Name == name { + return &infraSpec.Services[i] + } + } + return nil +} + +// todo: merge it into scaffold.BindToContainerApp +func printHintsAboutUseHostContainerApp(userResourceName string, usedResourceName string, + console input.Console, ctx context.Context) { + if console == nil { + return + } + console.Message(ctx, fmt.Sprintf("Environment variables in %s:", userResourceName)) + console.Message(ctx, fmt.Sprintf("%s_BASE_URL=xxx", strings.ToUpper(usedResourceName))) + console.Message(ctx, fmt.Sprintf("Environment variables in %s:", usedResourceName)) + console.Message(ctx, fmt.Sprintf("%s_BASE_URL=xxx", strings.ToUpper(userResourceName))) +} + +func serviceConfigEnv(svcConfig *ServiceConfig) []scaffold.Env { + var envs []scaffold.Env + if svcConfig != nil { + for key, val := range svcConfig.Env { + envs = append(envs, scaffold.Env{ + Name: key, + Value: val, + }) + } + } + return envs +} diff --git a/cli/azd/pkg/project/service_config.go b/cli/azd/pkg/project/service_config.go index aa3cf7bf640..f1cc057234e 100644 --- a/cli/azd/pkg/project/service_config.go +++ b/cli/azd/pkg/project/service_config.go @@ -45,6 +45,8 @@ type ServiceConfig struct { DotNetContainerApp *DotNetContainerAppOptions `yaml:"-,omitempty"` // Custom configuration for the service target Config map[string]any `yaml:"config,omitempty"` + // Environment variables for service + Env map[string]string `yaml:"env,omitempty"` // Computed lazily by useDotnetPublishForDockerBuild and cached. This is true when the project // is a dotnet project and there is not an explicit Dockerfile in the project directory. useDotNetPublishForDockerBuild *bool diff --git a/cli/azd/resources/scaffold/base/modules/set-event-hubs-namespace-connection-string.bicep b/cli/azd/resources/scaffold/base/modules/set-event-hubs-namespace-connection-string.bicep new file mode 100644 index 00000000000..64245640096 --- /dev/null +++ b/cli/azd/resources/scaffold/base/modules/set-event-hubs-namespace-connection-string.bicep @@ -0,0 +1,21 @@ +param eventHubsNamespaceName string +param connectionStringSecretName string +param keyVaultName string + +resource eventHubsNamespace 'Microsoft.EventHub/namespaces@2024-01-01' existing = { + name: eventHubsNamespaceName +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: keyVaultName +} + +resource connectionStringSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { + name: connectionStringSecretName + parent: keyVault + properties: { + value: listKeys(concat(resourceId('Microsoft.EventHub/namespaces', eventHubsNamespaceName), '/AuthorizationRules/RootManageSharedAccessKey'), eventHubsNamespace.apiVersion).primaryConnectionString + } +} + +output keyVaultUrl string = 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${connectionStringSecretName}' diff --git a/cli/azd/resources/scaffold/templates/resources.bicept b/cli/azd/resources/scaffold/templates/resources.bicept index 26180abdc28..056dea69a73 100644 --- a/cli/azd/resources/scaffold/templates/resources.bicept +++ b/cli/azd/resources/scaffold/templates/resources.bicept @@ -112,7 +112,7 @@ module postgreServer 'br/public:avm/res/db-for-postgre-sql/flexible-server:0.1.4 tier: 'Burstable' // Non-required parameters administratorLogin: databaseUser - administratorLoginPassword: databasePassword + administratorLoginPassword: postgreSqlDatabasePassword geoRedundantBackup: 'Disabled' passwordAuth:'Enabled' firewallRules: [ @@ -172,7 +172,50 @@ resource localUserOpenAIIdentity 'Microsoft.Authorization/roleAssignments@2022-0 } {{- end}} -{{- range .Services}} + +{{- if .AzureEventHubs }} + +module eventHubNamespace 'br/public:avm/res/event-hub/namespace:0.7.1' = { + name: 'eventHubNamespace' + params: { + name: '${abbrs.eventHubNamespaces}${resourceToken}' + location: location + roleAssignments: [ + {{- range .Services}} + {{- if (and .AzureEventHubs (eq .AzureEventHubs.AuthType "userAssignedManagedIdentity")) }} + { + principalId: {{bicepName .Name}}Identity.outputs.principalId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f526a384-b230-433a-b45c-95f59c4a2dec') + } + {{- end}} + {{- end}} + ] + {{- if (and .AzureEventHubs (eq .AzureEventHubs.AuthType "connectionString")) }} + disableLocalAuth: false + {{- end}} + eventhubs: [ + {{- range $eventHubName := .AzureEventHubs.EventHubNames}} + { + name: '{{ $eventHubName }}' + } + {{- end}} + ] + } +} +{{- if (and .AzureEventHubs (eq .AzureEventHubs.AuthType "connectionString")) }} +module eventHubsConnectionString './modules/set-event-hubs-namespace-connection-string.bicep' = { + name: 'eventHubsConnectionString' + params: { + eventHubsNamespaceName: eventHubNamespace.outputs.name + connectionStringSecretName: 'EVENT-HUBS-CONNECTION-STRING' + keyVaultName: keyVault.outputs.name + } +} +{{end}} +{{end}} + +{{- range $service := .Services}} module {{bicepName .Name}}Identity 'br/public:avm/res/managed-identity/user-assigned-identity:0.2.1' = { name: '{{bicepName .Name}}identity' @@ -234,41 +277,28 @@ module {{bicepName .Name}} 'br/public:avm/res/app/container-app:0.8.0' = { scaleMinReplicas: 1 scaleMaxReplicas: 10 secrets: { - secureList: union([ - {{- if .DbCosmosMongo}} - { - name: 'mongodb-url' - identity:{{bicepName .Name}}Identity.outputs.resourceId - keyVaultUrl: cosmos.outputs.exportedSecrets['MONGODB-URL'].secretUri - } - {{- end}} - {{- if .DbPostgres}} - { - name: 'db-pass' - value: databasePassword - } - { - name: 'db-url' - value: 'postgresql://${databaseUser}:${databasePassword}@${postgreServer.outputs.fqdn}:5432/${databaseName}' - } - {{- end}} - {{- if .DbRedis}} - { - name: 'redis-pass' - identity:{{bicepName .Name}}Identity.outputs.resourceId - keyVaultUrl: '${keyVault.outputs.uri}secrets/REDIS-PASSWORD' - } - { - name: 'redis-url' - identity:{{bicepName .Name}}Identity.outputs.resourceId - keyVaultUrl: '${keyVault.outputs.uri}secrets/REDIS-URL' - } - {{- end}} - ], - map({{bicepName .Name}}Secrets, secret => { - name: secret.secretRef - value: secret.value - })) + secureList: union([ + {{- range $env := .Envs}} + {{- if (eq (toBicepEnv $env).BicepEnvType "keyVaultSecret") }} + { + name: '{{ (toBicepEnv $env).SecretName }}' + identity:{{bicepName $service.Name}}Identity.outputs.resourceId + keyVaultUrl: {{ (toBicepEnv $env).SecretValue }} + } + {{- end}} + {{- if (eq (toBicepEnv $env).BicepEnvType "secret") }} + { + name: '{{ (toBicepEnv $env).SecretName }}' + value: {{ (toBicepEnv $env).SecretValue }} + } + {{- end}} + {{- end}} + ], + map({{bicepName .Name}}Secrets, secret => { + name: secret.secretRef + value: secret.value + }) + ) } containers: [ { @@ -438,7 +468,7 @@ module keyVault 'br/public:avm/res/key-vault/vault:0.6.1' = { {{- if .DbPostgres}} { name: 'db-pass' - value: databasePassword + value: postgreSqlDatabasePassword } {{- end}} ]