From 91f2b2dd84eed9851235d55c2cea417f79b2e9b9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 3 Jan 2025 14:22:10 +0100 Subject: [PATCH] Add options for API key in clients --- README.md | 5 +- cmd/benchmark.go | 4 +- cmd/stack.go | 1 + internal/benchrunner/runners/common/env.go | 1 + internal/elasticsearch/client.go | 9 +++ internal/kibana/client.go | 14 +++- internal/stack/clients.go | 17 +++++ internal/stack/config.go | 1 + internal/stack/initconfig.go | 2 + internal/stack/shellinit.go | 71 +++++++++++-------- ...nit_internal_test.go => shellinit_test.go} | 37 ++++++++-- tools/readme/readme.md.tmpl | 4 +- 12 files changed, 126 insertions(+), 40 deletions(-) rename internal/stack/{shellinit_internal_test.go => shellinit_test.go} (61%) diff --git a/README.md b/README.md index e75d646413..8f1dec73f6 100644 --- a/README.md +++ b/README.md @@ -480,6 +480,7 @@ The output of this command is intended to be evaluated by the current shell. For Relevant environment variables are: +- ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY - ELASTIC_PACKAGE_ELASTICSEARCH_HOST - ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME - ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD @@ -690,13 +691,15 @@ There are available some environment variables that could be used to change some - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) - - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch (e.g. elastic) + - `ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY`: API key to connect to elasticsearch and kibana. When set it takes precedence over username and password. + - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch and kibana (e.g. elastic) - `ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD`: Password of that user. - `ELASTIC_PACKAGE_ELASTICSEARCH_KIBANA_HOST`: Kibana URL (e.g. https://127.0.0.1:5601) - `ELASTIC_PACKAGE_ELASTICSEARCH_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services. - To configure an external metricstore while running benchmarks (more info at [system benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_benchmarking.md#setting-up-an-external-metricstore) or [rally benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/rally_benchmarking.md#setting-up-an-external-metricstore)): - `ELASTIC_PACKAGE_ESMETRICSTORE_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) + - `ELASTIC_PACKAGE_ESMETRICSTORE_API_KEY`: API key to connect to elasticsearch and kibana. When set it takes precedence over username and password. - `ELASTIC_PACKAGE_ESMETRICSTORE_USERNAME`: Username to connect to elasticsearch (e.g. elastic) - `ELASTIC_PACKAGE_ESMETRICSTORE_PASSWORD`: Password for the user. - `ELASTIC_PACKAGE_ESMETRICSTORE_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services. diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 37fc4ae512..07e993e208 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -655,16 +655,18 @@ func systemCommandAction(cmd *cobra.Command, args []string) error { func initializeESMetricsClient(ctx context.Context) (*elasticsearch.Client, error) { address := os.Getenv(benchcommon.ESMetricstoreHostEnv) + apiKey := os.Getenv(benchcommon.ESMetricstoreAPIKeyEnv) user := os.Getenv(benchcommon.ESMetricstoreUsernameEnv) pass := os.Getenv(benchcommon.ESMetricstorePasswordEnv) cacert := os.Getenv(benchcommon.ESMetricstoreCACertificateEnv) - if address == "" || user == "" || pass == "" { + if address == "" || ((user == "" || pass == "") && apiKey == "") { logger.Debugf("can't initialize metricstore, missing environment configuration") return nil, nil } esClient, err := stack.NewElasticsearchClient( elasticsearch.OptionWithAddress(address), + elasticsearch.OptionWithAPIKey(apiKey), elasticsearch.OptionWithUsername(user), elasticsearch.OptionWithPassword(pass), elasticsearch.OptionWithCertificateAuthority(cacert), diff --git a/cmd/stack.go b/cmd/stack.go index 63899ba2eb..df55bfae70 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -55,6 +55,7 @@ The output of this command is intended to be evaluated by the current shell. For Relevant environment variables are: +- ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY - ELASTIC_PACKAGE_ELASTICSEARCH_HOST - ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME - ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD diff --git a/internal/benchrunner/runners/common/env.go b/internal/benchrunner/runners/common/env.go index b084c3ed92..390df38fbd 100644 --- a/internal/benchrunner/runners/common/env.go +++ b/internal/benchrunner/runners/common/env.go @@ -7,6 +7,7 @@ package common import "github.com/elastic/elastic-package/internal/environment" var ( + ESMetricstoreAPIKeyEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_API_KEY") ESMetricstoreHostEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_HOST") ESMetricstoreUsernameEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_USERNAME") ESMetricstorePasswordEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_PASSWORD") diff --git a/internal/elasticsearch/client.go b/internal/elasticsearch/client.go index 8a95f8cb7e..f4c94e7d5d 100644 --- a/internal/elasticsearch/client.go +++ b/internal/elasticsearch/client.go @@ -35,6 +35,7 @@ type ClusterStateRequest = esapi.ClusterStateRequest // clientOptions are used to configure a client. type clientOptions struct { address string + apiKey string username string password string @@ -47,6 +48,13 @@ type clientOptions struct { type ClientOption func(*clientOptions) +// OptionWithAPIKey sets the API key to be used by the client for authentication. +func OptionWithAPIKey(apiKey string) ClientOption { + return func(opts *clientOptions) { + opts.apiKey = apiKey + } +} + // OptionWithAddress sets the address to be used by the client. func OptionWithAddress(address string) ClientOption { return func(opts *clientOptions) { @@ -109,6 +117,7 @@ func NewConfig(customOptions ...ClientOption) (elasticsearch.Config, error) { config := elasticsearch.Config{ Addresses: []string{options.address}, + APIKey: options.apiKey, Username: options.username, Password: options.password, } diff --git a/internal/kibana/client.go b/internal/kibana/client.go index 6d4db93ce7..74e7dd9bff 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -27,6 +27,7 @@ var ErrUndefinedHost = errors.New("missing kibana host") // Client is responsible for exporting dashboards from Kibana. type Client struct { host string + apiKey string username string password string @@ -94,6 +95,13 @@ func Address(address string) ClientOption { } } +// APIKey option sets the API key to be used by the client for authentication. +func APIKey(apiKey string) ClientOption { + return func(c *Client) { + c.apiKey = apiKey + } +} + // TLSSkipVerify option disables TLS verification. func TLSSkipVerify() ClientOption { return func(c *Client) { @@ -182,7 +190,11 @@ func (c *Client) newRequest(ctx context.Context, method, resourcePath string, re return nil, fmt.Errorf("could not create %v request to Kibana API resource: %s: %w", method, resourcePath, err) } - req.SetBasicAuth(c.username, c.password) + if c.apiKey != "" { + req.Header.Set("Authorization", "ApiKey "+c.apiKey) + } else { + req.SetBasicAuth(c.username, c.password) + } req.Header.Add("content-type", "application/json") req.Header.Add("kbn-xsrf", install.DefaultStackVersion) diff --git a/internal/stack/clients.go b/internal/stack/clients.go index 8c948d8620..b3915ec737 100644 --- a/internal/stack/clients.go +++ b/internal/stack/clients.go @@ -21,6 +21,7 @@ import ( func NewElasticsearchClient(customOptions ...elasticsearch.ClientOption) (*elasticsearch.Client, error) { options := []elasticsearch.ClientOption{ elasticsearch.OptionWithAddress(os.Getenv(ElasticsearchHostEnv)), + elasticsearch.OptionWithAPIKey(os.Getenv(ElasticsearchAPIKeyEnv)), elasticsearch.OptionWithPassword(os.Getenv(ElasticsearchPasswordEnv)), elasticsearch.OptionWithUsername(os.Getenv(ElasticsearchUsernameEnv)), elasticsearch.OptionWithCertificateAuthority(os.Getenv(CACertificateEnv)), @@ -58,6 +59,10 @@ func NewElasticsearchClientFromProfile(profile *profile.Profile, customOptions . elasticsearchHost = profileConfig.ElasticsearchHostPort logger.Debugf("Connecting with Elasticsearch host from current profile (profile: %s, host: %q)", profile.ProfileName, elasticsearchHost) } + elasticsearchAPIKey, found := os.LookupEnv(ElasticsearchAPIKeyEnv) + if !found { + elasticsearchAPIKey = profileConfig.ElasticsearchAPIKey + } elasticsearchPassword, found := os.LookupEnv(ElasticsearchPasswordEnv) if !found { elasticsearchPassword = profileConfig.ElasticsearchPassword @@ -73,6 +78,7 @@ func NewElasticsearchClientFromProfile(profile *profile.Profile, customOptions . options := []elasticsearch.ClientOption{ elasticsearch.OptionWithAddress(elasticsearchHost), + elasticsearch.OptionWithAPIKey(elasticsearchAPIKey), elasticsearch.OptionWithPassword(elasticsearchPassword), elasticsearch.OptionWithUsername(elasticsearchUsername), elasticsearch.OptionWithCertificateAuthority(caCertificate), @@ -86,6 +92,7 @@ func NewElasticsearchClientFromProfile(profile *profile.Profile, customOptions . func NewKibanaClient(customOptions ...kibana.ClientOption) (*kibana.Client, error) { options := []kibana.ClientOption{ kibana.Address(os.Getenv(KibanaHostEnv)), + kibana.APIKey(os.Getenv(ElasticsearchAPIKeyEnv)), kibana.Password(os.Getenv(ElasticsearchPasswordEnv)), kibana.Username(os.Getenv(ElasticsearchUsernameEnv)), kibana.CertificateAuthority(os.Getenv(CACertificateEnv)), @@ -123,6 +130,10 @@ func NewKibanaClientFromProfile(profile *profile.Profile, customOptions ...kiban kibanaHost = profileConfig.KibanaHostPort logger.Debugf("Connecting with Kibana host from current profile (profile: %s, host: %q)", profile.ProfileName, kibanaHost) } + elasticsearchAPIKey, found := os.LookupEnv(ElasticsearchAPIKeyEnv) + if !found { + elasticsearchAPIKey = profileConfig.ElasticsearchAPIKey + } elasticsearchPassword, found := os.LookupEnv(ElasticsearchPasswordEnv) if !found { elasticsearchPassword = profileConfig.ElasticsearchPassword @@ -138,6 +149,7 @@ func NewKibanaClientFromProfile(profile *profile.Profile, customOptions ...kiban options := []kibana.ClientOption{ kibana.Address(kibanaHost), + kibana.APIKey(elasticsearchAPIKey), kibana.Password(elasticsearchPassword), kibana.Username(elasticsearchUsername), kibana.CertificateAuthority(caCertificate), @@ -158,5 +170,10 @@ func FindCACertificate(profile *profile.Profile) (string, error) { caCertPath = profileConfig.CACertificatePath } + // Avoid returning an empty certificate path, fallback to the default path. + if caCertPath == "" { + caCertPath = profile.Path(CACertificateFile) + } + return caCertPath, nil } diff --git a/internal/stack/config.go b/internal/stack/config.go index 31ab437ac8..db97653229 100644 --- a/internal/stack/config.go +++ b/internal/stack/config.go @@ -20,6 +20,7 @@ type Config struct { Provider string `json:"provider,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` + ElasticsearchAPIKey string `json:"elasticsearch_api_key,omitempty"` ElasticsearchHost string `json:"elasticsearch_host,omitempty"` ElasticsearchUsername string `json:"elasticsearch_username,omitempty"` ElasticsearchPassword string `json:"elasticsearch_password,omitempty"` diff --git a/internal/stack/initconfig.go b/internal/stack/initconfig.go index 916db46be8..23b7c4c91a 100644 --- a/internal/stack/initconfig.go +++ b/internal/stack/initconfig.go @@ -9,6 +9,7 @@ import ( ) type InitConfig struct { + ElasticsearchAPIKey string ElasticsearchHostPort string ElasticsearchUsername string ElasticsearchPassword string @@ -23,6 +24,7 @@ func StackInitConfig(profile *profile.Profile) (*InitConfig, error) { } return &InitConfig{ + ElasticsearchAPIKey: config.ElasticsearchAPIKey, ElasticsearchHostPort: config.ElasticsearchHost, ElasticsearchUsername: config.ElasticsearchUsername, ElasticsearchPassword: config.ElasticsearchPassword, diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index a3db128a6e..73e1d899ac 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -20,9 +20,10 @@ import ( // Environment variables describing the stack. var ( + ElasticsearchAPIKeyEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_API_KEY") ElasticsearchHostEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_HOST") - ElasticsearchUsernameEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_USERNAME") ElasticsearchPasswordEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_PASSWORD") + ElasticsearchUsernameEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_USERNAME") KibanaHostEnv = environment.WithElasticPackagePrefix("KIBANA_HOST") CACertificateEnv = environment.WithElasticPackagePrefix("CA_CERT") ) @@ -38,60 +39,70 @@ func ShellInit(elasticStackProfile *profile.Profile, shellType string) (string, if err != nil { return "", nil } + return shellInitWithConfig(config, shellType) +} - // NOTE: to add new env vars, the template need to be adjusted - t, err := initTemplate(shellType) +func shellInitWithConfig(config *InitConfig, shellType string) (string, error) { + pattern, err := selectPattern(shellType) if err != nil { return "", fmt.Errorf("cannot get shell init template: %w", err) } - return fmt.Sprintf(t, - ElasticsearchHostEnv, config.ElasticsearchHostPort, - ElasticsearchUsernameEnv, config.ElasticsearchUsername, - ElasticsearchPasswordEnv, config.ElasticsearchPassword, - KibanaHostEnv, config.KibanaHostPort, - CACertificateEnv, config.CACertificatePath, - ), nil + template := genTemplate(pattern) + return template([]generatorEnvVar{ + {ElasticsearchAPIKeyEnv, config.ElasticsearchAPIKey}, + {ElasticsearchHostEnv, config.ElasticsearchHostPort}, + {ElasticsearchUsernameEnv, config.ElasticsearchUsername}, + {ElasticsearchPasswordEnv, config.ElasticsearchPassword}, + {KibanaHostEnv, config.KibanaHostPort}, + {CACertificateEnv, config.CACertificatePath}, + }), nil +} + +type generatorEnvVar struct { + name string + value string } const ( // shell init code for POSIX compliant shells. // IEEE POSIX Shell and Tools portion of the IEEE POSIX specification (IEEE Standard 1003.1) - posixTemplate = `export %s=%s -export %s=%s -export %s=%s -export %s=%s -export %s=%s` + posixPattern = `export %s=%s` // fish shell init code. // fish shell is similar but not compliant to POSIX. - fishTemplate = `set -x %s %s; -set -x %s %s; -set -x %s %s; -set -x %s %s; -set -x %s %s;` + fishPattern = `set -x %s %s;` // PowerShell init code. // Output to be evaluated with `elastic-package stack shellinit | Invoke-Expression - powershellTemplate = `$Env:%s="%s"; -$Env:%s="%s"; -$Env:%s="%s"; -$Env:%s="%s"; -$Env:%s="%s";` + powershellPattern = `$Env:%s="%s";` ) +func genTemplate(pattern string) func([]generatorEnvVar) string { + return func(vars []generatorEnvVar) string { + var builder strings.Builder + for i, v := range vars { + fmt.Fprintf(&builder, pattern, v.name, v.value) + if i < len(vars)-1 { + builder.WriteString("\n") + } + } + return builder.String() + } +} + // availableShellTypes list all available values for s in initTemplate var availableShellTypes = []string{"bash", "dash", "fish", "sh", "zsh", "pwsh", "powershell"} -// InitTemplate returns code templates for shell initialization -func initTemplate(s string) (string, error) { +// SelectPattern returns the patterns to generate list of environment variables for each shell. +func selectPattern(s string) (string, error) { switch s { case "bash", "dash", "sh", "zsh": - return posixTemplate, nil + return posixPattern, nil case "fish": - return fishTemplate, nil + return fishPattern, nil case "pwsh", "powershell": - return powershellTemplate, nil + return powershellPattern, nil default: return "", errors.New("shell type is unknown, should be one of " + strings.Join(availableShellTypes, ", ")) } diff --git a/internal/stack/shellinit_internal_test.go b/internal/stack/shellinit_test.go similarity index 61% rename from internal/stack/shellinit_internal_test.go rename to internal/stack/shellinit_test.go index cedc3a8b25..99f1ce3c0d 100644 --- a/internal/stack/shellinit_internal_test.go +++ b/internal/stack/shellinit_test.go @@ -22,22 +22,47 @@ func TestCodeTemplate(t *testing.T) { args args want string }{ - {"bash code template", args{"bash"}, posixTemplate}, - {"fish code template", args{"fish"}, fishTemplate}, - {"sh code template", args{"sh"}, posixTemplate}, - {"zsh code template", args{"zsh"}, posixTemplate}, + {"bash code template", args{"bash"}, posixPattern}, + {"fish code template", args{"fish"}, fishPattern}, + {"sh code template", args{"sh"}, posixPattern}, + {"zsh code template", args{"zsh"}, posixPattern}, + {"pwsh code template", args{"pwsh"}, powershellPattern}, + {"powershell code template", args{"powershell"}, powershellPattern}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, _ := initTemplate(tt.args.s); got != tt.want { + if got, _ := selectPattern(tt.args.s); got != tt.want { t.Errorf("CodeTemplate() = %v, want %v", got, tt.want) } }) } } +func TestShellInit(t *testing.T) { + config := InitConfig{ + ElasticsearchHostPort: "https://elastic.example.com:9200", + ElasticsearchUsername: "admin", + ElasticsearchPassword: "secret", + KibanaHostPort: "https://kibana.example.com:5601", + } + + expected := strings.TrimSpace(` +export ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY= +export ELASTIC_PACKAGE_ELASTICSEARCH_HOST=https://elastic.example.com:9200 +export ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME=admin +export ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD=secret +export ELASTIC_PACKAGE_KIBANA_HOST=https://kibana.example.com:5601 +export ELASTIC_PACKAGE_CA_CERT= +`) + + result, err := shellInitWithConfig(&config, "bash") + require.NoError(t, err) + + assert.Equal(t, expected, result) +} + func TestCodeTemplate_wrongInput(t *testing.T) { - _, err := initTemplate("invalid shell type") + _, err := selectPattern("invalid shell type") assert.Error(t, err, "shell type is unknown, should be one of "+strings.Join(availableShellTypes, ", ")) } diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index 0a4afdd374..8070afd434 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -250,13 +250,15 @@ There are available some environment variables that could be used to change some - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) - - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch (e.g. elastic) + - `ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY`: API key to connect to elasticsearch and kibana. When set it takes precedence over username and password. + - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch and kibana (e.g. elastic) - `ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD`: Password of that user. - `ELASTIC_PACKAGE_ELASTICSEARCH_KIBANA_HOST`: Kibana URL (e.g. https://127.0.0.1:5601) - `ELASTIC_PACKAGE_ELASTICSEARCH_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services. - To configure an external metricstore while running benchmarks (more info at [system benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_benchmarking.md#setting-up-an-external-metricstore) or [rally benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/rally_benchmarking.md#setting-up-an-external-metricstore)): - `ELASTIC_PACKAGE_ESMETRICSTORE_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) + - `ELASTIC_PACKAGE_ESMETRICSTORE_API_KEY`: API key to connect to elasticsearch and kibana. When set it takes precedence over username and password. - `ELASTIC_PACKAGE_ESMETRICSTORE_USERNAME`: Username to connect to elasticsearch (e.g. elastic) - `ELASTIC_PACKAGE_ESMETRICSTORE_PASSWORD`: Password for the user. - `ELASTIC_PACKAGE_ESMETRICSTORE_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services.