From 50eff6810e92f793fe296d204568e9dd861cec24 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 12 Apr 2024 16:09:45 +0200 Subject: [PATCH 1/3] Add profile setting to enable serverless in compose --- internal/stack/_static/docker-compose-stack.yml.tmpl | 3 ++- internal/stack/resources.go | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl index 19f6feda8..9c51f2ad7 100644 --- a/internal/stack/_static/docker-compose-stack.yml.tmpl +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -1,6 +1,7 @@ {{ $username := fact "username" }} {{ $password := fact "password" }} {{ $apm_enabled := fact "apm_enabled" }} +{{ $serverless_enabled := fact "serverless_enabled" }} version: '2.4' services: elasticsearch: @@ -10,7 +11,7 @@ services: start_period: 300s interval: 5s environment: - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g -Des.serverless={{ $serverless_enabled }}" - "ELASTIC_PASSWORD={{ $password }}" volumes: - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 441283f2d..e3e69a9da 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -61,6 +61,7 @@ const ( configGeoIPDir = "stack.geoip_dir" configLogstashEnabled = "stack.logstash_enabled" configSelfMonitorEnabled = "stack.self_monitor_enabled" + configServerlessEnabled = "stack.serverless_enabled" ) var ( @@ -152,6 +153,7 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "geoip_dir": profile.Config(configGeoIPDir, "./ingest-geoip"), "logstash_enabled": profile.Config(configLogstashEnabled, "false"), "self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"), + "serverless_enabled": profile.Config(configServerlessEnabled, "false"), }) os.MkdirAll(stackDir, 0755) From aa0dc68a34db5a7a923ba00ebc2a03ebe3c72d80 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 6 May 2024 17:33:45 +0200 Subject: [PATCH 2/3] Add kibana configuration --- .../_static/docker-compose-stack.yml.tmpl | 1 - internal/stack/_static/kibana.yml.tmpl | 186 +++++++++++++++++- 2 files changed, 182 insertions(+), 5 deletions(-) diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl index 9dd75b805..442989913 100644 --- a/internal/stack/_static/docker-compose-stack.yml.tmpl +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -2,7 +2,6 @@ {{ $password := fact "password" }} {{ $apm_enabled := fact "apm_enabled" }} {{ $serverless_enabled := fact "serverless_enabled" }} -{{ $serverless_project_type := fact "serverless_project_type" }} services: elasticsearch: image: "${ELASTICSEARCH_IMAGE_REF}" diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index d9de005d8..2604b478e 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -43,10 +43,6 @@ xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable expe xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" -{{ if not (semverLessThan $version "8.2.0") }} -xpack.cloudSecurityPosture.enabled: true -{{ end }} - {{ $self_monitor_enabled := fact "self_monitor_enabled" }} {{ if not (semverLessThan $version "8.0.0") }} xpack.fleet.packages: @@ -137,3 +133,185 @@ xpack.fleet.outputs: {{ indent $agent_key " " }} {{ end }} {{ end }} + +{{ $serverless_enabled := fact "serverless_enabled" }} +{{ $serverless_project_type := fact "serverless_project_type" }} +{{ if eq $serverless_enabled "true" }} + {{ if eq $serverless_project_type "observability" }} +# Observability Project config + +## Disable plugins +enterpriseSearch.enabled: false +xpack.cloudSecurityPosture.enabled: false +xpack.infra.enabled: true +xpack.securitySolution.enabled: false + +## Cloud settings +xpack.cloud.serverless.project_type: observability + +## Enable the Serverless Observability plugin +xpack.serverless.observability.enabled: true + +## Configure plugins + +## Set the home route +uiSettings.overrides.defaultRoute: /app/observability/landing + +# Customize empty page state for analytics apps +#no_data_page.analyticsNoDataPageFlavor: 'serverless_observability' + +## Set the dev project switch current type +xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability' + +## Disable adding the component template `.fleet_agent_id_verification-1` to every index template for each datastream for each integration +xpack.fleet.agentIdVerificationEnabled: false + +## Enable the capability for the observability feature ID in the serverless environment to take ownership of the rules. +## The value need to be a featureId observability Or stackAlerts Or siem +xpack.alerting.rules.overwriteProducer: 'observability' +xpack.observability.createO11yGenericFeatureId: true + +## APM Serverless Onboarding flow +#xpack.apm.serverlessOnboarding: true + +# Synthetics mTLS cert locations +#xpack.uptime.service.tls.certificate: /mnt/elastic-internal/http-certs/tls.crt +#xpack.uptime.service.tls.key: /mnt/elastic-internal/http-certs/tls.key + +# Fleet specific configuration +xpack.fleet.internal.registry.capabilities: [ + 'apm', + 'observability', + 'uptime', +] +xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false +xpack.fleet.internal.registry.spec.max: '3.0' +# Temporary until all packages implement new spec https://github.com/elastic/kibana/issues/166742 +xpack.fleet.internal.registry.spec.min: '1.0' +xpack.fleet.internal.registry.excludePackages: [ + # Security integrations + 'endpoint', + 'beaconing', + 'osquery_manager', + + # Removed in 8.11 integrations + 'cisco', + 'microsoft', + 'symantec', + 'cyberark', + + # ML integrations + 'dga', + + # Profiling integrations + 'profiler_agent', + ] + +## Required for force installation of integration packages +#xpack.fleet.packages: +# # fleet_server package installed to publish agent metrics +# - name: fleet_server +# version: latest +## Disable APM UI components and API calls +#xpack.apm.featureFlags.agentConfigurationAvailable: false +xpack.apm.featureFlags.agentConfigurationAvailable: true +#xpack.apm.featureFlags.configurableIndicesAvailable: true +xpack.apm.featureFlags.infrastructureTabAvailable: true +xpack.apm.featureFlags.infraUiAvailable: true +#xpack.apm.featureFlags.migrationToFleetAvailable: false +xpack.apm.featureFlags.migrationToFleetAvailable: true +#xpack.apm.featureFlags.sourcemapApiAvailable: false +xpack.apm.featureFlags.sourcemapApiAvailable: true +#xpack.apm.featureFlags.storageExplorerAvailable: false +xpack.apm.featureFlags.storageExplorerAvailable: true + +# Specify in telemetry the project type +#telemetry.labels.serverless: observability + +xpack.ml.ad.enabled: true +xpack.ml.dfa.enabled: false +xpack.ml.nlp.enabled: false +xpack.ml.compatibleModuleType: 'observability' + +# Disable the embedded Dev Console +console.ui.embeddedEnabled: false + {{ end }} + {{ if eq $serverless_project_type "security" }} +# Security Project config + +## Disable plugins +enterpriseSearch.enabled: false +xpack.apm.enabled: false +xpack.infra.enabled: false +xpack.observabilityLogsExplorer.enabled: false +xpack.observability.enabled: false + +## Cloud settings +xpack.cloud.serverless.project_type: security + +## Enable the Security Solution Serverless plugin +xpack.securitySolutionServerless.enabled: true +xpack.securitySolutionServerless.productTypes: + [ + { product_line: 'security', product_tier: 'complete' }, + { product_line: 'endpoint', product_tier: 'complete' }, + { product_line: 'cloud', product_tier: 'complete' }, + ] + +xpack.securitySolution.offeringSettings: { + sideNavEnabled: false, # Internal security side navigation disabled, the serverless global chrome navigation is used instead + ILMEnabled: false, # Index Lifecycle Management (ILM) functionalities disabled, not supported by serverless Elasticsearch + ESQLEnabled: false, # ES|QL disabled, not supported by serverless Elasticsearch + } + +newsfeed.enabled: true + +## Set the home route +uiSettings.overrides.defaultRoute: /app/security/get_started + +## Set the dev project switcher current type +xpack.serverless.plugin.developer.projectSwitcher.currentType: 'security' + +# Specify in telemetry the project type +#telemetry.labels.serverless: security + +# Fleet specific configuration +xpack.fleet.internal.registry.capabilities: ['security'] +xpack.fleet.internal.registry.spec.max: '3.0' +xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false +# Temporary until all packages implement new spec https://github.com/elastic/kibana/issues/166742 +xpack.fleet.internal.registry.spec.min: '1.0' +xpack.fleet.internal.registry.excludePackages: [ + # Oblt integrations + 'apm', + 'synthetics', + 'synthetics_dashboards', + + # Removed in 8.11 integrations + 'cisco', + 'microsoft', + 'symantec', + 'cyberark', + + # ML integrations + 'dga', + ] +# fleet_server package installed to publish agent metrics +#xpack.fleet.packages: +# - name: fleet_server +# version: latest + +xpack.ml.ad.enabled: true +xpack.ml.dfa.enabled: true +xpack.ml.nlp.enabled: false +xpack.ml.compatibleModuleType: 'security' + +# Disable the embedded Dev Console +console.ui.embeddedEnabled: false + {{ end }} +{{ else }} + {{ if not (semverLessThan $version "8.2.0") }} +xpack.cloudSecurityPosture.enabled: true + {{ end }} +{{ end }} + From 9459d498bb2c45a4e526f4d65eb80fe8e63fdd90 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 10 May 2024 13:43:40 +0200 Subject: [PATCH 3/3] Use serverless images --- .../_static/docker-compose-stack.yml.tmpl | 10 +- internal/stack/_static/elasticsearch.yml.tmpl | 8 + internal/stack/_static/kibana.yml.tmpl | 178 +----------------- 3 files changed, 23 insertions(+), 173 deletions(-) diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl index eddac4fc7..0eda95a37 100644 --- a/internal/stack/_static/docker-compose-stack.yml.tmpl +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -4,13 +4,17 @@ {{ $serverless_enabled := fact "serverless_enabled" }} services: elasticsearch: +{{ if eq $serverless_enabled "true" }} + image: "docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified" +{{ else }} image: "${ELASTICSEARCH_IMAGE_REF}" +{{ end }} healthcheck: test: "curl -s --cacert /usr/share/elasticsearch/config/certs/ca-cert.pem -f -u {{ $username }}:{{ $password }} https://127.0.0.1:9200/_cat/health | cut -f4 -d' ' | grep -E '(green|yellow)'" start_period: 300s interval: 5s environment: - - "ES_JAVA_OPTS=-Xms1g -Xmx1g -Des.serverless={{ $serverless_enabled }}" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "ELASTIC_PASSWORD={{ $password }}" volumes: - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" @@ -27,7 +31,11 @@ services: condition: service_healthy kibana: +{{ if eq $serverless_enabled "true" }} + image: "docker.elastic.co/kibana-ci/kibana-serverless:latest" +{{ else }} image: "${KIBANA_IMAGE_REF}" +{{ end }} depends_on: elasticsearch: condition: service_healthy diff --git a/internal/stack/_static/elasticsearch.yml.tmpl b/internal/stack/_static/elasticsearch.yml.tmpl index 693d7731f..128c2f4a0 100644 --- a/internal/stack/_static/elasticsearch.yml.tmpl +++ b/internal/stack/_static/elasticsearch.yml.tmpl @@ -28,3 +28,11 @@ tracing.apm.enabled: true tracing.apm.agent.server_url: "http://fleet-server:8200" tracing.apm.agent.environment: "dev" {{ end }} + +{{ $serverless_enabled := fact "serverless_enabled" }} +{{ if eq $serverless_enabled "true" }} +stateless.enabled: true +stateless.object_store.type: fs +stateless.object_store.bucket: stack +path.repo: /usr/share/elasticsearch/objectstore +{{ end }} diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index 2604b478e..c10fbaba2 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -43,6 +43,10 @@ xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable expe xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" +{{ if not (semverLessThan $version "8.2.0") }} +xpack.cloudSecurityPosture.enabled: true +{{ end }} + {{ $self_monitor_enabled := fact "self_monitor_enabled" }} {{ if not (semverLessThan $version "8.0.0") }} xpack.fleet.packages: @@ -138,180 +142,10 @@ xpack.fleet.outputs: {{ $serverless_project_type := fact "serverless_project_type" }} {{ if eq $serverless_enabled "true" }} {{ if eq $serverless_project_type "observability" }} -# Observability Project config - -## Disable plugins -enterpriseSearch.enabled: false -xpack.cloudSecurityPosture.enabled: false -xpack.infra.enabled: true -xpack.securitySolution.enabled: false - -## Cloud settings -xpack.cloud.serverless.project_type: observability - -## Enable the Serverless Observability plugin -xpack.serverless.observability.enabled: true - -## Configure plugins - -## Set the home route -uiSettings.overrides.defaultRoute: /app/observability/landing - -# Customize empty page state for analytics apps -#no_data_page.analyticsNoDataPageFlavor: 'serverless_observability' - -## Set the dev project switch current type -xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability' - -## Disable adding the component template `.fleet_agent_id_verification-1` to every index template for each datastream for each integration -xpack.fleet.agentIdVerificationEnabled: false - -## Enable the capability for the observability feature ID in the serverless environment to take ownership of the rules. -## The value need to be a featureId observability Or stackAlerts Or siem -xpack.alerting.rules.overwriteProducer: 'observability' -xpack.observability.createO11yGenericFeatureId: true - -## APM Serverless Onboarding flow -#xpack.apm.serverlessOnboarding: true - -# Synthetics mTLS cert locations -#xpack.uptime.service.tls.certificate: /mnt/elastic-internal/http-certs/tls.crt -#xpack.uptime.service.tls.key: /mnt/elastic-internal/http-certs/tls.key - -# Fleet specific configuration -xpack.fleet.internal.registry.capabilities: [ - 'apm', - 'observability', - 'uptime', -] -xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false -xpack.fleet.internal.registry.spec.max: '3.0' -# Temporary until all packages implement new spec https://github.com/elastic/kibana/issues/166742 -xpack.fleet.internal.registry.spec.min: '1.0' -xpack.fleet.internal.registry.excludePackages: [ - # Security integrations - 'endpoint', - 'beaconing', - 'osquery_manager', - - # Removed in 8.11 integrations - 'cisco', - 'microsoft', - 'symantec', - 'cyberark', - - # ML integrations - 'dga', - - # Profiling integrations - 'profiler_agent', - ] - -## Required for force installation of integration packages -#xpack.fleet.packages: -# # fleet_server package installed to publish agent metrics -# - name: fleet_server -# version: latest -## Disable APM UI components and API calls -#xpack.apm.featureFlags.agentConfigurationAvailable: false -xpack.apm.featureFlags.agentConfigurationAvailable: true -#xpack.apm.featureFlags.configurableIndicesAvailable: true -xpack.apm.featureFlags.infrastructureTabAvailable: true -xpack.apm.featureFlags.infraUiAvailable: true -#xpack.apm.featureFlags.migrationToFleetAvailable: false -xpack.apm.featureFlags.migrationToFleetAvailable: true -#xpack.apm.featureFlags.sourcemapApiAvailable: false -xpack.apm.featureFlags.sourcemapApiAvailable: true -#xpack.apm.featureFlags.storageExplorerAvailable: false -xpack.apm.featureFlags.storageExplorerAvailable: true - -# Specify in telemetry the project type -#telemetry.labels.serverless: observability - -xpack.ml.ad.enabled: true -xpack.ml.dfa.enabled: false -xpack.ml.nlp.enabled: false -xpack.ml.compatibleModuleType: 'observability' - -# Disable the embedded Dev Console -console.ui.embeddedEnabled: false +serverless: oblt {{ end }} {{ if eq $serverless_project_type "security" }} -# Security Project config - -## Disable plugins -enterpriseSearch.enabled: false -xpack.apm.enabled: false -xpack.infra.enabled: false -xpack.observabilityLogsExplorer.enabled: false -xpack.observability.enabled: false - -## Cloud settings -xpack.cloud.serverless.project_type: security - -## Enable the Security Solution Serverless plugin -xpack.securitySolutionServerless.enabled: true -xpack.securitySolutionServerless.productTypes: - [ - { product_line: 'security', product_tier: 'complete' }, - { product_line: 'endpoint', product_tier: 'complete' }, - { product_line: 'cloud', product_tier: 'complete' }, - ] - -xpack.securitySolution.offeringSettings: { - sideNavEnabled: false, # Internal security side navigation disabled, the serverless global chrome navigation is used instead - ILMEnabled: false, # Index Lifecycle Management (ILM) functionalities disabled, not supported by serverless Elasticsearch - ESQLEnabled: false, # ES|QL disabled, not supported by serverless Elasticsearch - } - -newsfeed.enabled: true - -## Set the home route -uiSettings.overrides.defaultRoute: /app/security/get_started - -## Set the dev project switcher current type -xpack.serverless.plugin.developer.projectSwitcher.currentType: 'security' - -# Specify in telemetry the project type -#telemetry.labels.serverless: security - -# Fleet specific configuration -xpack.fleet.internal.registry.capabilities: ['security'] -xpack.fleet.internal.registry.spec.max: '3.0' -xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false -# Temporary until all packages implement new spec https://github.com/elastic/kibana/issues/166742 -xpack.fleet.internal.registry.spec.min: '1.0' -xpack.fleet.internal.registry.excludePackages: [ - # Oblt integrations - 'apm', - 'synthetics', - 'synthetics_dashboards', - - # Removed in 8.11 integrations - 'cisco', - 'microsoft', - 'symantec', - 'cyberark', - - # ML integrations - 'dga', - ] -# fleet_server package installed to publish agent metrics -#xpack.fleet.packages: -# - name: fleet_server -# version: latest - -xpack.ml.ad.enabled: true -xpack.ml.dfa.enabled: true -xpack.ml.nlp.enabled: false -xpack.ml.compatibleModuleType: 'security' - -# Disable the embedded Dev Console -console.ui.embeddedEnabled: false - {{ end }} -{{ else }} - {{ if not (semverLessThan $version "8.2.0") }} -xpack.cloudSecurityPosture.enabled: true +serverless: security {{ end }} {{ end }}