feat: Enable prometheus merge as a configuration in the Kyma Istio CR

api/v1alpha2/istio_merge.go

@@ -2,6 +2,7 @@ package v1alpha2
 
 import (
 	"encoding/json"
+
 	"istio.io/istio/operator/pkg/values"
 	"istio.io/istio/pkg/util/protomarshal"
 	appsv1 "k8s.io/api/apps/v1"
@@ -66,6 +67,15 @@ func (m *meshConfigBuilder) BuildNumTrustedProxies(numTrustedProxies *int) *mesh
 	return m
 }
 
+func (m *meshConfigBuilder) BuildPrometheusMergeConfig(prometheusMerge bool) *meshConfigBuilder {
+	err := m.c.SetPath("enablePrometheusMerge", prometheusMerge)
+	if err != nil {
+		return nil
+	}
+
+	return m
+}
+
 func (m *meshConfigBuilder) AddProxyMetadata(key, value string) (*meshConfigBuilder, error) {
 	err := m.c.SetPath("defaultConfig.proxyMetadata."+key, value)
 	if err != nil {
@@ -160,6 +170,7 @@ func (i *Istio) mergeConfig(op iopv1alpha1.IstioOperator) (iopv1alpha1.IstioOper
 	newMeshConfig := mcb.
 		BuildNumTrustedProxies(i.Spec.Config.NumTrustedProxies).
 		BuildExternalAuthorizerConfiguration(i.Spec.Config.Authorizers).
+		BuildPrometheusMergeConfig(i.Spec.Config.Telemetry.Metrics.PrometheusMerge).
 		Build()
 
 	op.Spec.MeshConfig = newMeshConfig

api/v1alpha2/istio_structs.go

@@ -21,6 +21,10 @@ type Config struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:validation:Enum=Local;Cluster
 	GatewayExternalTrafficPolicy *string `json:"gatewayExternalTrafficPolicy,omitempty"`
+
+	// Defines the telemetry configuration of Istio.
+	// +kubebuilder:validation:Optional
+	Telemetry Telemetry `json:"telemetry,omitempty"`
 }
 
 type Components struct {

api/v1alpha2/merge_test.go

@@ -2,9 +2,12 @@ package v1alpha2
 
 import (
 	"encoding/json"
+	"testing"
+
 	"github.com/kyma-project/istio/operator/internal/tests"
 	"github.com/onsi/ginkgo/v2/types"
 	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 	meshv1alpha1 "istio.io/api/mesh/v1alpha1"
 	iopv1alpha1 "istio.io/istio/operator/pkg/apis"
 	"istio.io/istio/operator/pkg/values"
@@ -14,7 +17,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
-	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -373,6 +375,41 @@ var _ = Describe("Merge", func() {
 		Expect(numTrustedProxies).To(Equal(float64(5)))
 	})
 
+	It("should set prometheusMerge on IstioOperator Telemetry Metrics to true when meshConfig has a enablePrometheusMerge with true", func() {
+		// given
+		m := &meshv1alpha1.MeshConfig{
+			EnablePrometheusMerge: wrapperspb.Bool(false),
+		}
+		meshConfigRaw := convert(m)
+
+		iop := iopv1alpha1.IstioOperator{
+			Spec: iopv1alpha1.IstioOperatorSpec{
+				MeshConfig: meshConfigRaw,
+			},
+		}
+		istioCR := Istio{Spec: IstioSpec{Config: Config{
+			Telemetry: Telemetry{
+				Metrics: Metrics{
+					PrometheusMerge: true,
+				},
+			},
+		}}}
+
+		// when
+		out, err := istioCR.MergeInto(iop)
+
+		// then
+		Expect(err).ShouldNot(HaveOccurred())
+
+		meshConfig, err := values.MapFromObject(out.Spec.MeshConfig)
+		Expect(err).ShouldNot(HaveOccurred())
+
+		enabledPrometheusMerge, exists := meshConfig.GetPath("enablePrometheusMerge")
+		Expect(exists).To(BeTrue())
+		Expect(enabledPrometheusMerge).To(BeTrue())
+
+	})
+
 	It("should create IngressGateway overlay with externalTrafficPolicy set to Local", func() {
 		// given
 

api/v1alpha2/telemetry.go

@@ -0,0 +1,15 @@
+package v1alpha2
+
+type Telemetry struct {
+	// Istio telemetry configuration related to metrics
+	// +kubebuilder:validation:Optional
+	Metrics Metrics `json:"metrics,omitempty"`
+}
+
+type Metrics struct {
+	// Defines whether the prometheusMerge feature is enabled. If yes, appropriate prometheus.io annotations will be added to all data plane pods to set up scraping.
+	// If these annotations already exist, they will be overwritten. With this option, the Envoy sidecar will merge Istio’s metrics with the application metrics.
+	// The merged metrics will be scraped from :15020/stats/prometheus.
+	// +kubebuilder:validation:Optional
+	PrometheusMerge bool `json:"prometheusMerge,omitempty"`
+}

config/crd/bases/operator.kyma-project.io_istios.yaml

@@ -1405,6 +1405,20 @@ spec:
                     maximum: 4294967295
                     minimum: 0
                     type: integer
+                  telemetry:
+                    description: Defines the telemetry configuration of Istio.
+                    properties:
+                      metrics:
+                        description: Istio telemetry configuration related to metrics
+                        properties:
+                          prometheusMerge:
+                            description: |-
+                              Defines whether the prometheusMerge feature is enabled. If yes, appropriate prometheus.io annotations will be added to all data plane pods to set up scraping.
+                              If these annotations already exist, they will be overwritten. With this option, the Envoy sidecar will merge Istio’s metrics with the application metrics.
+                              The merged metrics will be scraped from :15020/stats/prometheus.
+                            type: boolean
+                        type: object
+                    type: object
                 type: object
               experimental:
                 properties:

config/ui-extensions/istios/details

@@ -23,6 +23,9 @@ body:
     - source: spec.compatibilityMode
       name: compatibilityMode
       placeholder: 'false'
+    - source: spec.config.telemetry.metrics.prometheusMerge
+      name: config.prometheusMerge
+      placeholder: 'false'
 
     - name: config.authorizers
       widget: Table

config/ui-extensions/istios/form

@@ -20,6 +20,11 @@
     name: compatibilityMode
     value:
       type: bool
+  - path: config.telemetry.metrics.prometheusMerge
+    simple: true
+    name: config.prometheusMerge
+    value:
+      type: bool
 - path: spec.config.authorizers
   name: config.authorizers
   widget: GenericList

config/ui-extensions/istios/translations

@@ -13,6 +13,7 @@ en:
   config.authorizers.headers.toDownstream.onAllow: Forward To Downstream On Allow
   config.authorizers.headers.toDownstream.onDeny: Forward To Downstream On Deny
   config.gatewayExternalTrafficPolicy: Gateway external traffic policy
+  config.prometheusMerge: Enable PrometheusMerge
 
   k8s.hpaSpec: Horizontal Pod Autoscaler
   k8s.hpaSpec.minReplicas: Minimum number of replicas

controllers/istio_controller_test.go

@@ -318,7 +318,7 @@ var _ = Describe("Istio Controller", func() {
 			Expect(err).To(Not(HaveOccurred()))
 
 			Expect(updatedIstioCR.Status.State).Should(Equal(operatorv1alpha2.Ready))
-			Expect(updatedIstioCR.Annotations["operator.kyma-project.io/lastAppliedConfiguration"]).To(ContainSubstring("{\"config\":{\"numTrustedProxies\":2},"))
+			Expect(updatedIstioCR.Annotations["operator.kyma-project.io/lastAppliedConfiguration"]).To(ContainSubstring("{\"config\":{\"numTrustedProxies\":2,\"telemetry\":{\"metrics\":{}}},"))
 
 			Expect(updatedIstioCR.Status.Conditions).ToNot(BeNil())
 			Expect(*updatedIstioCR.Status.Conditions).To(HaveLen(1))

docs/contributor/adr/0010-support-for-enable-prometheusMerge-configuration.md

@@ -0,0 +1,63 @@
+# Support for Enable PrometheusMerge Configuration
+
+## Status
+Approved
+
+## Context
+There is a need to support the configuration of enabling `prometheusMerge` to be available in the Istio CR. The background behind this need is explained in [this issue](https://github.com/kyma-project/telemetry-manager/issues/1468) and the impact of this feature being enabled was discussed in an [ADR](https://github.com/kyma-project/telemetry-manager/blob/main/docs/contributor/arch/015-impact-of-istio-prometheus-merge-on-metric-pipelines.md) created in the Telemetry Manager repository.
+
+## Considerations
+Based on the [original ADR](https://github.com/kyma-project/telemetry-manager/blob/main/docs/contributor/arch/015-impact-of-istio-prometheus-merge-on-metric-pipelines.md), the plan was to originally enable the `prometheusMerge` feature by default. But this has caused problems for users deploying prometheus with a scrape loop sidecar. More details can be found in [this PR](https://github.com/kyma-project/istio/pull/1184).
+
+
+## Decision
+The enablement of the `prometheusMerge` feature needs to be a configurable option for the users. By default it will be set to `false`, but users will then need to actively set it to `true` and understand the effects of the feature.
+
+To allow for this configuration, the following API is proposed in the CR under the config option, `prometheusMerge` will have the default value of `false`.
+```yaml
+apiVersion: operator.kyma-project.io/v1alpha2
+kind: Istio
+metadata:
+    name: default
+    namespace: kyma-system
+spec:
+ config:
+   telemetry:
+     metrics:
+       prometheusMerge: false
+```
+
+The `telemetry` and `metrics` fields are introduced here to account for future plans to introduce more feature from the Istio Telemetry API into the CR.
+
+## Consequences
+Istio CustomResourceDefinition will be extended with an additional configuration field of `telemetry.metrics.prometheusMerge` that will allow for configuration of the `prometheusMerge` setting in Istio Mesh Config. The field will be an optional configuration with the default value of `false`.
+
+The `telemetry` and `metrics` field will show up in the CR when retrieved from the API server even if not explicitly defined in the manifest.
+
+### Sample configuration
+
+1.
+```
+apiVersion: istio.operator.kyma-project.io/v1alpha2
+kind: Istio
+metadata:
+  name: default
+
+```
+This will use the default value of `false` for the `prometheusMerge` option.
+
+2.
+```
+apiVersion: istio.operator.kyma-project.io/v1alpha2
+kind: Istio
+metadata:
+  name: default
+spec:
+  config:
+    telemetry:
+      metrics:
+        prometheusMerge: true
+```
+This will set the value of `prometheusMerge` to `true`.
+
+

docs/release-notes/1.15.0.md

@@ -0,0 +1,3 @@
+## New Features
+
+- Extended Istio custom resource with `prometheusMerge` configuration [#1232](https://github.com/kyma-project/istio/pull/1232)

docs/user/04-00-istio-custom-resource.md

@@ -48,6 +48,7 @@ This table lists all the possible parameters of Istio CR together with their des
 | **config.authorizers**                                      | \[\]authorizer | Specifies the list of external authorizers configured in the Istio service mesh config.                                                                                                                                                                                                                                                       |
 | **config.numTrustedProxies**                                | integer        | Specifies the number of trusted proxies deployed in front of the Istio gateway proxy.                                                                                                                                                                                                                                                         |
 | **config.gatewayExternalTrafficPolicy**                     | string         | Defines the external traffic policy for Istio Ingress Gateway Service. Valid configurations are `Local` or `Cluster`. The external traffic policy set to `Local` preserves the client IP in the request but also introduces the risk of unbalanced traffic distribution.                                                                      |
+| **config.telemetry.metrics.prometheusMerge**                | bool           | Enables the prometheusMerge feature from Istio, which will merge application's and istio's metrics and expose them together at `:15020/stats/prometheus` for scraping using plain HTTP                                                                                                                                                        |
 | **experimental**                                            | object         | Defines additional experimental features that can be enabled in experimental builds.                                                                                                                                                                                                                                                          |
 | **experimental.pilot**                                      | object         | Defines additional experimental features that can be enabled in Istio pilot component.                                                                                                                                                                                                                                                        |
 | **experimental.pilot.enableAlphaGatewayAPI**                | bool           | Enables support for alpha Kubernetes Gateway API.                                                                                                                                                                                                                                                                                             |

internal/reconciliations/istio/configuration/configuration_test.go

@@ -42,7 +42,7 @@ var _ = Describe("Istio Configuration", func() {
 			// then
 			Expect(err).ShouldNot(HaveOccurred())
 			Expect(istioCR.Annotations).To(Not(BeEmpty()))
-			Expect(istioCR.Annotations[lastAppliedConfiguration]).To(Equal(fmt.Sprintf(`{"config":{"numTrustedProxies":1},"IstioTag":"%s"}`, mockIstioTag)))
+			Expect(istioCR.Annotations[lastAppliedConfiguration]).To(Equal(fmt.Sprintf(`{"config":{"numTrustedProxies":1,"telemetry":{"metrics":{}}},"IstioTag":"%s"}`, mockIstioTag)))
 
 			appliedConfig, err := configuration.GetLastAppliedConfiguration(&istioCR)
 

internal/restarter/predicates/prometheus_merge.go

@@ -0,0 +1,52 @@
+package predicates
+
+import (
+	"github.com/kyma-project/istio/operator/api/v1alpha2"
+	"github.com/kyma-project/istio/operator/internal/reconciliations/istio/configuration"
+	v1 "k8s.io/api/core/v1"
+)
+
+type PrometheusMergeRestartPredicate struct {
+	oldPrometheusMerge bool
+	newPrometheusMerge bool
+}
+
+func NewPrometheusMergeRestartPredicate(istioCR *v1alpha2.Istio) (*PrometheusMergeRestartPredicate, error) {
+	lastAppliedConfig, err := configuration.GetLastAppliedConfiguration(istioCR)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PrometheusMergeRestartPredicate{
+		oldPrometheusMerge: lastAppliedConfig.IstioSpec.Config.Telemetry.Metrics.PrometheusMerge,
+		newPrometheusMerge: istioCR.Spec.Config.Telemetry.Metrics.PrometheusMerge,
+	}, nil
+}
+
+func (p PrometheusMergeRestartPredicate) Matches(pod v1.Pod) bool {
+	// No change in configuration, no restart needed
+	if p.oldPrometheusMerge == p.newPrometheusMerge {
+		return false
+	}
+
+	annotations := pod.GetAnnotations()
+	const (
+		prometheusMergePath = "/stats/prometheus"
+		prometheusMergePort = "15020"
+	)
+
+	hasPrometheusMergePath := annotations["prometheus.io/path"] == prometheusMergePath
+	hasPrometheusMergePort := annotations["prometheus.io/port"] == prometheusMergePort
+
+	// When enabling PrometheusMerge, restart if prometheusMerge annotations are missing or incorrect
+	if p.newPrometheusMerge {
+		return !hasPrometheusMergePath || !hasPrometheusMergePort
+	}
+
+	// When disabling PrometheusMerge, restart if prometheusMerge annotations are present and correct
+	return hasPrometheusMergePath || hasPrometheusMergePort
+}
+
+func (p PrometheusMergeRestartPredicate) MustMatch() bool {
+	return false
+}