From a2e6b290ed94909b8fa990166e6e0c61e811b740 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 4 Sep 2024 19:47:20 +0200 Subject: [PATCH 01/14] create service monitor for query Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/types.go | 4 ++ api/v1alpha1/zz_generated.deepcopy.go | 7 ++- .../monitoring.thanos.io_thanoscompacts.yaml | 4 ++ .../monitoring.thanos.io_thanosqueries.yaml | 4 ++ .../monitoring.thanos.io_thanosstores.yaml | 4 ++ go.mod | 1 + go.sum | 2 + internal/pkg/manifests/options.go | 2 + internal/pkg/manifests/query/builder.go | 10 +++- internal/pkg/manifests/service_monitor.go | 36 ++++++++++++ .../pkg/manifests/service_monitor_test.go | 57 +++++++++++++++++++ 11 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 internal/pkg/manifests/service_monitor.go create mode 100644 internal/pkg/manifests/service_monitor_test.go diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go index ad17411..448af33 100644 --- a/api/v1alpha1/types.go +++ b/api/v1alpha1/types.go @@ -75,6 +75,10 @@ type CommonThanosFields struct { // +kubebuilder:default:=logfmt // +kubebuilder:validation:Optional LogFormat *string `json:"logFormat,omitempty"` + // Enable self monitoring for the Thanos component. + // +kubebuilder:validation:Optional + // +kubebuilder:default:=true + EnableSelfMonitor *bool `json:"enableSelfMonitor,omitempty"` } type Additional struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 05c5c8c..593c531 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -144,6 +144,11 @@ func (in *CommonThanosFields) DeepCopyInto(out *CommonThanosFields) { *out = new(string) **out = **in } + if in.EnableSelfMonitor != nil { + in, out := &in.EnableSelfMonitor, &out.EnableSelfMonitor + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonThanosFields. diff --git a/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml b/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml index 91eaad2..984fa3d 100644 --- a/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml @@ -99,6 +99,10 @@ spec: pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean groupConfig: description: GroupConfig defines settings for group handling. properties: diff --git a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml index a825b6c..95ff9d2 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml @@ -3574,6 +3574,10 @@ spec: type: object type: object x-kubernetes-map-type: atomic + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean image: description: Container image to use for the Thanos components. type: string diff --git a/config/crd/bases/monitoring.thanos.io_thanosstores.yaml b/config/crd/bases/monitoring.thanos.io_thanosstores.yaml index d486282..4a32e60 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosstores.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosstores.yaml @@ -3551,6 +3551,10 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean ignoreDeletionMarksDelay: default: 24h description: |- diff --git a/go.mod b/go.mod index 51ec7bf..59dee9c 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 github.com/prometheus/client_golang v1.20.4 + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.1 github.com/prometheus/common v0.59.1 github.com/prometheus/prometheus v0.54.1 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 7a25dfa..47236af 100644 --- a/go.sum +++ b/go.sum @@ -356,6 +356,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.1 h1:QU2cs0xxKYvF1JfibP/8vs+pFy6OvIpqNR2lYC4jYNU= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.1/go.mod h1:Rd8YnCqz+2FYsiGmE2DMlaLjQRB4v2jFNnzCt9YY4IM= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index c3cd52f..33f792b 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -39,6 +39,8 @@ type Options struct { LogLevel *string // LogFormat is the log format for the component LogFormat *string + // EnableServiceMonitor is a flag to enable ServiceMonitor + EnableServiceMonitor *bool } // ToFlags returns the flags for the Options diff --git a/internal/pkg/manifests/query/builder.go b/internal/pkg/manifests/query/builder.go index be3971b..eeb68b4 100644 --- a/internal/pkg/manifests/query/builder.go +++ b/internal/pkg/manifests/query/builder.go @@ -36,7 +36,8 @@ type Options struct { LookbackDelta string MaxConcurrent int - Endpoints []Endpoint + Endpoints []Endpoint + EnableServiceMonitor bool } type EndpointType string @@ -65,6 +66,9 @@ func BuildQuery(opts Options) []client.Object { objs = append(objs, manifests.BuildServiceAccount(Name, opts.Namespace, GetRequiredLabels())) objs = append(objs, newQueryDeployment(opts, selectorLabels, objectMetaLabels)) objs = append(objs, newQueryService(opts, selectorLabels, objectMetaLabels)) + if opts.EnableServiceMonitor { + objs = append(objs, manifests.BuildServiceMonitor(opts.Options, fmt.Sprintf("%d", HTTPPort))) + } return objs } @@ -206,6 +210,10 @@ func newQueryService(opts Options, selectorLabels, objectMetaLabels map[string]s servicePorts = append(servicePorts, opts.Additional.ServicePorts...) } + if opts.EnableServiceMonitor { + objectMetaLabels["thanos-self-monitoring"] = opts.Name + } + return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: opts.Name, diff --git a/internal/pkg/manifests/service_monitor.go b/internal/pkg/manifests/service_monitor.go new file mode 100644 index 0000000..84f89bb --- /dev/null +++ b/internal/pkg/manifests/service_monitor.go @@ -0,0 +1,36 @@ +package manifests + +import ( + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor { + opts.Labels["service-monitor"] = "thanos" + return &monitoringv1.ServiceMonitor{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceMonitor", + APIVersion: monitoringv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.Name, + Namespace: opts.Namespace, + Labels: opts.Labels, + }, + Spec: monitoringv1.ServiceMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"thanos-self-monitoring": "true"}, + }, + NamespaceSelector: monitoringv1.NamespaceSelector{ + MatchNames: []string{opts.Namespace}, + }, + Endpoints: []monitoringv1.Endpoint{ + { + Interval: "30s", + Port: port, + Path: "/metrics", + }, + }, + }, + } +} diff --git a/internal/pkg/manifests/service_monitor_test.go b/internal/pkg/manifests/service_monitor_test.go new file mode 100644 index 0000000..57fc951 --- /dev/null +++ b/internal/pkg/manifests/service_monitor_test.go @@ -0,0 +1,57 @@ +package manifests + +import "testing" + +func TestBuildServiceMonitor(t *testing.T) { + opts := Options{ + Name: "thanos-stack", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "thanos", + "app.kubernetes.io/instance": "thanos-stack", + }, + } + for _, tc := range []struct { + name string + opts Options + }{ + { + name: "test service account correctness", + opts: opts, + }, + } { + t.Run(tc.name, func(t *testing.T) { + sm := BuildServiceMonitor(tc.opts, "9090") + if sm.GetName() != tc.opts.Name { + t.Errorf("expected service monitor name to be %s, got %s", tc.opts.Name, sm.GetName()) + } + if sm.GetNamespace() != tc.opts.Namespace { + t.Errorf("expected service monitor namespace to be %s, got %s", tc.opts.Namespace, sm.GetNamespace()) + } + if len(sm.Spec.Selector.MatchLabels) != 2 { + t.Errorf("expected service monitor to have 2 match labels, got %d", len(sm.Spec.Selector.MatchLabels)) + } + if sm.Spec.Selector.MatchLabels["app.kubernetes.io/name"] != "thanos" { + t.Errorf("expected service monitor match label app.kubernetes.io/name to be thanos, got %s", sm.Spec.Selector.MatchLabels["app.kubernetes.io/name"]) + } + if sm.Spec.Selector.MatchLabels["app.kubernetes.io/instance"] != "thanos-stack" { + t.Errorf("expected service monitor match label app.kubernetes.io/instance to be thanos-stack, got %s", sm.Spec.Selector.MatchLabels["app.kubernetes.io/instance"]) + } + if len(sm.Spec.NamespaceSelector.MatchNames) != 1 { + t.Errorf("expected service monitor to have 1 match name, got %d", len(sm.Spec.NamespaceSelector.MatchNames)) + } + if sm.Spec.NamespaceSelector.MatchNames[0] != tc.opts.Namespace { + t.Errorf("expected service monitor match name to be %s, got %s", tc.opts.Namespace, sm.Spec.NamespaceSelector.MatchNames[0]) + } + if len(sm.Spec.Endpoints) != 1 { + t.Errorf("expected service monitor to have 1 endpoint, got %d", len(sm.Spec.Endpoints)) + } + if sm.Spec.Endpoints[0].Port != "9090" { + t.Errorf("expected service monitor endpoint port to be 9090, got %s", sm.Spec.Endpoints[0].Port) + } + if sm.Spec.Endpoints[0].Path != "/metrics" { + t.Errorf("expected service monitor endpoint path to be /metrics, got %s", sm.Spec.Endpoints[0].Path) + } + }) + } +} From 4af215766d0b658e3eb44d6a186e85fd809c9b89 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 4 Sep 2024 20:01:15 +0200 Subject: [PATCH 02/14] create service monitor for query Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/types.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 5 ----- internal/pkg/manifests/options.go | 2 +- internal/pkg/manifests/query/builder.go | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go index 448af33..7cef6e7 100644 --- a/api/v1alpha1/types.go +++ b/api/v1alpha1/types.go @@ -78,7 +78,7 @@ type CommonThanosFields struct { // Enable self monitoring for the Thanos component. // +kubebuilder:validation:Optional // +kubebuilder:default:=true - EnableSelfMonitor *bool `json:"enableSelfMonitor,omitempty"` + EnableSelfMonitor bool `json:"enableSelfMonitor,omitempty"` } type Additional struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 593c531..ca716da 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -144,11 +144,6 @@ func (in *CommonThanosFields) DeepCopyInto(out *CommonThanosFields) { *out = new(string) **out = **in } - if in.EnableSelfMonitor != nil { - in, out := &in.EnableSelfMonitor, &out.EnableSelfMonitor - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonThanosFields. diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index 33f792b..318ba2e 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -40,7 +40,7 @@ type Options struct { // LogFormat is the log format for the component LogFormat *string // EnableServiceMonitor is a flag to enable ServiceMonitor - EnableServiceMonitor *bool + EnableServiceMonitor bool } // ToFlags returns the flags for the Options diff --git a/internal/pkg/manifests/query/builder.go b/internal/pkg/manifests/query/builder.go index eeb68b4..ceb16a0 100644 --- a/internal/pkg/manifests/query/builder.go +++ b/internal/pkg/manifests/query/builder.go @@ -66,7 +66,7 @@ func BuildQuery(opts Options) []client.Object { objs = append(objs, manifests.BuildServiceAccount(Name, opts.Namespace, GetRequiredLabels())) objs = append(objs, newQueryDeployment(opts, selectorLabels, objectMetaLabels)) objs = append(objs, newQueryService(opts, selectorLabels, objectMetaLabels)) - if opts.EnableServiceMonitor { + if opts.Options.EnableServiceMonitor { objs = append(objs, manifests.BuildServiceMonitor(opts.Options, fmt.Sprintf("%d", HTTPPort))) } return objs From 79991f5a3b700a80dc037aba426bc09655cd2c18 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 4 Sep 2024 20:06:44 +0200 Subject: [PATCH 03/14] create service monitor for query Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ca716da..05c5c8c 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) From e622b78eb01c1c26794310c0b3d3f7bbc0e404c1 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 4 Sep 2024 20:20:28 +0200 Subject: [PATCH 04/14] create service monitor for query Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- cmd/main.go | 3 ++ .../monitoring.thanos.io_thanosqueries.yaml | 4 ++ .../monitoring.thanos.io_thanosreceives.yaml | 8 ++++ .../monitoring.thanos.io_thanosrulers.yaml | 4 ++ config/rbac/role.yaml | 12 ++++++ go.mod | 13 ++++--- go.sum | 28 +++++++------- internal/controller/thanosquery_controller.go | 3 ++ internal/pkg/manifests/mutations.go | 15 ++++++++ internal/pkg/manifests/service_monitor.go | 31 ++++++++++++++-- .../pkg/manifests/service_monitor_test.go | 13 +++---- test/configs/prometheus-crd.yaml | 13 +++++++ test/e2e/e2e_test.go | 37 +++++++++++++++++-- test/utils/utils.go | 32 ++++++++++++++++ 15 files changed, 182 insertions(+), 36 deletions(-) create mode 100644 test/configs/prometheus-crd.yaml diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 05c5c8c..ca716da 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/cmd/main.go b/cmd/main.go index c74ce3f..0e48aea 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,6 +23,8 @@ import ( "net/http/pprof" "os" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/client_golang/prometheus" versioncollector "github.com/prometheus/client_golang/prometheus/collectors/version" @@ -57,6 +59,7 @@ func init() { utilruntime.Must(monitoringthanosiov1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme + utilruntime.Must(monitoringv1.AddToScheme(scheme)) } func main() { diff --git a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml index 95ff9d2..f9d9894 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml @@ -7162,6 +7162,10 @@ spec: default: true description: CompressResponses enables response compression type: boolean + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean image: description: Container image to use for the Thanos components. type: string diff --git a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml index 63d7474..3b6d573 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml @@ -3571,6 +3571,10 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean hashrings: description: Hashrings is a list of hashrings to route to. items: @@ -7309,6 +7313,10 @@ spec: - name type: object type: array + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean externalLabels: additionalProperties: type: string diff --git a/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml b/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml index 6944b07..a86af0b 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml @@ -3561,6 +3561,10 @@ spec: - key type: object x-kubernetes-map-type: atomic + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean evaluationInterval: default: 1m description: EvaluationInterval is the default interval at which rules diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 5f93612..bccb6fe 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -39,6 +39,18 @@ rules: - get - list - watch +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - monitoring.thanos.io resources: diff --git a/go.mod b/go.mod index 59dee9c..67923cf 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,8 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dennwc/varint v1.0.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -42,9 +43,9 @@ require ( github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/swag v0.22.9 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect @@ -94,7 +95,7 @@ require ( golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b // indirect @@ -106,7 +107,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 47236af..5b65ff1 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -108,8 +108,8 @@ github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/Ir github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= @@ -142,12 +142,12 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= -github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= -github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -592,8 +592,8 @@ golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -768,8 +768,8 @@ k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 h1:1Wof1cGQgA5pqgo8MxKPtf+qN6Sh/0JzznmeGPm1HnE= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8/go.mod h1:Os6V6dZwLNii3vxFpxcNaTmH8LJJBkOTg1N0tOA0fvA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/controller/thanosquery_controller.go b/internal/controller/thanosquery_controller.go index 4f510ea..0155b95 100644 --- a/internal/controller/thanosquery_controller.go +++ b/internal/controller/thanosquery_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/go-logr/logr" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/prometheus/client_golang/prometheus" monitoringthanosiov1alpha1 "github.com/thanos-community/thanos-operator/api/v1alpha1" @@ -84,6 +85,7 @@ func NewThanosQueryReconciler(logger logr.Logger, client client.Client, scheme * //+kubebuilder:rbac:groups=monitoring.thanos.io,resources=thanosqueries/finalizers,verbs=update //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=services;configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -229,6 +231,7 @@ func (r *ThanosQueryReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&corev1.Service{}). Owns(&appsv1.Deployment{}). + Owns(&monitoringv1.ServiceMonitor{}). Watches( &corev1.Service{}, r.enqueueForService(), diff --git a/internal/pkg/manifests/mutations.go b/internal/pkg/manifests/mutations.go index 3b84c9a..eaf79df 100644 --- a/internal/pkg/manifests/mutations.go +++ b/internal/pkg/manifests/mutations.go @@ -4,6 +4,8 @@ import ( "fmt" "reflect" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/imdario/mergo" appsv1 "k8s.io/api/apps/v1" @@ -22,6 +24,7 @@ import ( // - ServiceAccount // - Deployment // - StatefulSet +// - ServiceMonitor func MutateFuncFor(existing, desired client.Object) controllerutil.MutateFn { return func() error { existingAnnotations := existing.GetAnnotations() @@ -73,6 +76,11 @@ func MutateFuncFor(existing, desired client.Object) controllerutil.MutateFn { wantSts := desired.(*appsv1.StatefulSet) mutateStatefulSet(sts, wantSts) + case *monitoringv1.ServiceMonitor: + sm := existing.(*monitoringv1.ServiceMonitor) + wantSm := desired.(*monitoringv1.ServiceMonitor) + mutateServiceMonitor(sm, wantSm) + default: t := reflect.TypeOf(existing).String() return fmt.Errorf("missing mutate implementation for resource type %v", t) @@ -151,3 +159,10 @@ func mutatePodSpec(existing *corev1.PodSpec, desired *corev1.PodSpec) { existing.TopologySpreadConstraints = desired.TopologySpreadConstraints existing.Volumes = desired.Volumes } + +func mutateServiceMonitor(existing, desired *monitoringv1.ServiceMonitor) {\ + existing.Labels = desired.Labels + existing.Spec.Selector.MatchLabels = desired.Spec.Selector.MatchLabels + existing.Spec.NamespaceSelector = desired.Spec.NamespaceSelector + existing.Spec.Endpoints = desired.Spec.Endpoints +} diff --git a/internal/pkg/manifests/service_monitor.go b/internal/pkg/manifests/service_monitor.go index 84f89bb..34e1c0f 100644 --- a/internal/pkg/manifests/service_monitor.go +++ b/internal/pkg/manifests/service_monitor.go @@ -1,12 +1,22 @@ package manifests import ( + "context" + + "k8s.io/apimachinery/pkg/types" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor { - opts.Labels["service-monitor"] = "thanos" + labels := make(map[string]string) + for k, v := range opts.Labels { + labels[k] = v + } + labels["service-monitor-selector"] = "thanos" return &monitoringv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{ Kind: "ServiceMonitor", @@ -15,11 +25,11 @@ func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor ObjectMeta: metav1.ObjectMeta{ Name: opts.Name, Namespace: opts.Namespace, - Labels: opts.Labels, + Labels: labels, }, Spec: monitoringv1.ServiceMonitorSpec{ Selector: metav1.LabelSelector{ - MatchLabels: map[string]string{"thanos-self-monitoring": "true"}, + MatchLabels: map[string]string{"thanos-self-monitoring": opts.Name}, }, NamespaceSelector: monitoringv1.NamespaceSelector{ MatchNames: []string{opts.Namespace}, @@ -34,3 +44,18 @@ func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor }, } } + +func DeleteServiceMonitor(ctx context.Context, client client.Client, name string, namespace string) error { + sm := &monitoringv1.ServiceMonitor{} + if err := client.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace}, sm); err != nil { + if errors.IsNotFound(err) { + return nil + } + } + if err := client.Delete(ctx, sm); err != nil { + return err + } + return nil +} diff --git a/internal/pkg/manifests/service_monitor_test.go b/internal/pkg/manifests/service_monitor_test.go index 57fc951..2dc27d8 100644 --- a/internal/pkg/manifests/service_monitor_test.go +++ b/internal/pkg/manifests/service_monitor_test.go @@ -28,14 +28,8 @@ func TestBuildServiceMonitor(t *testing.T) { if sm.GetNamespace() != tc.opts.Namespace { t.Errorf("expected service monitor namespace to be %s, got %s", tc.opts.Namespace, sm.GetNamespace()) } - if len(sm.Spec.Selector.MatchLabels) != 2 { - t.Errorf("expected service monitor to have 2 match labels, got %d", len(sm.Spec.Selector.MatchLabels)) - } - if sm.Spec.Selector.MatchLabels["app.kubernetes.io/name"] != "thanos" { - t.Errorf("expected service monitor match label app.kubernetes.io/name to be thanos, got %s", sm.Spec.Selector.MatchLabels["app.kubernetes.io/name"]) - } - if sm.Spec.Selector.MatchLabels["app.kubernetes.io/instance"] != "thanos-stack" { - t.Errorf("expected service monitor match label app.kubernetes.io/instance to be thanos-stack, got %s", sm.Spec.Selector.MatchLabels["app.kubernetes.io/instance"]) + if len(sm.Spec.Selector.MatchLabels) != 1 { + t.Errorf("expected service monitor to have 1 match labels, got %d", len(sm.Spec.Selector.MatchLabels)) } if len(sm.Spec.NamespaceSelector.MatchNames) != 1 { t.Errorf("expected service monitor to have 1 match name, got %d", len(sm.Spec.NamespaceSelector.MatchNames)) @@ -43,6 +37,9 @@ func TestBuildServiceMonitor(t *testing.T) { if sm.Spec.NamespaceSelector.MatchNames[0] != tc.opts.Namespace { t.Errorf("expected service monitor match name to be %s, got %s", tc.opts.Namespace, sm.Spec.NamespaceSelector.MatchNames[0]) } + if sm.Spec.Selector.MatchLabels["thanos-self-monitoring"] != tc.opts.Name { + t.Errorf("expected service monitor match label thanos-self-monitoring to be %s, got %s", tc.opts.Name, sm.Spec.Selector.MatchLabels["thanos-self-monitoring"]) + } if len(sm.Spec.Endpoints) != 1 { t.Errorf("expected service monitor to have 1 endpoint, got %d", len(sm.Spec.Endpoints)) } diff --git a/test/configs/prometheus-crd.yaml b/test/configs/prometheus-crd.yaml new file mode 100644 index 0000000..401942f --- /dev/null +++ b/test/configs/prometheus-crd.yaml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: test-prometheus + namespace: default +spec: + replicas: 1 + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: + matchLabels: + service-monitor-selector: thanos + version: v2.54.0 + retention: "10d" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 2cdb9c0..d8468b8 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -31,14 +31,15 @@ import ( "github.com/thanos-community/thanos-operator/internal/pkg/manifests" "github.com/thanos-community/thanos-operator/internal/pkg/manifests/receive" "github.com/thanos-community/thanos-operator/internal/pkg/manifests/store" + "k8s.io/apimachinery/pkg/util/intstr" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/thanos-community/thanos-operator/test/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -68,6 +69,9 @@ var _ = Describe("controller", Ordered, func() { By("installing prometheus operator") Expect(utils.InstallPrometheusOperator()).To(Succeed()) + By("installing prometheus") + Expect(utils.ApplyPrometheusCRD()).To(Succeed()) + By("installing the cert-manager") Expect(utils.InstallCertManager()).To(Succeed()) @@ -83,7 +87,7 @@ var _ = Describe("controller", Ordered, func() { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { - fmt.Println("failed to add scheme") + fmt.Println("failed to add scheme ") os.Exit(1) } if err := appsv1.AddToScheme(scheme); err != nil { @@ -94,6 +98,10 @@ var _ = Describe("controller", Ordered, func() { fmt.Println("failed to add scheme") os.Exit(1) } + if err := monitoringv1.AddToScheme(scheme); err != nil { + fmt.Println("failed to add scheme") + os.Exit(1) + } cl, err := client.New(config.GetConfigOrDie(), client.Options{ Scheme: scheme, @@ -182,7 +190,7 @@ var _ = Describe("controller", Ordered, func() { } return nil } - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + EventuallyWithOffset(1, verifyControllerUp, time.Minute*2, time.Second).Should(Succeed()) }) }) @@ -525,4 +533,25 @@ var _ = Describe("controller", Ordered, func() { }) }) }) + + Context("Thanos Query Service Monitor", func() { + It("should remove service monitor when disabled", func() { + query := &v1alpha1.ThanosQuery{} + err := c.Get(context.Background(), client.ObjectKey{Name: queryName, Namespace: namespace}, query) + Expect(err).To(BeNil()) + + Eventually(func() bool { + return utils.VerifyServiceMonitor(c, queryName, namespace) + }, time.Minute*5, time.Second*10).Should(BeTrue()) + + query.Spec.CommonThanosFields.EnableSelfMonitor = false + err = c.Update(context.Background(), query) + Expect(err).To(BeNil()) + + Eventually(func() bool { + return utils.VerifyServiceMonitorDeleted(c, queryName, namespace) + }, time.Minute*5, time.Second*10).Should(BeTrue()) + + }) + }) }) diff --git a/test/utils/utils.go b/test/utils/utils.go index b2d14db..7a6e5d4 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -28,6 +28,10 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/api/errors" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + . "github.com/onsi/ginkgo/v2" //nolint:golint,revive "github.com/golang/snappy" @@ -528,3 +532,31 @@ func VerifyCfgMapOrSecretEnvVarExists(c client.Client, obj client.Object, name, return false } } + +func ApplyPrometheusCRD() error { + // Apply Prometheus CRD + wd, _ := os.Getwd() + promCmd := exec.Command("kubectl", "apply", "-f", wd+"/test/configs/prometheus-crd.yaml") + if _, err := Run(promCmd); err != nil { + return fmt.Errorf("failed to apply Prometheus CRD: %v", err) + } + return nil +} + +func VerifyServiceMonitor(c client.Client, name, ns string) bool { + sm := &monitoringv1.ServiceMonitor{} + err := c.Get(context.Background(), client.ObjectKey{ + Name: name, + Namespace: ns, + }, sm) + return err == nil +} + +func VerifyServiceMonitorDeleted(c client.Client, name, ns string) bool { + sm := &monitoringv1.ServiceMonitor{} + err := c.Get(context.Background(), client.ObjectKey{ + Name: name, + Namespace: ns, + }, sm) + return errors.IsNotFound(err) +} From 133919dab5d96a3543385d106fb601884e33f2cf Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Mon, 16 Sep 2024 10:02:51 +0200 Subject: [PATCH 05/14] use portname in service monitor Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/types.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 7 +- internal/controller/suite_test.go | 11 +- .../controller/thanosquery_controller_test.go | 19 +- internal/pkg/manifests/mutations.go | 4 +- .../pkg/manifests/service_monitor_test.go | 6 +- test/configs/prometheus-crd.yaml | 5 +- test/configs/rbac.yaml | 43 ++ test/configs/service-monitor.yaml | 607 ++++++++++++++++++ test/e2e/e2e_test.go | 57 +- test/utils/utils.go | 33 +- 11 files changed, 745 insertions(+), 49 deletions(-) create mode 100644 test/configs/rbac.yaml create mode 100644 test/configs/service-monitor.yaml diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go index 7cef6e7..448af33 100644 --- a/api/v1alpha1/types.go +++ b/api/v1alpha1/types.go @@ -78,7 +78,7 @@ type CommonThanosFields struct { // Enable self monitoring for the Thanos component. // +kubebuilder:validation:Optional // +kubebuilder:default:=true - EnableSelfMonitor bool `json:"enableSelfMonitor,omitempty"` + EnableSelfMonitor *bool `json:"enableSelfMonitor,omitempty"` } type Additional struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ca716da..1d9d1dc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -144,6 +144,11 @@ func (in *CommonThanosFields) DeepCopyInto(out *CommonThanosFields) { *out = new(string) **out = **in } + if in.EnableSelfMonitor != nil { + in, out := &in.EnableSelfMonitor, &out.EnableSelfMonitor + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonThanosFields. diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 322baa5..898fdd0 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -23,6 +23,8 @@ import ( "runtime" "testing" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" @@ -64,7 +66,11 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "test", "configs", "service-monitor.yaml"), + }, + ErrorIfCRDPathMissing: true, // The BinaryAssetsDirectory is only required if you want to run the tests directly @@ -85,6 +91,9 @@ var _ = BeforeSuite(func() { err = monitoringthanosiov1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = monitoringv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ diff --git a/internal/controller/thanosquery_controller_test.go b/internal/controller/thanosquery_controller_test.go index 3f99234..93299b9 100644 --- a/internal/controller/thanosquery_controller_test.go +++ b/internal/controller/thanosquery_controller_test.go @@ -21,16 +21,17 @@ import ( "fmt" "time" + "github.com/thanos-community/thanos-operator/internal/pkg/manifests" + manifestquery "github.com/thanos-community/thanos-operator/internal/pkg/manifests/query" + appsv1 "k8s.io/api/apps/v1" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" monitoringthanosiov1alpha1 "github.com/thanos-community/thanos-operator/api/v1alpha1" - "github.com/thanos-community/thanos-operator/internal/pkg/manifests" - manifestquery "github.com/thanos-community/thanos-operator/internal/pkg/manifests/query" "github.com/thanos-community/thanos-operator/internal/pkg/manifests/receive" "github.com/thanos-community/thanos-operator/test/utils" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -242,6 +243,18 @@ var _ = Describe("ThanosQuery Controller", Ordered, func() { }, time.Second*30, time.Second*10).Should(Succeed()) }) + By("removing service monitor when disabled", func() { + Expect(utils.VerifyServiceMonitor(k8sClient, resourceName, ns)).To(BeTrue()) + + enableSelfMonitor := false + resource.Spec.EnableSelfMonitor = &enableSelfMonitor + Expect(k8sClient.Update(context.Background(), resource)).Should(Succeed()) + + Eventually(func() bool { + return utils.VerifyServiceMonitorDeleted(k8sClient, resourceName, ns) + }, time.Minute*1, time.Second*10).Should(BeTrue()) + }) + By("checking paused state", func() { isPaused := true resource.Spec.Paused = &isPaused diff --git a/internal/pkg/manifests/mutations.go b/internal/pkg/manifests/mutations.go index eaf79df..f966f8f 100644 --- a/internal/pkg/manifests/mutations.go +++ b/internal/pkg/manifests/mutations.go @@ -160,8 +160,8 @@ func mutatePodSpec(existing *corev1.PodSpec, desired *corev1.PodSpec) { existing.Volumes = desired.Volumes } -func mutateServiceMonitor(existing, desired *monitoringv1.ServiceMonitor) {\ - existing.Labels = desired.Labels +func mutateServiceMonitor(existing, desired *monitoringv1.ServiceMonitor) { + existing.Labels = desired.Labels existing.Spec.Selector.MatchLabels = desired.Spec.Selector.MatchLabels existing.Spec.NamespaceSelector = desired.Spec.NamespaceSelector existing.Spec.Endpoints = desired.Spec.Endpoints diff --git a/internal/pkg/manifests/service_monitor_test.go b/internal/pkg/manifests/service_monitor_test.go index 2dc27d8..f46b126 100644 --- a/internal/pkg/manifests/service_monitor_test.go +++ b/internal/pkg/manifests/service_monitor_test.go @@ -21,7 +21,7 @@ func TestBuildServiceMonitor(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - sm := BuildServiceMonitor(tc.opts, "9090") + sm := BuildServiceMonitor(tc.opts, "http") if sm.GetName() != tc.opts.Name { t.Errorf("expected service monitor name to be %s, got %s", tc.opts.Name, sm.GetName()) } @@ -43,8 +43,8 @@ func TestBuildServiceMonitor(t *testing.T) { if len(sm.Spec.Endpoints) != 1 { t.Errorf("expected service monitor to have 1 endpoint, got %d", len(sm.Spec.Endpoints)) } - if sm.Spec.Endpoints[0].Port != "9090" { - t.Errorf("expected service monitor endpoint port to be 9090, got %s", sm.Spec.Endpoints[0].Port) + if sm.Spec.Endpoints[0].Port != "http" { + t.Errorf("expected service monitor endpoint port to be http, got %s", sm.Spec.Endpoints[0].Port) } if sm.Spec.Endpoints[0].Path != "/metrics" { t.Errorf("expected service monitor endpoint path to be /metrics, got %s", sm.Spec.Endpoints[0].Path) diff --git a/test/configs/prometheus-crd.yaml b/test/configs/prometheus-crd.yaml index 401942f..d4f2fa0 100644 --- a/test/configs/prometheus-crd.yaml +++ b/test/configs/prometheus-crd.yaml @@ -4,10 +4,9 @@ metadata: name: test-prometheus namespace: default spec: + serviceAccountName: prometheus replicas: 1 serviceMonitorNamespaceSelector: {} - serviceMonitorSelector: - matchLabels: - service-monitor-selector: thanos + serviceMonitorSelector: {} version: v2.54.0 retention: "10d" diff --git a/test/configs/rbac.yaml b/test/configs/rbac.yaml new file mode 100644 index 0000000..5b8fd88 --- /dev/null +++ b/test/configs/rbac.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: default + diff --git a/test/configs/service-monitor.yaml b/test/configs/service-monitor.yaml new file mode 100644 index 0000000..ff9c09d --- /dev/null +++ b/test/configs/service-monitor.yaml @@ -0,0 +1,607 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.1 + creationTimestamp: null + name: servicemonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: ServiceMonitor + listKind: ServiceMonitorList + plural: servicemonitors + singular: servicemonitor + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ServiceMonitor defines monitoring for a set of services. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired Service selection for target discovery + by Prometheus. + properties: + endpoints: + description: A list of endpoints allowed as part of this ServiceMonitor. + items: + description: Endpoint defines a scrapeable endpoint serving Prometheus + metrics. + properties: + authorization: + description: Authorization section for this endpoint + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object + basicAuth: + description: 'BasicAuth allow an endpoint to authenticate over + basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints' + properties: + password: + description: The secret in the service monitor namespace + that contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + username: + description: The secret in the service monitor namespace + that contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + bearerTokenFile: + description: File to read bearer token for scraping targets. + type: string + bearerTokenSecret: + description: Secret to mount to read bearer token for scraping + targets. The secret needs to be in the same namespace as the + service monitor and accessible by the Prometheus Operator. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + honorLabels: + description: HonorLabels chooses the metric's labels on collisions + with target labels. + type: boolean + honorTimestamps: + description: HonorTimestamps controls whether Prometheus respects + the timestamps present in scraped data. + type: boolean + interval: + description: Interval at which metrics should be scraped + type: string + metricRelabelings: + description: MetricRelabelConfigs to apply to samples before + ingestion. + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It + defines ``-section of Prometheus + configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source + label values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular + expression for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + oauth2: + description: OAuth2 for the URL. Only valid in Prometheus versions + 2.27.0 and newer. + properties: + clientId: + description: The secret or configmap containing the OAuth2 + client id + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + clientSecret: + description: The secret containing the OAuth2 client secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + params: + additionalProperties: + items: + type: string + type: array + description: Optional HTTP URL parameters + type: object + path: + description: HTTP path to scrape for metrics. + type: string + port: + description: Name of the service port this endpoint refers to. + Mutually exclusive with targetPort. + type: string + proxyUrl: + description: ProxyURL eg http://proxyserver:2195 Directs scrapes + to proxy through this endpoint. + type: string + relabelings: + description: 'RelabelConfigs to apply to samples before scraping. + Prometheus Operator automatically adds relabelings for a few + standard Kubernetes fields and replaces original scrape job + name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It + defines ``-section of Prometheus + configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source + label values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular + expression for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + scheme: + description: HTTP scheme to use for scraping. + type: string + scrapeTimeout: + description: Timeout after which the scrape is ended + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: Name or number of the target port of the Pod behind + the Service, the port must be specified with container port + property. Mutually exclusive with port. + x-kubernetes-int-or-string: true + tlsConfig: + description: TLS configuration to use when scraping the endpoint + properties: + ca: + description: Struct containing the CA cert to use for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for + the targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus + container for the targets. + type: string + keySecret: + description: Secret containing the client key file for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + type: object + type: array + jobLabel: + description: "Chooses the label of the Kubernetes `Endpoints`. Its + value will be used for the `job`-label's value of the created metrics. + \n Default & fallback value: the name of the respective Kubernetes + `Endpoint`." + type: string + labelLimit: + description: Per-scrape limit on number of labels that will be accepted + for a sample. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer + labelNameLengthLimit: + description: Per-scrape limit on length of labels name that will be + accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer + labelValueLengthLimit: + description: Per-scrape limit on length of labels value that will + be accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer + namespaceSelector: + description: Selector to select which namespaces the Kubernetes Endpoints + objects are discovered from. + properties: + any: + description: Boolean describing whether all namespaces are selected + in contrast to a list restricting them. + type: boolean + matchNames: + description: List of namespace names. + items: + type: string + type: array + type: object + podTargetLabels: + description: PodTargetLabels transfers labels on the Kubernetes `Pod` + onto the created metrics. + items: + type: string + type: array + sampleLimit: + description: SampleLimit defines per-scrape limit on number of scraped + samples that will be accepted. + format: int64 + type: integer + selector: + description: Selector to select Endpoints objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + targetLabels: + description: TargetLabels transfers labels from the Kubernetes `Service` + onto the created metrics. All labels set in `selector.matchLabels` + are automatically transferred. + items: + type: string + type: array + targetLimit: + description: TargetLimit defines a limit on the number of scraped + targets that will be accepted. + format: int64 + type: integer + required: + - endpoints + - selector + type: object + required: + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d8468b8..fdeab30 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -31,15 +31,14 @@ import ( "github.com/thanos-community/thanos-operator/internal/pkg/manifests" "github.com/thanos-community/thanos-operator/internal/pkg/manifests/receive" "github.com/thanos-community/thanos-operator/internal/pkg/manifests/store" - "k8s.io/apimachinery/pkg/util/intstr" - - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/thanos-community/thanos-operator/test/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -69,9 +68,6 @@ var _ = Describe("controller", Ordered, func() { By("installing prometheus operator") Expect(utils.InstallPrometheusOperator()).To(Succeed()) - By("installing prometheus") - Expect(utils.ApplyPrometheusCRD()).To(Succeed()) - By("installing the cert-manager") Expect(utils.InstallCertManager()).To(Succeed()) @@ -87,7 +83,7 @@ var _ = Describe("controller", Ordered, func() { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { - fmt.Println("failed to add scheme ") + fmt.Println("failed to add scheme") os.Exit(1) } if err := appsv1.AddToScheme(scheme); err != nil { @@ -98,10 +94,6 @@ var _ = Describe("controller", Ordered, func() { fmt.Println("failed to add scheme") os.Exit(1) } - if err := monitoringv1.AddToScheme(scheme); err != nil { - fmt.Println("failed to add scheme") - os.Exit(1) - } cl, err := client.New(config.GetConfigOrDie(), client.Options{ Scheme: scheme, @@ -190,7 +182,7 @@ var _ = Describe("controller", Ordered, func() { } return nil } - EventuallyWithOffset(1, verifyControllerUp, time.Minute*2, time.Second).Should(Succeed()) + EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) }) }) @@ -394,6 +386,26 @@ var _ = Describe("controller", Ordered, func() { }, time.Minute*1, time.Second*10).Should(BeTrue()) }) }) + Context("Thanos Query Service Monitor", func() { + It("should remove service monitor when disabled", func() { + query := &v1alpha1.ThanosQuery{} + err := c.Get(context.Background(), client.ObjectKey{Name: queryName, Namespace: namespace}, query) + Expect(err).To(BeNil()) + + Eventually(func() bool { + return utils.VerifyServiceMonitor(c, queryName, namespace) + }, time.Minute*5, time.Second*10).Should(BeTrue()) + + query.Spec.CommonThanosFields.EnableSelfMonitor = false + err = c.Update(context.Background(), query) + Expect(err).To(BeNil()) + + Eventually(func() bool { + return utils.VerifyServiceMonitorDeleted(c, queryName, namespace) + }, time.Minute*5, time.Second*10).Should(BeTrue()) + + }) + }) }) Describe("Thanos Ruler", Ordered, func() { @@ -533,25 +545,4 @@ var _ = Describe("controller", Ordered, func() { }) }) }) - - Context("Thanos Query Service Monitor", func() { - It("should remove service monitor when disabled", func() { - query := &v1alpha1.ThanosQuery{} - err := c.Get(context.Background(), client.ObjectKey{Name: queryName, Namespace: namespace}, query) - Expect(err).To(BeNil()) - - Eventually(func() bool { - return utils.VerifyServiceMonitor(c, queryName, namespace) - }, time.Minute*5, time.Second*10).Should(BeTrue()) - - query.Spec.CommonThanosFields.EnableSelfMonitor = false - err = c.Update(context.Background(), query) - Expect(err).To(BeNil()) - - Eventually(func() bool { - return utils.VerifyServiceMonitorDeleted(c, queryName, namespace) - }, time.Minute*5, time.Second*10).Should(BeTrue()) - - }) - }) }) diff --git a/test/utils/utils.go b/test/utils/utils.go index 7a6e5d4..0e5bc5d 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -19,6 +19,7 @@ package utils import ( "bytes" "context" + "encoding/json" "fmt" "net/http" "net/url" @@ -60,6 +61,17 @@ const ( certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" ) +type PrometheusResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` // Use interface{} because value can be mixed types + } `json:"result"` + } `json:"data"` +} + func warnError(err error) { _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } @@ -534,12 +546,15 @@ func VerifyCfgMapOrSecretEnvVarExists(c client.Client, obj client.Object, name, } func ApplyPrometheusCRD() error { - // Apply Prometheus CRD wd, _ := os.Getwd() - promCmd := exec.Command("kubectl", "apply", "-f", wd+"/test/configs/prometheus-crd.yaml") + promCmd := exec.Command("kubectl", "apply", "-f", wd+"/test/configs/prometheus-crd.yaml", "--server-side") if _, err := Run(promCmd); err != nil { return fmt.Errorf("failed to apply Prometheus CRD: %v", err) } + cmd := exec.Command("kubectl", "apply", "-f", wd+"/test/configs/rbac.yaml", "--server-side") + if _, err := Run(cmd); err != nil { + return fmt.Errorf("failed to apply RBAC: %v for Prometheus", err) + } return nil } @@ -560,3 +575,17 @@ func VerifyServiceMonitorDeleted(c client.Client, name, ns string) bool { }, sm) return errors.IsNotFound(err) } + +func QueryPrometheus(query string) (*PrometheusResponse, error) { + url := fmt.Sprintf("http://localhost:9090/api/v1/query?query=%s", query) + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + var promResp PrometheusResponse + if err := json.NewDecoder(resp.Body).Decode(&promResp); err != nil { + return nil, err + } + return &promResp, nil +} From 071841e643e143974fb1bb6ea8f3157b75b1a97d Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 25 Sep 2024 18:58:51 +0200 Subject: [PATCH 06/14] update Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/types.go | 16 ++ api/v1alpha1/zz_generated.deepcopy.go | 39 ++++- .../monitoring.thanos.io_thanoscompacts.yaml | 16 ++ .../monitoring.thanos.io_thanosqueries.yaml | 33 ++++ .../monitoring.thanos.io_thanosreceives.yaml | 154 ++++++++++++++++++ .../monitoring.thanos.io_thanosrulers.yaml | 16 ++ .../monitoring.thanos.io_thanosstores.yaml | 16 ++ internal/controller/suite_test.go | 1 - internal/controller/thanosquery_controller.go | 1 + .../controller/thanosquery_controller_test.go | 12 -- internal/pkg/manifests/options.go | 12 ++ internal/pkg/manifests/query/builder.go | 4 +- internal/pkg/manifests/service_monitor.go | 6 +- test/e2e/e2e_test.go | 53 +++++- test/utils/utils.go | 113 +++++++++++++ 15 files changed, 470 insertions(+), 22 deletions(-) diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go index 448af33..0168d56 100644 --- a/api/v1alpha1/types.go +++ b/api/v1alpha1/types.go @@ -79,6 +79,9 @@ type CommonThanosFields struct { // +kubebuilder:validation:Optional // +kubebuilder:default:=true EnableSelfMonitor *bool `json:"enableSelfMonitor,omitempty"` + // ServiceMonitorConfig is the configuration for the ServiceMonitor. + // +kubebuilder:validation:Optional + ServiceMonitorConfig *ServiceMonitorConfig `json:"serviceMonitor,omitempty"` } type Additional struct { @@ -108,6 +111,19 @@ type Additional struct { ServicePorts []corev1.ServicePort `json:"additionalServicePorts,omitempty"` } +type ServiceMonitorConfig struct { + //Enabled + // +kubebuilder:validation:Optional + // +kubebuilder:default:=true + Enabled *bool `json:"enabled,omitempty"` + //Labels to add to the ServiceMonitor. + // +kubebuilder:validation:Optional + Labels map[string]string `json:"labels,omitempty"` + //Namespace to deploy the ServiceMonitor in. + // +kubebuilder:validation:Optional + Namespace *string `json:"namespace,omitempty"` +} + func (osc *ObjectStorageConfig) ToSecretKeySelector() corev1.SecretKeySelector { return corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: osc.Name}, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1d9d1dc..b9a44bc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -149,6 +149,11 @@ func (in *CommonThanosFields) DeepCopyInto(out *CommonThanosFields) { *out = new(bool) **out = **in } + if in.ServiceMonitorConfig != nil { + in, out := &in.ServiceMonitorConfig, &out.ServiceMonitorConfig + *out = new(ServiceMonitorConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommonThanosFields. @@ -412,6 +417,38 @@ func (in *RouterSpec) DeepCopy() *RouterSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceMonitorConfig) DeepCopyInto(out *ServiceMonitorConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMonitorConfig. +func (in *ServiceMonitorConfig) DeepCopy() *ServiceMonitorConfig { + if in == nil { + return nil + } + out := new(ServiceMonitorConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShardingConfig) DeepCopyInto(out *ShardingConfig) { *out = *in diff --git a/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml b/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml index 984fa3d..e89f07d 100644 --- a/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml @@ -318,6 +318,22 @@ spec: - oneHour - raw type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object shardingConfig: description: ShardingConfig is the sharding configuration for the compact component. diff --git a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml index f9d9894..8d125bd 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml @@ -7392,6 +7392,23 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the + ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object version: description: |- Version of Thanos to be deployed. @@ -7464,6 +7481,22 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object version: description: |- Version of Thanos to be deployed. diff --git a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml index 3b6d573..ef7a362 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml @@ -3803,6 +3803,143 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + image: + description: Container image to use for the Thanos components. + type: string + imagePullPolicy: + default: IfNotPresent + description: |- + Image pull policy for the Thanos containers. + See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. + enum: + - Always + - Never + - IfNotPresent + type: string + imagePullSecrets: + description: |- + An optional list of references to Secrets in the same namespace + to use for pulling images from registries. + See http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + logFormat: + default: logfmt + description: Log format for Thanos. + enum: + - logfmt + - json + type: string + logLevel: + description: Log level for Thanos. + enum: + - debug + - info + - warn + - error + type: string + paused: + description: |- + When a resource is paused, no actions except for deletion + will be performed on the underlying objects. + type: boolean + resourceRequirements: + description: ResourceRequirements for the Thanos component container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the + ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object + version: + description: |- + Version of Thanos to be deployed. + If not specified, the operator assumes the latest upstream version of + Thanos available at the time when the version of the operator was released. + type: string required: - defaultObjectStorageConfig - hashrings @@ -7463,6 +7600,23 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the + ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object version: description: |- Version of Thanos to be deployed. diff --git a/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml b/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml index a86af0b..f379799 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml @@ -3805,6 +3805,22 @@ spec: type: object type: object x-kubernetes-map-type: atomic + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object storageSize: description: StorageSize is the size of the storage to be used by the Thanos Ruler StatefulSet. diff --git a/config/crd/bases/monitoring.thanos.io_thanosstores.yaml b/config/crd/bases/monitoring.thanos.io_thanosstores.yaml index 4a32e60..9e0927b 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosstores.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosstores.yaml @@ -3745,6 +3745,22 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for the ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor in. + type: string + type: object shardingStrategy: description: ShardingStrategy defines the sharding strategy for the Store Gateways across object storage blocks. diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 898fdd0..e76b07b 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -68,7 +68,6 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "crd", "bases"), - filepath.Join("..", "..", "test", "configs", "service-monitor.yaml"), }, ErrorIfCRDPathMissing: true, diff --git a/internal/controller/thanosquery_controller.go b/internal/controller/thanosquery_controller.go index 0155b95..68654a4 100644 --- a/internal/controller/thanosquery_controller.go +++ b/internal/controller/thanosquery_controller.go @@ -239,6 +239,7 @@ func (r *ThanosQueryReconciler) SetupWithManager(mgr ctrl.Manager) error { ). Complete(r) + // if servicemonitor CRD exists in the cluster, watch for changes to ServiceMonitor resources if err != nil { r.Recorder.Event(&monitoringthanosiov1alpha1.ThanosQuery{}, corev1.EventTypeWarning, "SetupFailed", fmt.Sprintf("Failed to set up controller: %v", err)) return err diff --git a/internal/controller/thanosquery_controller_test.go b/internal/controller/thanosquery_controller_test.go index 93299b9..f8942fc 100644 --- a/internal/controller/thanosquery_controller_test.go +++ b/internal/controller/thanosquery_controller_test.go @@ -243,18 +243,6 @@ var _ = Describe("ThanosQuery Controller", Ordered, func() { }, time.Second*30, time.Second*10).Should(Succeed()) }) - By("removing service monitor when disabled", func() { - Expect(utils.VerifyServiceMonitor(k8sClient, resourceName, ns)).To(BeTrue()) - - enableSelfMonitor := false - resource.Spec.EnableSelfMonitor = &enableSelfMonitor - Expect(k8sClient.Update(context.Background(), resource)).Should(Succeed()) - - Eventually(func() bool { - return utils.VerifyServiceMonitorDeleted(k8sClient, resourceName, ns) - }, time.Minute*1, time.Second*10).Should(BeTrue()) - }) - By("checking paused state", func() { isPaused := true resource.Spec.Paused = &isPaused diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index 318ba2e..48b4985 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -3,6 +3,7 @@ package manifests import ( "fmt" + "github.com/thanos-community/thanos-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -41,6 +42,8 @@ type Options struct { LogFormat *string // EnableServiceMonitor is a flag to enable ServiceMonitor EnableServiceMonitor bool + //ServiceMonitorConfig is the configuration for the ServiceMonitor + ServiceMonitorConfig *v1alpha1.ServiceMonitorConfig } // ToFlags returns the flags for the Options @@ -57,6 +60,15 @@ func (o Options) ToFlags() []string { fmt.Sprintf("--log.level=%s", *o.LogLevel), fmt.Sprintf("--log.format=%s", *o.LogFormat), } + if o.ServiceMonitorConfig == nil || o.ServiceMonitorConfig.Enabled == nil { + o.ServiceMonitorConfig.Enabled = ptr.To(true) + + } + if o.ServiceMonitorConfig == nil || *o.ServiceMonitorConfig.Namespace == "" { + o.ServiceMonitorConfig.Namespace = ptr.To(o.Namespace) + } + + return o } // GetContainerImage for the Options diff --git a/internal/pkg/manifests/query/builder.go b/internal/pkg/manifests/query/builder.go index ceb16a0..cdaf0b5 100644 --- a/internal/pkg/manifests/query/builder.go +++ b/internal/pkg/manifests/query/builder.go @@ -66,8 +66,8 @@ func BuildQuery(opts Options) []client.Object { objs = append(objs, manifests.BuildServiceAccount(Name, opts.Namespace, GetRequiredLabels())) objs = append(objs, newQueryDeployment(opts, selectorLabels, objectMetaLabels)) objs = append(objs, newQueryService(opts, selectorLabels, objectMetaLabels)) - if opts.Options.EnableServiceMonitor { - objs = append(objs, manifests.BuildServiceMonitor(opts.Options, fmt.Sprintf("%d", HTTPPort))) + if opts.ServiceMonitorConfig.Enabled != nil && *opts.ServiceMonitorConfig.Enabled { + objs = append(objs, manifests.BuildServiceMonitor(opts.Options, HTTPPortName)) } return objs } diff --git a/internal/pkg/manifests/service_monitor.go b/internal/pkg/manifests/service_monitor.go index 34e1c0f..c4e8124 100644 --- a/internal/pkg/manifests/service_monitor.go +++ b/internal/pkg/manifests/service_monitor.go @@ -16,7 +16,9 @@ func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor for k, v := range opts.Labels { labels[k] = v } - labels["service-monitor-selector"] = "thanos" + for k, v := range opts.ServiceMonitorConfig.Labels { + labels[k] = v + } return &monitoringv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{ Kind: "ServiceMonitor", @@ -24,7 +26,7 @@ func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor }, ObjectMeta: metav1.ObjectMeta{ Name: opts.Name, - Namespace: opts.Namespace, + Namespace: *opts.ServiceMonitorConfig.Namespace, Labels: labels, }, Spec: monitoringv1.ServiceMonitorSpec{ diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index fdeab30..daea771 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -19,26 +19,29 @@ package e2e import ( "context" "fmt" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "os" "os/exec" "time" + rbacv1 "k8s.io/api/rbac/v1" + + "github.com/thanos-community/thanos-operator/internal/pkg/manifests" + "github.com/thanos-community/thanos-operator/internal/pkg/manifests/store" + "k8s.io/apimachinery/pkg/util/intstr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/thanos-community/thanos-operator/api/v1alpha1" "github.com/thanos-community/thanos-operator/internal/controller" - "github.com/thanos-community/thanos-operator/internal/pkg/manifests" "github.com/thanos-community/thanos-operator/internal/pkg/manifests/receive" - "github.com/thanos-community/thanos-operator/internal/pkg/manifests/store" "github.com/thanos-community/thanos-operator/test/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -58,6 +61,8 @@ const ( queryName = "example-query" rulerName = "example-ruler" + + prometheusPort = 9090 ) var _ = Describe("controller", Ordered, func() { @@ -94,6 +99,14 @@ var _ = Describe("controller", Ordered, func() { fmt.Println("failed to add scheme") os.Exit(1) } + if err := monitoringv1.AddToScheme(scheme); err != nil { + fmt.Println("failed to add scheme") + os.Exit(1) + } + if err := rbacv1.AddToScheme(scheme); err != nil { + fmt.Println("failed to add scheme") + os.Exit(1) + } cl, err := client.New(config.GetConfigOrDie(), client.Options{ Scheme: scheme, @@ -103,6 +116,9 @@ var _ = Describe("controller", Ordered, func() { os.Exit(1) } c = cl + + By("Setup prometheus") + Expect(utils.SetUpPrometheus(c)).To(Succeed()) }) AfterAll(func() { @@ -386,6 +402,35 @@ var _ = Describe("controller", Ordered, func() { }, time.Minute*1, time.Second*10).Should(BeTrue()) }) }) + Context("Discover Thanos Query as a target", func() { + It("Prometheus should discover Thanos Query as a target", func() { + pods := &corev1.PodList{} + selector := client.MatchingLabels{ + "prometheus": "test-prometheus", + } + err := c.List(context.Background(), pods, selector, &client.ListOptions{Namespace: "default"}) + Expect(err).To(BeNil()) + Expect(len(pods.Items)).To(Equal(1)) + + pod := pods.Items[0].Name + port := intstr.IntOrString{IntVal: prometheusPort} + cancelFn, err := utils.StartPortForward(context.Background(), port, "https", pod, "default") + Expect(err).To(BeNil()) + defer cancelFn() + + Eventually(func() error { + promResp, err := utils.QueryPrometheus("up{service=\"" + queryName + "\"}") + if err != nil { + return err + } + if len(promResp.Data.Result) > 0 && promResp.Data.Result[0].Metric["service"] != queryName { + return fmt.Errorf("query service is not up in Prometheus") + } + return nil + }, time.Minute*1, time.Second*5).Should(Succeed()) + }) + }) + Context("Thanos Query Service Monitor", func() { It("should remove service monitor when disabled", func() { query := &v1alpha1.ThanosQuery{} diff --git a/test/utils/utils.go b/test/utils/utils.go index 0e5bc5d..3cdc528 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -29,6 +29,10 @@ import ( "strings" "time" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "k8s.io/apimachinery/pkg/api/errors" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -558,6 +562,115 @@ func ApplyPrometheusCRD() error { return nil } +func SetUpPrometheus(c client.Client) error { + if err := CreateServiceAccount(c); err != nil { + return err + } + if err := CreateClusterRole(c); err != nil { + return err + } + if err := CreateClusterRoleBinding(c); err != nil { + return err + } + if err := CreatePrometheus(c); err != nil { + return err + } + return nil +} + +func CreatePrometheus(c client.Client) error { + promRes := &monitoringv1.Prometheus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-prometheus", + Namespace: "default", + }, + Spec: monitoringv1.PrometheusSpec{ + CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ + ServiceAccountName: "prometheus", + Replicas: pointer.Int32(1), + ServiceMonitorNamespaceSelector: &metav1.LabelSelector{}, + ServiceMonitorSelector: &metav1.LabelSelector{}, + Version: "v2.54.0", + }, + Retention: "10d", + }, + } + if err := c.Create(context.Background(), promRes); err != nil { + return err + } + return nil +} + +// create service account for prometheus +func CreateServiceAccount(c client.Client) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus", + Namespace: "default", + }, + } + if err := c.Create(context.Background(), sa); err != nil { + return err + } + return nil +} + +func CreateClusterRole(c client.Client) error { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"nodes", "nodes/proxy", "services", "endpoints", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }, + { + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get"}, + }, + }, + } + if err := c.Create(context.Background(), cr); err != nil { + return err + } + return nil +} + +func CreateClusterRoleBinding(c client.Client) error { + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "prometheus", + Namespace: "default", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "prometheus", + }, + } + if err := c.Create(context.Background(), crb); err != nil { + return err + } + return nil +} + func VerifyServiceMonitor(c client.Client, name, ns string) bool { sm := &monitoringv1.ServiceMonitor{} err := c.Get(context.Background(), client.ObjectKey{ From 4cee742ef743e73c84bfe4022b190e2896e0deed Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 25 Sep 2024 19:03:28 +0200 Subject: [PATCH 07/14] fix test Signed-off-by: Coleen Iona Quadros --- .../monitoring.thanos.io_thanosreceives.yaml | 163 +++--------------- internal/pkg/manifests/options.go | 9 - test/configs/prometheus-crd.yaml | 12 -- test/configs/rbac.yaml | 43 ----- test/e2e/e2e_test.go | 6 +- test/utils/utils.go | 13 -- 6 files changed, 26 insertions(+), 220 deletions(-) delete mode 100644 test/configs/prometheus-crd.yaml delete mode 100644 test/configs/rbac.yaml diff --git a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml index ef7a362..bf84e7d 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml @@ -3571,16 +3571,16 @@ spec: - key type: object x-kubernetes-map-type: atomic - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean hashrings: description: Hashrings is a list of hashrings to route to. items: description: IngesterHashringSpec represents the configuration for a hashring to be used by the Thanos Receive StatefulSet. properties: + enableSelfMonitor: + default: true + description: Enable self monitoring for the Thanos component. + type: boolean externalLabels: additionalProperties: type: string @@ -3753,6 +3753,24 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceMonitor: + description: ServiceMonitorConfig is the configuration for + the ServiceMonitor. + properties: + enabled: + default: true + description: Enabled + type: boolean + labels: + additionalProperties: + type: string + description: Labels to add to the ServiceMonitor. + type: object + namespace: + description: Namespace to deploy the ServiceMonitor + in. + type: string + type: object storageSize: description: StorageSize is the size of the storage to be used by the Thanos Receive StatefulSet. @@ -3803,143 +3821,6 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - image: - description: Container image to use for the Thanos components. - type: string - imagePullPolicy: - default: IfNotPresent - description: |- - Image pull policy for the Thanos containers. - See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy for more details. - enum: - - Always - - Never - - IfNotPresent - type: string - imagePullSecrets: - description: |- - An optional list of references to Secrets in the same namespace - to use for pulling images from registries. - See http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - logFormat: - default: logfmt - description: Log format for Thanos. - enum: - - logfmt - - json - type: string - logLevel: - description: Log level for Thanos. - enum: - - debug - - info - - warn - - error - type: string - paused: - description: |- - When a resource is paused, no actions except for deletion - will be performed on the underlying objects. - type: boolean - resourceRequirements: - description: ResourceRequirements for the Thanos component container. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - serviceMonitor: - description: ServiceMonitorConfig is the configuration for the - ServiceMonitor. - properties: - enabled: - default: true - description: Enabled - type: boolean - labels: - additionalProperties: - type: string - description: Labels to add to the ServiceMonitor. - type: object - namespace: - description: Namespace to deploy the ServiceMonitor in. - type: string - type: object - version: - description: |- - Version of Thanos to be deployed. - If not specified, the operator assumes the latest upstream version of - Thanos available at the time when the version of the operator was released. - type: string required: - defaultObjectStorageConfig - hashrings diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index 48b4985..f3564b1 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -60,15 +60,6 @@ func (o Options) ToFlags() []string { fmt.Sprintf("--log.level=%s", *o.LogLevel), fmt.Sprintf("--log.format=%s", *o.LogFormat), } - if o.ServiceMonitorConfig == nil || o.ServiceMonitorConfig.Enabled == nil { - o.ServiceMonitorConfig.Enabled = ptr.To(true) - - } - if o.ServiceMonitorConfig == nil || *o.ServiceMonitorConfig.Namespace == "" { - o.ServiceMonitorConfig.Namespace = ptr.To(o.Namespace) - } - - return o } // GetContainerImage for the Options diff --git a/test/configs/prometheus-crd.yaml b/test/configs/prometheus-crd.yaml deleted file mode 100644 index d4f2fa0..0000000 --- a/test/configs/prometheus-crd.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: Prometheus -metadata: - name: test-prometheus - namespace: default -spec: - serviceAccountName: prometheus - replicas: 1 - serviceMonitorNamespaceSelector: {} - serviceMonitorSelector: {} - version: v2.54.0 - retention: "10d" diff --git a/test/configs/rbac.yaml b/test/configs/rbac.yaml deleted file mode 100644 index 5b8fd88..0000000 --- a/test/configs/rbac.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: prometheus -rules: -- apiGroups: [""] - resources: - - nodes - - nodes/metrics - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: ["get", "list", "watch"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: default - diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index daea771..0febf3f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -19,11 +19,12 @@ package e2e import ( "context" "fmt" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "os" "os/exec" "time" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + rbacv1 "k8s.io/api/rbac/v1" "github.com/thanos-community/thanos-operator/internal/pkg/manifests" @@ -441,7 +442,8 @@ var _ = Describe("controller", Ordered, func() { return utils.VerifyServiceMonitor(c, queryName, namespace) }, time.Minute*5, time.Second*10).Should(BeTrue()) - query.Spec.CommonThanosFields.EnableSelfMonitor = false + enable := false + query.Spec.CommonThanosFields.EnableSelfMonitor = &enable err = c.Update(context.Background(), query) Expect(err).To(BeNil()) diff --git a/test/utils/utils.go b/test/utils/utils.go index 3cdc528..d502a46 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -549,19 +549,6 @@ func VerifyCfgMapOrSecretEnvVarExists(c client.Client, obj client.Object, name, } } -func ApplyPrometheusCRD() error { - wd, _ := os.Getwd() - promCmd := exec.Command("kubectl", "apply", "-f", wd+"/test/configs/prometheus-crd.yaml", "--server-side") - if _, err := Run(promCmd); err != nil { - return fmt.Errorf("failed to apply Prometheus CRD: %v", err) - } - cmd := exec.Command("kubectl", "apply", "-f", wd+"/test/configs/rbac.yaml", "--server-side") - if _, err := Run(cmd); err != nil { - return fmt.Errorf("failed to apply RBAC: %v for Prometheus", err) - } - return nil -} - func SetUpPrometheus(c client.Client) error { if err := CreateServiceAccount(c); err != nil { return err From 633c2bd65c1af1ec87e36519cc5d13687d4eb9bd Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 25 Sep 2024 19:10:17 +0200 Subject: [PATCH 08/14] update --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b9a44bc..6e03474 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/go.mod b/go.mod index 67923cf..6d9cee9 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/imdario/mergo v0.3.16 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 - github.com/prometheus/client_golang v1.20.4 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.1 + github.com/prometheus/client_golang v1.20.4 github.com/prometheus/common v0.59.1 github.com/prometheus/prometheus v0.54.1 github.com/stretchr/testify v1.9.0 From 2ec3d9175793b1497bb4906a39bf795cab1eb196 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Wed, 25 Sep 2024 19:41:46 +0200 Subject: [PATCH 09/14] refactor Signed-off-by: Coleen Iona Quadros --- api/v1alpha1/types.go | 4 ---- api/v1alpha1/zz_generated.deepcopy.go | 5 ----- config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml | 4 ---- config/crd/bases/monitoring.thanos.io_thanosqueries.yaml | 8 -------- config/crd/bases/monitoring.thanos.io_thanosreceives.yaml | 8 -------- config/crd/bases/monitoring.thanos.io_thanosrulers.yaml | 4 ---- config/crd/bases/monitoring.thanos.io_thanosstores.yaml | 4 ---- internal/controller/transform.go | 1 + internal/pkg/manifests/options.go | 2 -- internal/pkg/manifests/query/builder.go | 2 +- internal/pkg/manifests/service_monitor.go | 1 + test/e2e/e2e_test.go | 2 +- 12 files changed, 4 insertions(+), 41 deletions(-) diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go index 0168d56..22a1461 100644 --- a/api/v1alpha1/types.go +++ b/api/v1alpha1/types.go @@ -75,10 +75,6 @@ type CommonThanosFields struct { // +kubebuilder:default:=logfmt // +kubebuilder:validation:Optional LogFormat *string `json:"logFormat,omitempty"` - // Enable self monitoring for the Thanos component. - // +kubebuilder:validation:Optional - // +kubebuilder:default:=true - EnableSelfMonitor *bool `json:"enableSelfMonitor,omitempty"` // ServiceMonitorConfig is the configuration for the ServiceMonitor. // +kubebuilder:validation:Optional ServiceMonitorConfig *ServiceMonitorConfig `json:"serviceMonitor,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6e03474..a8fd8f5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -144,11 +144,6 @@ func (in *CommonThanosFields) DeepCopyInto(out *CommonThanosFields) { *out = new(string) **out = **in } - if in.EnableSelfMonitor != nil { - in, out := &in.EnableSelfMonitor, &out.EnableSelfMonitor - *out = new(bool) - **out = **in - } if in.ServiceMonitorConfig != nil { in, out := &in.ServiceMonitorConfig, &out.ServiceMonitorConfig *out = new(ServiceMonitorConfig) diff --git a/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml b/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml index e89f07d..ee46efe 100644 --- a/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanoscompacts.yaml @@ -99,10 +99,6 @@ spec: pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string type: object - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean groupConfig: description: GroupConfig defines settings for group handling. properties: diff --git a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml index 8d125bd..c4bde90 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosqueries.yaml @@ -3574,10 +3574,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean image: description: Container image to use for the Thanos components. type: string @@ -7162,10 +7158,6 @@ spec: default: true description: CompressResponses enables response compression type: boolean - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean image: description: Container image to use for the Thanos components. type: string diff --git a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml index bf84e7d..b594d9b 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosreceives.yaml @@ -3577,10 +3577,6 @@ spec: description: IngesterHashringSpec represents the configuration for a hashring to be used by the Thanos Receive StatefulSet. properties: - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean externalLabels: additionalProperties: type: string @@ -7331,10 +7327,6 @@ spec: - name type: object type: array - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean externalLabels: additionalProperties: type: string diff --git a/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml b/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml index f379799..ad5c48b 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosrulers.yaml @@ -3561,10 +3561,6 @@ spec: - key type: object x-kubernetes-map-type: atomic - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean evaluationInterval: default: 1m description: EvaluationInterval is the default interval at which rules diff --git a/config/crd/bases/monitoring.thanos.io_thanosstores.yaml b/config/crd/bases/monitoring.thanos.io_thanosstores.yaml index 9e0927b..b54d453 100644 --- a/config/crd/bases/monitoring.thanos.io_thanosstores.yaml +++ b/config/crd/bases/monitoring.thanos.io_thanosstores.yaml @@ -3551,10 +3551,6 @@ spec: - key type: object x-kubernetes-map-type: atomic - enableSelfMonitor: - default: true - description: Enable self monitoring for the Thanos component. - type: boolean ignoreDeletionMarksDelay: default: 24h description: |- diff --git a/internal/controller/transform.go b/internal/controller/transform.go index 098655e..950fe4e 100644 --- a/internal/controller/transform.go +++ b/internal/controller/transform.go @@ -170,6 +170,7 @@ func commonToOpts( LogLevel: common.LogLevel, LogFormat: common.LogFormat, Additional: additionalToOpts(additional), + ServiceMonitorConfig: common.ServiceMonitorConfig, } } diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index f3564b1..a07736a 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -40,8 +40,6 @@ type Options struct { LogLevel *string // LogFormat is the log format for the component LogFormat *string - // EnableServiceMonitor is a flag to enable ServiceMonitor - EnableServiceMonitor bool //ServiceMonitorConfig is the configuration for the ServiceMonitor ServiceMonitorConfig *v1alpha1.ServiceMonitorConfig } diff --git a/internal/pkg/manifests/query/builder.go b/internal/pkg/manifests/query/builder.go index cdaf0b5..9bc656e 100644 --- a/internal/pkg/manifests/query/builder.go +++ b/internal/pkg/manifests/query/builder.go @@ -210,7 +210,7 @@ func newQueryService(opts Options, selectorLabels, objectMetaLabels map[string]s servicePorts = append(servicePorts, opts.Additional.ServicePorts...) } - if opts.EnableServiceMonitor { + if opts.ServiceMonitorConfig.Enabled != nil && *opts.ServiceMonitorConfig.Enabled { objectMetaLabels["thanos-self-monitoring"] = opts.Name } diff --git a/internal/pkg/manifests/service_monitor.go b/internal/pkg/manifests/service_monitor.go index c4e8124..b42d723 100644 --- a/internal/pkg/manifests/service_monitor.go +++ b/internal/pkg/manifests/service_monitor.go @@ -19,6 +19,7 @@ func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor for k, v := range opts.ServiceMonitorConfig.Labels { labels[k] = v } + labels["thanos-self-monitoring"] = opts.Name return &monitoringv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{ Kind: "ServiceMonitor", diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0febf3f..ea1c1df 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -443,7 +443,7 @@ var _ = Describe("controller", Ordered, func() { }, time.Minute*5, time.Second*10).Should(BeTrue()) enable := false - query.Spec.CommonThanosFields.EnableSelfMonitor = &enable + query.Spec.CommonThanosFields.ServiceMonitorConfig.Enabled = &enable err = c.Update(context.Background(), query) Expect(err).To(BeNil()) From e82fc033c2af91c82ba174dec02d2c9e62679f85 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Thu, 26 Sep 2024 16:04:18 +0200 Subject: [PATCH 10/14] fix tests Signed-off-by: Coleen Iona Quadros --- internal/controller/suite_test.go | 1 + internal/controller/thanosquery_controller.go | 16 +++++++++- .../controller/thanosquery_controller_test.go | 12 ++++++++ internal/controller/transform.go | 29 ++++++++++++++++++- internal/pkg/handlers/handlers.go | 26 +++++++++++++++++ internal/pkg/manifests/options.go | 9 ++++-- internal/pkg/manifests/query/builder.go | 3 +- .../pkg/manifests/service_monitor_test.go | 14 ++++++++- test/e2e/e2e_test.go | 22 -------------- test/utils/utils.go | 8 ++--- 10 files changed, 107 insertions(+), 33 deletions(-) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index e76b07b..898fdd0 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -68,6 +68,7 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "test", "configs", "service-monitor.yaml"), }, ErrorIfCRDPathMissing: true, diff --git a/internal/controller/thanosquery_controller.go b/internal/controller/thanosquery_controller.go index 68654a4..129104b 100644 --- a/internal/controller/thanosquery_controller.go +++ b/internal/controller/thanosquery_controller.go @@ -141,11 +141,25 @@ func (r *ThanosQueryReconciler) syncResources(ctx context.Context, query monitor objs = append(objs, frontendObjs...) } - if errCount := r.handler.CreateOrUpdate(ctx, query.GetNamespace(), &query, objs); errCount > 0 { + var errCount int + if errCount = r.handler.CreateOrUpdate(ctx, query.GetNamespace(), &query, objs); errCount > 0 { r.ControllerBaseMetrics.ClientErrorsTotal.WithLabelValues(manifestquery.Name).Add(float64(errCount)) return fmt.Errorf("failed to create or update %d resources for the querier and query frontend", errCount) } + if query.Spec.ServiceMonitorConfig == nil || (query.Spec.ServiceMonitorConfig == nil && query.Spec.ServiceMonitorConfig.Enabled != nil && !*query.Spec.ServiceMonitorConfig.Enabled) { + if errCount = r.handler.DeleteResource(ctx, []client.Object{&monitoringv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: query.GetName(), + Namespace: *query.Spec.ServiceMonitorConfig.Namespace, + }, + }, + }); errCount > 0 { + r.ControllerBaseMetrics.ClientErrorsTotal.WithLabelValues(manifestquery.Name).Add(float64(errCount)) + return fmt.Errorf("failed to delete %d resources for the querier and query frontend", errCount) + } + } + return nil } diff --git a/internal/controller/thanosquery_controller_test.go b/internal/controller/thanosquery_controller_test.go index f8942fc..f9dd6e5 100644 --- a/internal/controller/thanosquery_controller_test.go +++ b/internal/controller/thanosquery_controller_test.go @@ -243,6 +243,18 @@ var _ = Describe("ThanosQuery Controller", Ordered, func() { }, time.Second*30, time.Second*10).Should(Succeed()) }) + By("removing service monitor when disabled", func() { + Expect(utils.VerifyServiceMonitor(k8sClient, resourceName, ns)).To(BeTrue()) + + enableSelfMonitor := false + resource.Spec.ServiceMonitorConfig.Enabled = &enableSelfMonitor + Expect(k8sClient.Update(context.Background(), resource)).Should(Succeed()) + + Eventually(func() bool { + return utils.VerifyServiceMonitorDeleted(k8sClient, resourceName, ns) + }, time.Minute*1, time.Second*10).Should(BeTrue()) + }) + By("checking paused state", func() { isPaused := true resource.Spec.Paused = &isPaused diff --git a/internal/controller/transform.go b/internal/controller/transform.go index 950fe4e..fca4896 100644 --- a/internal/controller/transform.go +++ b/internal/controller/transform.go @@ -3,6 +3,8 @@ package controller import ( "fmt" + "k8s.io/utils/ptr" + "github.com/thanos-community/thanos-operator/api/v1alpha1" "github.com/thanos-community/thanos-operator/internal/pkg/manifests" manifestquery "github.com/thanos-community/thanos-operator/internal/pkg/manifests/query" @@ -170,7 +172,7 @@ func commonToOpts( LogLevel: common.LogLevel, LogFormat: common.LogFormat, Additional: additionalToOpts(additional), - ServiceMonitorConfig: common.ServiceMonitorConfig, + ServiceMonitorConfig: serviceMonitorConfigToOpts(common.ServiceMonitorConfig, namespace, labels), } } @@ -185,3 +187,28 @@ func additionalToOpts(in v1alpha1.Additional) manifests.Additional { ServicePorts: in.ServicePorts, } } + +func serviceMonitorConfigToOpts(in *v1alpha1.ServiceMonitorConfig, namespace string, labels map[string]string) manifests.ServiceMonitorConfig { + if in == nil { + return manifests.ServiceMonitorConfig{ + Enabled: ptr.To(true), + Namespace: &namespace, + Labels: labels, + } + } + + if in != nil && in.Enabled == nil { + in.Enabled = ptr.To(true) + } + if in != nil && in.Namespace == nil { + in.Namespace = &namespace + } + if in != nil && in.Labels == nil { + in.Labels = labels + } + return manifests.ServiceMonitorConfig{ + Enabled: in.Enabled, + Labels: in.Labels, + Namespace: in.Namespace, + } +} diff --git a/internal/pkg/handlers/handlers.go b/internal/pkg/handlers/handlers.go index fb6e859..c477607 100644 --- a/internal/pkg/handlers/handlers.go +++ b/internal/pkg/handlers/handlers.go @@ -4,6 +4,7 @@ import ( "context" "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" "github.com/thanos-community/thanos-operator/internal/pkg/manifests" @@ -67,3 +68,28 @@ func (h *Handler) CreateOrUpdate(ctx context.Context, namespace string, owner cl } return errCount } + +// Delete resources if they exist. +func (h *Handler) DeleteResource(ctx context.Context, objs []client.Object) int { + var errCount int + for _, obj := range objs { + if err := h.client.Delete(ctx, obj); err != nil && !errors.IsNotFound(err) { + h.logger.Error( + err, "failed to delete resource", + "gvk", obj.GetObjectKind().GroupVersionKind().String(), + "resource", obj.GetName(), + "namespace", obj.GetNamespace(), + ) + errCount++ + continue + } + + h.logger.V(1).Info( + "resource deleted", + "gvk", obj.GetObjectKind().GroupVersionKind().String(), + "resource", obj.GetName(), + "namespace", obj.GetNamespace(), + ) + } + return errCount +} diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index a07736a..ecf9f9b 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -3,7 +3,6 @@ package manifests import ( "fmt" - "github.com/thanos-community/thanos-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -41,7 +40,7 @@ type Options struct { // LogFormat is the log format for the component LogFormat *string //ServiceMonitorConfig is the configuration for the ServiceMonitor - ServiceMonitorConfig *v1alpha1.ServiceMonitorConfig + ServiceMonitorConfig } // ToFlags returns the flags for the Options @@ -226,3 +225,9 @@ func (rc RelabelConfigs) ToFlags() string { } type Duration string + +type ServiceMonitorConfig struct { + Enabled *bool + Labels map[string]string + Namespace *string +} diff --git a/internal/pkg/manifests/query/builder.go b/internal/pkg/manifests/query/builder.go index 9bc656e..2dd753b 100644 --- a/internal/pkg/manifests/query/builder.go +++ b/internal/pkg/manifests/query/builder.go @@ -36,8 +36,7 @@ type Options struct { LookbackDelta string MaxConcurrent int - Endpoints []Endpoint - EnableServiceMonitor bool + Endpoints []Endpoint } type EndpointType string diff --git a/internal/pkg/manifests/service_monitor_test.go b/internal/pkg/manifests/service_monitor_test.go index f46b126..bf5a040 100644 --- a/internal/pkg/manifests/service_monitor_test.go +++ b/internal/pkg/manifests/service_monitor_test.go @@ -1,6 +1,10 @@ package manifests -import "testing" +import ( + "testing" + + "k8s.io/utils/ptr" +) func TestBuildServiceMonitor(t *testing.T) { opts := Options{ @@ -10,6 +14,14 @@ func TestBuildServiceMonitor(t *testing.T) { "app.kubernetes.io/name": "thanos", "app.kubernetes.io/instance": "thanos-stack", }, + ServiceMonitorConfig: ServiceMonitorConfig{ + Enabled: ptr.To(true), + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "thanos-operator", + "test": "test", + }, + Namespace: ptr.To("monitoring"), + }, } for _, tc := range []struct { name string diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index ea1c1df..1e12ece 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -431,28 +431,6 @@ var _ = Describe("controller", Ordered, func() { }, time.Minute*1, time.Second*5).Should(Succeed()) }) }) - - Context("Thanos Query Service Monitor", func() { - It("should remove service monitor when disabled", func() { - query := &v1alpha1.ThanosQuery{} - err := c.Get(context.Background(), client.ObjectKey{Name: queryName, Namespace: namespace}, query) - Expect(err).To(BeNil()) - - Eventually(func() bool { - return utils.VerifyServiceMonitor(c, queryName, namespace) - }, time.Minute*5, time.Second*10).Should(BeTrue()) - - enable := false - query.Spec.CommonThanosFields.ServiceMonitorConfig.Enabled = &enable - err = c.Update(context.Background(), query) - Expect(err).To(BeNil()) - - Eventually(func() bool { - return utils.VerifyServiceMonitorDeleted(c, queryName, namespace) - }, time.Minute*5, time.Second*10).Should(BeTrue()) - - }) - }) }) Describe("Thanos Ruler", Ordered, func() { diff --git a/test/utils/utils.go b/test/utils/utils.go index d502a46..663441e 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -550,16 +550,16 @@ func VerifyCfgMapOrSecretEnvVarExists(c client.Client, obj client.Object, name, } func SetUpPrometheus(c client.Client) error { - if err := CreateServiceAccount(c); err != nil { + if err := CreateServiceAccount(c); err != nil && !errors.IsAlreadyExists(err) { return err } - if err := CreateClusterRole(c); err != nil { + if err := CreateClusterRole(c); err != nil && !errors.IsAlreadyExists(err) { return err } - if err := CreateClusterRoleBinding(c); err != nil { + if err := CreateClusterRoleBinding(c); err != nil && !errors.IsAlreadyExists(err) { return err } - if err := CreatePrometheus(c); err != nil { + if err := CreatePrometheus(c); err != nil && !errors.IsAlreadyExists(err) { return err } return nil From 2899dbe73f6ae4086cc08ff1d4b339ad3173d181 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Thu, 26 Sep 2024 16:18:36 +0200 Subject: [PATCH 11/14] refactor Signed-off-by: Coleen Iona Quadros --- internal/pkg/manifests/service_monitor_test.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/pkg/manifests/service_monitor_test.go b/internal/pkg/manifests/service_monitor_test.go index bf5a040..c5651e8 100644 --- a/internal/pkg/manifests/service_monitor_test.go +++ b/internal/pkg/manifests/service_monitor_test.go @@ -10,17 +10,9 @@ func TestBuildServiceMonitor(t *testing.T) { opts := Options{ Name: "thanos-stack", Namespace: "ns", - Labels: map[string]string{ - "app.kubernetes.io/name": "thanos", - "app.kubernetes.io/instance": "thanos-stack", - }, ServiceMonitorConfig: ServiceMonitorConfig{ - Enabled: ptr.To(true), - Labels: map[string]string{ - "app.kubernetes.io/managed-by": "thanos-operator", - "test": "test", - }, - Namespace: ptr.To("monitoring"), + Enabled: ptr.To(true), + Namespace: ptr.To("ns"), }, } for _, tc := range []struct { From 527e8358f0cda814c198d81284fbe1368cf747b9 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Thu, 26 Sep 2024 18:43:34 +0200 Subject: [PATCH 12/14] fix test Signed-off-by: Coleen Iona Quadros --- go.mod | 1 + go.sum | 1 + internal/controller/thanosquery_controller.go | 6 +++--- internal/controller/thanosquery_controller_test.go | 11 ++++++++--- internal/controller/transform.go | 8 ++++---- internal/pkg/manifests/options.go | 4 ++-- internal/pkg/manifests/query/builder.go | 4 ++-- internal/pkg/manifests/service_monitor.go | 2 +- internal/pkg/manifests/service_monitor_test.go | 6 ++---- 9 files changed, 24 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 6d9cee9..b03821d 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.22.3 require ( github.com/go-logr/logr v1.4.2 github.com/golang/snappy v0.0.4 + github.com/google/martian v2.1.0+incompatible github.com/imdario/mergo v0.3.16 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 diff --git a/go.sum b/go.sum index 5b65ff1..78cf8c1 100644 --- a/go.sum +++ b/go.sum @@ -212,6 +212,7 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= diff --git a/internal/controller/thanosquery_controller.go b/internal/controller/thanosquery_controller.go index 129104b..ce6565c 100644 --- a/internal/controller/thanosquery_controller.go +++ b/internal/controller/thanosquery_controller.go @@ -147,11 +147,11 @@ func (r *ThanosQueryReconciler) syncResources(ctx context.Context, query monitor return fmt.Errorf("failed to create or update %d resources for the querier and query frontend", errCount) } - if query.Spec.ServiceMonitorConfig == nil || (query.Spec.ServiceMonitorConfig == nil && query.Spec.ServiceMonitorConfig.Enabled != nil && !*query.Spec.ServiceMonitorConfig.Enabled) { + if query.Spec.ServiceMonitorConfig != nil && query.Spec.ServiceMonitorConfig.Enabled != nil && !*query.Spec.ServiceMonitorConfig.Enabled { if errCount = r.handler.DeleteResource(ctx, []client.Object{&monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ - Name: query.GetName(), - Namespace: *query.Spec.ServiceMonitorConfig.Namespace, + Name: QueryNameFromParent(query.GetName()), + Namespace: query.GetNamespace(), }, }, }); errCount > 0 { diff --git a/internal/controller/thanosquery_controller_test.go b/internal/controller/thanosquery_controller_test.go index f9dd6e5..e104038 100644 --- a/internal/controller/thanosquery_controller_test.go +++ b/internal/controller/thanosquery_controller_test.go @@ -244,14 +244,19 @@ var _ = Describe("ThanosQuery Controller", Ordered, func() { }) By("removing service monitor when disabled", func() { - Expect(utils.VerifyServiceMonitor(k8sClient, resourceName, ns)).To(BeTrue()) + Expect(utils.VerifyServiceMonitor(k8sClient, QueryNameFromParent(resourceName), ns)).To(BeTrue()) enableSelfMonitor := false - resource.Spec.ServiceMonitorConfig.Enabled = &enableSelfMonitor + + resource.Spec.CommonThanosFields = monitoringthanosiov1alpha1.CommonThanosFields{ + ServiceMonitorConfig: &monitoringthanosiov1alpha1.ServiceMonitorConfig{ + Enabled: &enableSelfMonitor, + }, + } Expect(k8sClient.Update(context.Background(), resource)).Should(Succeed()) Eventually(func() bool { - return utils.VerifyServiceMonitorDeleted(k8sClient, resourceName, ns) + return utils.VerifyServiceMonitorDeleted(k8sClient, QueryNameFromParent(resourceName), ns) }, time.Minute*1, time.Second*10).Should(BeTrue()) }) diff --git a/internal/controller/transform.go b/internal/controller/transform.go index fca4896..ac5b97e 100644 --- a/internal/controller/transform.go +++ b/internal/controller/transform.go @@ -191,8 +191,8 @@ func additionalToOpts(in v1alpha1.Additional) manifests.Additional { func serviceMonitorConfigToOpts(in *v1alpha1.ServiceMonitorConfig, namespace string, labels map[string]string) manifests.ServiceMonitorConfig { if in == nil { return manifests.ServiceMonitorConfig{ - Enabled: ptr.To(true), - Namespace: &namespace, + Enabled: true, + Namespace: namespace, Labels: labels, } } @@ -207,8 +207,8 @@ func serviceMonitorConfigToOpts(in *v1alpha1.ServiceMonitorConfig, namespace str in.Labels = labels } return manifests.ServiceMonitorConfig{ - Enabled: in.Enabled, + Enabled: *in.Enabled, Labels: in.Labels, - Namespace: in.Namespace, + Namespace: *in.Namespace, } } diff --git a/internal/pkg/manifests/options.go b/internal/pkg/manifests/options.go index ecf9f9b..e9a3fa8 100644 --- a/internal/pkg/manifests/options.go +++ b/internal/pkg/manifests/options.go @@ -227,7 +227,7 @@ func (rc RelabelConfigs) ToFlags() string { type Duration string type ServiceMonitorConfig struct { - Enabled *bool + Enabled bool Labels map[string]string - Namespace *string + Namespace string } diff --git a/internal/pkg/manifests/query/builder.go b/internal/pkg/manifests/query/builder.go index 2dd753b..4c2cb2f 100644 --- a/internal/pkg/manifests/query/builder.go +++ b/internal/pkg/manifests/query/builder.go @@ -65,7 +65,7 @@ func BuildQuery(opts Options) []client.Object { objs = append(objs, manifests.BuildServiceAccount(Name, opts.Namespace, GetRequiredLabels())) objs = append(objs, newQueryDeployment(opts, selectorLabels, objectMetaLabels)) objs = append(objs, newQueryService(opts, selectorLabels, objectMetaLabels)) - if opts.ServiceMonitorConfig.Enabled != nil && *opts.ServiceMonitorConfig.Enabled { + if opts.ServiceMonitorConfig.Enabled { objs = append(objs, manifests.BuildServiceMonitor(opts.Options, HTTPPortName)) } return objs @@ -209,7 +209,7 @@ func newQueryService(opts Options, selectorLabels, objectMetaLabels map[string]s servicePorts = append(servicePorts, opts.Additional.ServicePorts...) } - if opts.ServiceMonitorConfig.Enabled != nil && *opts.ServiceMonitorConfig.Enabled { + if opts.ServiceMonitorConfig.Enabled { objectMetaLabels["thanos-self-monitoring"] = opts.Name } diff --git a/internal/pkg/manifests/service_monitor.go b/internal/pkg/manifests/service_monitor.go index b42d723..3bf57be 100644 --- a/internal/pkg/manifests/service_monitor.go +++ b/internal/pkg/manifests/service_monitor.go @@ -27,7 +27,7 @@ func BuildServiceMonitor(opts Options, port string) *monitoringv1.ServiceMonitor }, ObjectMeta: metav1.ObjectMeta{ Name: opts.Name, - Namespace: *opts.ServiceMonitorConfig.Namespace, + Namespace: opts.Namespace, // Future: use namespace from ServiceMonitorConfig Labels: labels, }, Spec: monitoringv1.ServiceMonitorSpec{ diff --git a/internal/pkg/manifests/service_monitor_test.go b/internal/pkg/manifests/service_monitor_test.go index c5651e8..4cc65ab 100644 --- a/internal/pkg/manifests/service_monitor_test.go +++ b/internal/pkg/manifests/service_monitor_test.go @@ -2,8 +2,6 @@ package manifests import ( "testing" - - "k8s.io/utils/ptr" ) func TestBuildServiceMonitor(t *testing.T) { @@ -11,8 +9,8 @@ func TestBuildServiceMonitor(t *testing.T) { Name: "thanos-stack", Namespace: "ns", ServiceMonitorConfig: ServiceMonitorConfig{ - Enabled: ptr.To(true), - Namespace: ptr.To("ns"), + Enabled: true, + Namespace: "ns", }, } for _, tc := range []struct { From 9995edf3608272551ec76100dc01979ddeba3f23 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Thu, 26 Sep 2024 20:01:31 +0200 Subject: [PATCH 13/14] lint Signed-off-by: Coleen Iona Quadros --- go.mod | 1 - go.sum | 1 - internal/controller/transform.go | 1 - test/e2e/e2e_test.go | 2 +- test/utils/utils.go | 5 +++-- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 22f6177..a74f413 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ toolchain go1.22.3 require ( github.com/go-logr/logr v1.4.2 github.com/golang/snappy v0.0.4 - github.com/google/martian v2.1.0+incompatible github.com/imdario/mergo v0.3.16 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 diff --git a/go.sum b/go.sum index 78cf8c1..5b65ff1 100644 --- a/go.sum +++ b/go.sum @@ -212,7 +212,6 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= diff --git a/internal/controller/transform.go b/internal/controller/transform.go index c445c0e..d407f21 100644 --- a/internal/controller/transform.go +++ b/internal/controller/transform.go @@ -15,7 +15,6 @@ import ( manifestsstore "github.com/thanos-community/thanos-operator/internal/pkg/manifests/store" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/utils/ptr" ) func queryV1Alpha1ToOptions(in v1alpha1.ThanosQuery) manifestquery.Options { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 51c1557..79d388a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -62,7 +62,7 @@ const ( compactName = "example-compact" prometheusPort = 9090 - + hashringName = "default" ) diff --git a/test/utils/utils.go b/test/utils/utils.go index 663441e..4dad566 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -29,9 +29,10 @@ import ( "strings" "time" + "k8s.io/utils/ptr" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" "k8s.io/apimachinery/pkg/api/errors" @@ -574,7 +575,7 @@ func CreatePrometheus(c client.Client) error { Spec: monitoringv1.PrometheusSpec{ CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ ServiceAccountName: "prometheus", - Replicas: pointer.Int32(1), + Replicas: ptr.To(int32(1)), ServiceMonitorNamespaceSelector: &metav1.LabelSelector{}, ServiceMonitorSelector: &metav1.LabelSelector{}, Version: "v2.54.0", From 753e4d52479f983616b4e15c334e66b35ceb4694 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Fri, 27 Sep 2024 09:36:54 +0200 Subject: [PATCH 14/14] lint Signed-off-by: Coleen Iona Quadros --- internal/controller/thanosquery_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/controller/thanosquery_controller.go b/internal/controller/thanosquery_controller.go index 2c492cb..173dd99 100644 --- a/internal/controller/thanosquery_controller.go +++ b/internal/controller/thanosquery_controller.go @@ -22,7 +22,6 @@ import ( "github.com/go-logr/logr" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - "github.com/prometheus/client_golang/prometheus" monitoringthanosiov1alpha1 "github.com/thanos-community/thanos-operator/api/v1alpha1" "github.com/thanos-community/thanos-operator/internal/pkg/handlers"