From 0d980c8211686d709de0edcaca51c2ffadca02f0 Mon Sep 17 00:00:00 2001 From: bychkov Date: Wed, 13 Sep 2023 15:49:03 +0300 Subject: [PATCH 1/3] Collector for working with data from the system profile --- REFERENCE.md | 2 + exporter/exporter.go | 12 +++ exporter/profile_status_collector.go | 96 +++++++++++++++++++++++ exporter/profile_status_collector_test.go | 69 ++++++++++++++++ main.go | 5 ++ 5 files changed, 184 insertions(+) create mode 100644 exporter/profile_status_collector.go create mode 100644 exporter/profile_status_collector_test.go diff --git a/REFERENCE.md b/REFERENCE.md index bf36356e7..65c8b3649 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -23,5 +23,7 @@ |--collector.collstats|Enable collecting metrics from $collStats| |--collect-all|Enable all collectors. Same as specifying all --collector.\| |--collector.collstats-limit=0|Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \ collections. 0=No limit| +|--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval| +|--collector.profile|Enable collecting metrics from profile| |--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC || |--version|Show version and exit| \ No newline at end of file diff --git a/exporter/exporter.go b/exporter/exporter.go index 65af03158..d1bf469a9 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -59,6 +59,7 @@ type Opts struct { DisableDefaultRegistry bool DiscoveringMode bool GlobalConnPool bool + ProfileTimeTS int CollectAll bool EnableDBStats bool @@ -68,6 +69,7 @@ type Opts struct { EnableTopMetrics bool EnableIndexStats bool EnableCollStats bool + EnableProfile bool EnableOverrideDescendingIndex bool @@ -170,6 +172,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.opts.EnableTopMetrics = true e.opts.EnableReplicasetStatus = true e.opts.EnableIndexStats = true + e.opts.EnableProfile = true } // arbiter only have isMaster privileges @@ -180,6 +183,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.opts.EnableTopMetrics = false e.opts.EnableReplicasetStatus = false e.opts.EnableIndexStats = false + e.opts.EnableProfile = false } // If we manually set the collection names we want or auto discovery is set. @@ -210,6 +214,12 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol registry.MustRegister(cc) } + if e.opts.EnableProfile && nodeType != typeMongos && limitsOk && requestOpts.EnableProfile && e.opts.ProfileTimeTS != 0 { + pc := newProfileCollector(ctx, client, e.opts.Logger, + e.opts.CompatibleMode, topologyInfo, e.opts.ProfileTimeTS) + registry.MustRegister(pc) + } + if e.opts.EnableTopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableTopMetrics { tc := newTopCollector(ctx, client, e.opts.Logger, e.opts.CompatibleMode, topologyInfo) @@ -292,6 +302,8 @@ func (e *Exporter) Handler() http.Handler { requestOpts.EnableIndexStats = true case "collstats": requestOpts.EnableCollStats = true + case "profile": + requestOpts.EnableProfile = true } } diff --git a/exporter/profile_status_collector.go b/exporter/profile_status_collector.go new file mode 100644 index 000000000..910cd1559 --- /dev/null +++ b/exporter/profile_status_collector.go @@ -0,0 +1,96 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + "time" + + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type profileCollector struct { + ctx context.Context + base *baseCollector + compatibleMode bool + topologyInfo labelsGetter + profiletimets int +} + +// newProfileCollector creates a collector for being processed queries. +func newProfileCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, + compatible bool, topology labelsGetter, profileTimeTS int, +) *profileCollector { + return &profileCollector{ + ctx: ctx, + base: newBaseCollector(client, logger), + compatibleMode: compatible, + topologyInfo: topology, + profiletimets: profileTimeTS, + } +} + +func (d *profileCollector) Describe(ch chan<- *prometheus.Desc) { + d.base.Describe(d.ctx, ch, d.collect) +} + +func (d *profileCollector) Collect(ch chan<- prometheus.Metric) { + d.base.Collect(ch) +} + +func (d *profileCollector) collect(ch chan<- prometheus.Metric) { + defer measureCollectTime(ch, "mongodb", "profile")() + + logger := d.base.logger + client := d.base.client + timeScrape := d.profiletimets + + databases, err := databases(d.ctx, client, nil, nil) + if err != nil { + errors.Wrap(err, "cannot get the database names list") + return + } + + // Now time + '--collector.profile-time-ts' + ts := primitive.NewDateTimeFromTime(time.Now().Add(-time.Duration(time.Second * time.Duration(timeScrape)))) + + labels := d.topologyInfo.baseLabels() + + // Get all slow queries from all databases + cmd := bson.M{"ts": bson.M{"$gte": ts}} + for _, db := range databases { + res, err := client.Database(db).Collection("system.profile").CountDocuments(d.ctx, cmd) + if err != nil { + errors.Wrapf(err, "cannot read system.profile") + break + } + labels["database"] = db + + m := primitive.M{"count": res} + + logger.Debug("profile response from MongoDB:") + debugResult(logger, primitive.M{db: m}) + + for _, metric := range makeMetrics("profile_slow_query", m, labels, d.compatibleMode) { + ch <- metric + } + } +} diff --git a/exporter/profile_status_collector_test.go b/exporter/profile_status_collector_test.go new file mode 100644 index 000000000..df47d5a34 --- /dev/null +++ b/exporter/profile_status_collector_test.go @@ -0,0 +1,69 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + + "github.com/percona/mongodb_exporter/internal/tu" +) + +func TestProfileCollector(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + client := tu.DefaultTestClient(ctx, t) + + database := client.Database("testdb") + database.Drop(ctx) //nolint + + defer func() { + err := database.Drop(ctx) + assert.NoError(t, err) + }() + + // Enable database profiler https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/ + cmd := bson.M{"profile": 2} + _ = database.RunCommand(ctx, cmd) + + ti := labelsGetterMock{} + + c := newProfileCollector(ctx, client, logrus.New(), false, ti, 30) + + expected := strings.NewReader(` + # HELP mongodb_profile_slow_query_count profile_slow_query. + # TYPE mongodb_profile_slow_query_count counter + mongodb_profile_slow_query_count{database="admin"} 0 + mongodb_profile_slow_query_count{database="config"} 0 + mongodb_profile_slow_query_count{database="local"} 0 + mongodb_profile_slow_query_count{database="testdb"} 0` + + "\n") + + filter := []string{ + "mongodb_profile_slow_query_count", + } + + err := testutil.CollectAndCompare(c, expected, filter...) + assert.NoError(t, err) +} diff --git a/main.go b/main.go index 839031a80..012265d83 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,7 @@ type GlobalFlags struct { EnableTopMetrics bool `name:"collector.topmetrics" help:"Enable collecting metrics from top admin command"` EnableIndexStats bool `name:"collector.indexstats" help:"Enable collecting metrics from $indexStats"` EnableCollStats bool `name:"collector.collstats" help:"Enable collecting metrics from $collStats"` + EnableProfile bool `name:"collector.profile" help:"Enable collecting metrics from profile"` EnableOverrideDescendingIndex bool `name:"metrics.overridedescendingindex" help:"Enable descending index name override to replace -1 with _DESC"` @@ -58,6 +59,8 @@ type GlobalFlags struct { CollStatsLimit int `name:"collector.collstats-limit" help:"Disable collstats, dbstats, topmetrics and indexstats collector if there are more than collections. 0=No limit" default:"0"` + ProfileTimeTS int `name:"collector.profile-time-ts" help:"Set time for scrape slow queries." default:"30"` + DiscoveringMode bool `name:"discovering-mode" help:"Enable autodiscover collections" negatable:""` CompatibleMode bool `name:"compatible-mode" help:"Enable old mongodb-exporter compatible metrics" negatable:""` Version bool `name:"version" help:"Show version and exit"` @@ -130,11 +133,13 @@ func buildExporter(opts GlobalFlags) *exporter.Exporter { EnableDBStatsFreeStorage: opts.EnableDBStatsFreeStorage, EnableIndexStats: opts.EnableIndexStats, EnableCollStats: opts.EnableCollStats, + EnableProfile: opts.EnableProfile, EnableOverrideDescendingIndex: opts.EnableOverrideDescendingIndex, CollStatsLimit: opts.CollStatsLimit, CollectAll: opts.CollectAll, + ProfileTimeTS: opts.ProfileTimeTS, } e := exporter.New(exporterOpts) From 3f8e386c9c7a466eda0f02056208c9816e1fc40c Mon Sep 17 00:00:00 2001 From: bychkov Date: Fri, 15 Sep 2023 09:25:27 +0300 Subject: [PATCH 2/3] add description --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 00b09a6bf..5caf9ff57 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,16 @@ HELP mongodb_mongod_wiredtiger_log_bytes_total mongodb_mongod_wiredtiger_log_byt # TYPE mongodb_mongod_wiredtiger_log_bytes_total untyped mongodb_mongod_wiredtiger_log_bytes_total{type="unwritten"} 2.6208e+06 ``` +#### Enabling profile metrics gathering +`--collector.profile` +To collect metrics, you need to enable the profiler in [MongoDB](https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/): +Usage example: `db.setProfilingLevel(2)` + +|Level|Description| +|-----|-----------| +|0| The profiler is off and does not collect any data. This is the default profiler level.| +|1| The profiler collects data for operations that take longer than the value of `slowms` or that match a filter.
When a filter is set:
  • The `slowms` and `sampleRate` options are not used for profiling.
  • The profiler only captures operations that match the filter.
+|2|The profiler collects data for all operations.| #### Cluster role labels The exporter sets some topology labels in all metrics. From a53c0d8ef5f20b37bfdc622a9d3c38f54d7c071d Mon Sep 17 00:00:00 2001 From: Alex Tymchuk Date: Fri, 15 Sep 2023 13:15:29 +0300 Subject: [PATCH 3/3] fix: tab instead of spaces --- exporter/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index 374dd7137..4787abe7a 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -221,7 +221,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol coc := newCurrentopCollector(ctx, client, e.opts.Logger, e.opts.CompatibleMode, topologyInfo) registry.MustRegister(coc) - } + } if e.opts.EnableProfile && nodeType != typeMongos && limitsOk && requestOpts.EnableProfile && e.opts.ProfileTimeTS != 0 { pc := newProfileCollector(ctx, client, e.opts.Logger,