From 9571d21871f3223ea110e4228d3a7281b58e5ac6 Mon Sep 17 00:00:00 2001 From: ngcat Date: Thu, 14 Sep 2023 07:30:18 +0800 Subject: [PATCH 1/5] add user and password to uri if the uri does not contain user password (#560) * add user and password to uri if the uri does not contain user password * improve compatibility * extract whole `buildURI` logic and write a test * add description for MONGODB_USER and MONGODB_PASSWORD in README.md * fix for lint check * make duplications test code simplified * change properties order to make read easier * add back mongodb:// append logic * clean duplicate mongodb:// append logic * try to fix lint * move third party dependency below to internal dependency --- README.md | 15 +++++++++-- main.go | 22 ++++++++++++--- main_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 00b09a6bf..5cacdc90a 100644 --- a/README.md +++ b/README.md @@ -71,14 +71,25 @@ Connecting user should have sufficient rights to query needed stats: More info about roles in MongoDB [documentation](https://docs.mongodb.com/manual/reference/built-in-roles/#mongodb-authrole-clusterMonitor). #### Example -``` +```sh mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 ``` +#### MongoDB Authentication +You can supply the mongodb user/password direct in the `--mongodb.uri=` like `--mongodb.uri=mongodb://user:pass@127.0.0.1:17001`, you can also supply the mongodb user/password with `--mongodb.user=`, `--mongodb.password=` +but the user and password info will be leaked via `ps` or `top` command, for security issue, you can use `MONGODB_USER` and `MONGODB_PASSWORD` env variable to set user/password for given uri +```sh +MONGODB_USER=XXX MONGODB_PASSWORD=YYY mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2 +# or +export MONGODB_USER=XXX +export MONGODB_PASSWORD=YYY +mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2 +``` + #### Enabling collstats metrics gathering `--mongodb.collstats-colls` receives a list of databases and collections to monitor using collstats. Usage example: `--mongodb.collstats-colls=database1.collection1,database2.collection2` -``` +```sh mongodb_exporter_linux_amd64/mongodb_exporter --mongodb.uri=mongodb://127.0.0.1:17001 --mongodb.collstats-colls=db1.c1,db2.c2 ``` #### Enabling compatibility mode. diff --git a/main.go b/main.go index 839031a80..8eb751898 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,8 @@ var ( // GlobalFlags has command line flags to configure the exporter. type GlobalFlags struct { + User string `name:"mongodb.user" help:"monitor user, need clusterMonitor role in admin db and read role in local db" env:"MONGODB_USER" placeholder:"monitorUser"` + Password string `name:"mongodb.password" help:"monitor user password" env:"MONGODB_PASSWORD" placeholder:"monitorPassword"` CollStatsNamespaces string `name:"mongodb.collstats-colls" help:"List of comma separared databases.collections to get $collStats" placeholder:"db1,db2.col2"` IndexStatsCollections string `name:"mongodb.indexstats-colls" help:"List of comma separared databases.collections to get $indexStats" placeholder:"db1.col1,db2.col2"` URI string `name:"mongodb.uri" help:"MongoDB connection URI" env:"MONGODB_URI" placeholder:"mongodb://user:pass@127.0.0.1:27017/admin?ssl=true"` @@ -89,6 +91,21 @@ func main() { e.Run() } +func buildURI(uri string, user string, password string) string { + // IF user@pass not contained in uri AND custom user and pass supplied in arguments + // DO concat a new uri with user and pass arguments value + if !strings.Contains(uri, "@") && user != "" && password != "" { + // trim mongodb:// prefix to handle user and pass logic + uri = strings.TrimPrefix(uri, "mongodb://") + // add user and pass to the uri + uri = fmt.Sprintf("%s:%s@%s", user, password, uri) + } + if !strings.HasPrefix(uri, "mongodb") { + uri = "mongodb://" + uri + } + return uri +} + func buildExporter(opts GlobalFlags) *exporter.Exporter { log := logrus.New() @@ -103,10 +120,7 @@ func buildExporter(opts GlobalFlags) *exporter.Exporter { log.Debugf("Compatible mode: %v", opts.CompatibleMode) - if !strings.HasPrefix(opts.URI, "mongodb") { - log.Debugf("Prepending mongodb:// to the URI") - opts.URI = "mongodb://" + opts.URI - } + opts.URI = buildURI(opts.URI, opts.User, opts.Password) log.Debugf("Connection URI: %s", opts.URI) diff --git a/main_test.go b/main_test.go index 886eb6d0f..bf37e38a0 100644 --- a/main_test.go +++ b/main_test.go @@ -17,6 +17,8 @@ package main import ( "testing" + + "github.com/stretchr/testify/assert" ) func TestBuildExporter(t *testing.T) { @@ -37,3 +39,77 @@ func TestBuildExporter(t *testing.T) { buildExporter(opts) } + +func TestBuildURI(t *testing.T) { + tests := []struct { + situation string + origin string + newUser string + newPassword string + expect string + }{ + { + situation: "uri with prefix and auth, and auth supplied in opt.User/Password", + origin: "mongodb://usr:pwd@127.0.0.1", + newUser: "xxx", + newPassword: "yyy", + expect: "mongodb://usr:pwd@127.0.0.1", + }, + { + situation: "uri with prefix and auth, no auth supplied in opt.User/Password", + origin: "mongodb://usr:pwd@127.0.0.1", + newUser: "", + newPassword: "", + expect: "mongodb://usr:pwd@127.0.0.1", + }, + { + situation: "uri with no prefix and auth, and auth supplied in opt.User/Password", + origin: "usr:pwd@127.0.0.1", + newUser: "xxx", + newPassword: "yyy", + expect: "mongodb://usr:pwd@127.0.0.1", + }, + { + situation: "uri with no prefix and auth, no auth supplied in opt.User/Password", + origin: "usr:pwd@127.0.0.1", + newUser: "", + newPassword: "", + expect: "mongodb://usr:pwd@127.0.0.1", + }, + { + situation: "uri with prefix and no auth, and auth supplied in opt.User/Password", + origin: "mongodb://127.0.0.1", + newUser: "xxx", + newPassword: "yyy", + expect: "mongodb://xxx:yyy@127.0.0.1", + }, + { + situation: "uri with prefix and no auth, no auth supplied in opt.User/Password", + origin: "mongodb://127.0.0.1", + newUser: "", + newPassword: "", + expect: "mongodb://127.0.0.1", + }, + { + situation: "uri with no prefix and no auth, and auth supplied in opt.User/Password", + origin: "127.0.0.1", + newUser: "xxx", + newPassword: "yyy", + expect: "mongodb://xxx:yyy@127.0.0.1", + }, + { + situation: "uri with no prefix and no auth, no auth supplied in opt.User/Password", + origin: "127.0.0.1", + newUser: "", + newPassword: "", + expect: "mongodb://127.0.0.1", + }, + } + for _, tc := range tests { + newUri := buildURI(tc.origin, tc.newUser, tc.newPassword) + // t.Logf("Origin: %s", tc.origin) + // t.Logf("Expect: %s", tc.expect) + // t.Logf("Result: %s", newUri) + assert.Equal(t, newUri, tc.expect) + } +} From bedbd210428d89b9d687800f38d23bce18c42d2d Mon Sep 17 00:00:00 2001 From: Andrey Tregubov <55836868+tregubov-av@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:52:20 +0300 Subject: [PATCH 2/5] Export currentOP uptime query metrics (#706) * Export currentOP uptime query metrics #704 * Update exporter/currentop_collector.go Co-authored-by: Artem Gavrilov * Export currentOP uptime query metrics --------- Co-authored-by: Artem Gavrilov --- REFERENCE.md | 1 + exporter/currentop_collector.go | 144 +++++++++++++++++++++++++++ exporter/currentop_collector_test.go | 77 ++++++++++++++ exporter/exporter.go | 11 ++ main.go | 2 + 5 files changed, 235 insertions(+) create mode 100644 exporter/currentop_collector.go create mode 100644 exporter/currentop_collector_test.go diff --git a/REFERENCE.md b/REFERENCE.md index bf36356e7..b02e4711c 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -19,6 +19,7 @@ |--collector.replicasetstatus|Enable collecting metrics from replSetGetStatus| |--collector.dbstats|Enable collecting metrics from dbStats|| |--collector.topmetrics|Enable collecting metrics from top admin command| +|--collector.currentopmetrics|Enable collecting metrics from currentop admin command| |--collector.indexstats|Enable collecting metrics from $indexStats| |--collector.collstats|Enable collecting metrics from $collStats| |--collect-all|Enable all collectors. Same as specifying all --collector.\| diff --git a/exporter/currentop_collector.go b/exporter/currentop_collector.go new file mode 100644 index 000000000..287981d30 --- /dev/null +++ b/exporter/currentop_collector.go @@ -0,0 +1,144 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + "errors" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type currentopCollector struct { + ctx context.Context + base *baseCollector + compatibleMode bool + topologyInfo labelsGetter +} + +var ErrInvalidOrMissingInprogEntry = errors.New("invalid or missing inprog entry in currentop results") + +// newCurrentopCollector creates a collector for being processed queries. +func newCurrentopCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, + compatible bool, topology labelsGetter, +) *currentopCollector { + return ¤topCollector{ + ctx: ctx, + base: newBaseCollector(client, logger), + compatibleMode: compatible, + topologyInfo: topology, + } +} + +func (d *currentopCollector) Describe(ch chan<- *prometheus.Desc) { + d.base.Describe(d.ctx, ch, d.collect) +} + +func (d *currentopCollector) Collect(ch chan<- prometheus.Metric) { + d.base.Collect(ch) +} + +func (d *currentopCollector) collect(ch chan<- prometheus.Metric) { + defer measureCollectTime(ch, "mongodb", "currentop")() + + logger := d.base.logger + client := d.base.client + + // Get all requests that are being processed except system requests (admin and local). + cmd := bson.D{ + {Key: "currentOp", Value: true}, + {Key: "active", Value: true}, + {Key: "microsecs_running", Value: bson.D{ + {Key: "$exists", Value: true}, + }}, + {Key: "op", Value: bson.D{{Key: "$ne", Value: ""}}}, + {Key: "ns", Value: bson.D{ + {Key: "$ne", Value: ""}, + {Key: "$not", Value: bson.D{{Key: "$regex", Value: "^admin.*|^local.*"}}}, + }}, + } + res := client.Database("admin").RunCommand(d.ctx, cmd) + + var r primitive.M + if err := res.Decode(&r); err != nil { + ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err) + return + } + + logger.Debug("currentop response from MongoDB:") + debugResult(logger, r) + + inprog, ok := r["inprog"].(primitive.A) + + if !ok { + ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(ErrInvalidOrMissingInprogEntry), + ErrInvalidOrMissingInprogEntry) + } + + for _, bsonMap := range inprog { + + bsonMapElement, ok := bsonMap.(primitive.M) + if !ok { + logger.Errorf("Invalid type primitive.M assertion for bsonMap: %T", bsonMapElement) + continue + } + opid, ok := bsonMapElement["opid"].(int32) + if !ok { + logger.Errorf("Invalid type int32 assertion for 'opid': %T", bsonMapElement) + continue + } + namespace, ok := bsonMapElement["ns"].(string) + if !ok { + logger.Errorf("Invalid type string assertion for 'ns': %T", bsonMapElement) + continue + } + db, collection := splitNamespace(namespace) + op, ok := bsonMapElement["op"].(string) + if !ok { + logger.Errorf("Invalid type string assertion for 'op': %T", bsonMapElement) + continue + } + desc, ok := bsonMapElement["desc"].(string) + if !ok { + logger.Errorf("Invalid type string assertion for 'desc': %T", bsonMapElement) + continue + } + microsecs_running, ok := bsonMapElement["microsecs_running"].(int64) + if !ok { + logger.Errorf("Invalid type int64 assertion for 'microsecs_running': %T", bsonMapElement) + continue + } + + labels := d.topologyInfo.baseLabels() + labels["opid"] = strconv.Itoa(int(opid)) + labels["op"] = op + labels["desc"] = desc + labels["database"] = db + labels["collection"] = collection + labels["ns"] = namespace + + m := primitive.M{"uptime": microsecs_running} + + for _, metric := range makeMetrics("currentop_query", m, labels, d.compatibleMode) { + ch <- metric + } + } +} diff --git a/exporter/currentop_collector_test.go b/exporter/currentop_collector_test.go new file mode 100644 index 000000000..5e9d07917 --- /dev/null +++ b/exporter/currentop_collector_test.go @@ -0,0 +1,77 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + + "github.com/percona/mongodb_exporter/internal/tu" +) + +func TestCurrentopCollector(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + var wg sync.WaitGroup + + client := tu.DefaultTestClient(ctx, t) + + database := client.Database("testdb") + database.Drop(ctx) + + defer func() { + err := database.Drop(ctx) + assert.NoError(t, err) + }() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 3; i++ { + coll := fmt.Sprintf("testcol_%02d", i) + _, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": 1, "f2": "2"}) + assert.NoError(t, err) + } + }() + + ti := labelsGetterMock{} + + c := newCurrentopCollector(ctx, client, logrus.New(), false, ti) + + // Filter metrics by reason: + // 1. The result will be different on different hardware + // 2. Can't check labels like 'decs' and 'opid' because they don't return a known value for comparison + // It looks like: + // # HELP mongodb_currentop_query_uptime currentop_query. + // # TYPE mongodb_currentop_query_uptime untyped + // mongodb_currentop_query_uptime{collection="testcol_00",database="testdb",decs="conn6365",ns="testdb.testcol_00",op="insert",opid="448307"} 2524 + + filter := []string{ + "mongodb_currentop_query_uptime", + } + + count := testutil.CollectAndCount(c, filter...) + assert.True(t, count > 0) + wg.Wait() +} diff --git a/exporter/exporter.go b/exporter/exporter.go index 65af03158..0bc9781ac 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -65,6 +65,7 @@ type Opts struct { EnableDBStatsFreeStorage bool EnableDiagnosticData bool EnableReplicasetStatus bool + EnableCurrentopMetrics bool EnableTopMetrics bool EnableIndexStats bool EnableCollStats bool @@ -170,6 +171,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.opts.EnableTopMetrics = true e.opts.EnableReplicasetStatus = true e.opts.EnableIndexStats = true + e.opts.EnableCurrentopMetrics = true } // arbiter only have isMaster privileges @@ -180,6 +182,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.opts.EnableTopMetrics = false e.opts.EnableReplicasetStatus = false e.opts.EnableIndexStats = false + e.opts.EnableCurrentopMetrics = false } // If we manually set the collection names we want or auto discovery is set. @@ -210,6 +213,12 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol registry.MustRegister(cc) } + if e.opts.EnableCurrentopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableCurrentopMetrics { + coc := newCurrentopCollector(ctx, client, e.opts.Logger, + e.opts.CompatibleMode, topologyInfo) + registry.MustRegister(coc) + } + if e.opts.EnableTopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableTopMetrics { tc := newTopCollector(ctx, client, e.opts.Logger, e.opts.CompatibleMode, topologyInfo) @@ -288,6 +297,8 @@ func (e *Exporter) Handler() http.Handler { requestOpts.EnableDBStats = true case "topmetrics": requestOpts.EnableTopMetrics = true + case "currentopmetrics": + requestOpts.EnableCurrentopMetrics = true case "indexstats": requestOpts.EnableIndexStats = true case "collstats": diff --git a/main.go b/main.go index 8eb751898..229e4a33a 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,7 @@ type GlobalFlags struct { EnableDBStats bool `name:"collector.dbstats" help:"Enable collecting metrics from dbStats"` EnableDBStatsFreeStorage bool `name:"collector.dbstatsfreestorage" help:"Enable collecting free space metrics from dbStats"` EnableTopMetrics bool `name:"collector.topmetrics" help:"Enable collecting metrics from top admin command"` + EnableCurrentopMetrics bool `name:"collector.currentopmetrics" help:"Enable collecting metrics currentop admin command"` EnableIndexStats bool `name:"collector.indexstats" help:"Enable collecting metrics from $indexStats"` EnableCollStats bool `name:"collector.collstats" help:"Enable collecting metrics from $collStats"` @@ -139,6 +140,7 @@ func buildExporter(opts GlobalFlags) *exporter.Exporter { EnableDiagnosticData: opts.EnableDiagnosticData, EnableReplicasetStatus: opts.EnableReplicasetStatus, + EnableCurrentopMetrics: opts.EnableCurrentopMetrics, EnableTopMetrics: opts.EnableTopMetrics, EnableDBStats: opts.EnableDBStats, EnableDBStatsFreeStorage: opts.EnableDBStatsFreeStorage, From 6b2e831e87b9b0107d2e74ff93af4e77c3f0efaf Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Fri, 15 Sep 2023 12:01:13 +0200 Subject: [PATCH 3/5] PMM-7 Use pkg/errors (#711) --- exporter/currentop_collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/currentop_collector.go b/exporter/currentop_collector.go index 287981d30..392f825ab 100644 --- a/exporter/currentop_collector.go +++ b/exporter/currentop_collector.go @@ -17,9 +17,9 @@ package exporter import ( "context" - "errors" "strconv" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "go.mongodb.org/mongo-driver/bson" From fcb3850a0b328b2c41d0749448b2e14f03b39907 Mon Sep 17 00:00:00 2001 From: mikle7771 <78665250+mikle7771@users.noreply.github.com> Date: Fri, 15 Sep 2023 17:07:59 +0300 Subject: [PATCH 4/5] Collector for working with data from the system profile (#710) * Collector for working with data from the system profile * add description * fix: tab instead of spaces --------- Co-authored-by: Alex Tymchuk --- README.md | 10 +++ REFERENCE.md | 2 + exporter/exporter.go | 12 +++ exporter/profile_status_collector.go | 96 +++++++++++++++++++++++ exporter/profile_status_collector_test.go | 69 ++++++++++++++++ main.go | 5 ++ 6 files changed, 194 insertions(+) create mode 100644 exporter/profile_status_collector.go create mode 100644 exporter/profile_status_collector_test.go diff --git a/README.md b/README.md index 5cacdc90a..f20cefb7a 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,16 @@ HELP mongodb_mongod_wiredtiger_log_bytes_total mongodb_mongod_wiredtiger_log_byt # TYPE mongodb_mongod_wiredtiger_log_bytes_total untyped mongodb_mongod_wiredtiger_log_bytes_total{type="unwritten"} 2.6208e+06 ``` +#### Enabling profile metrics gathering +`--collector.profile` +To collect metrics, you need to enable the profiler in [MongoDB](https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/): +Usage example: `db.setProfilingLevel(2)` + +|Level|Description| +|-----|-----------| +|0| The profiler is off and does not collect any data. This is the default profiler level.| +|1| The profiler collects data for operations that take longer than the value of `slowms` or that match a filter.
When a filter is set:
  • The `slowms` and `sampleRate` options are not used for profiling.
  • The profiler only captures operations that match the filter.
+|2|The profiler collects data for all operations.| #### Cluster role labels The exporter sets some topology labels in all metrics. diff --git a/REFERENCE.md b/REFERENCE.md index b02e4711c..a78f7edc7 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -24,5 +24,7 @@ |--collector.collstats|Enable collecting metrics from $collStats| |--collect-all|Enable all collectors. Same as specifying all --collector.\| |--collector.collstats-limit=0|Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \ collections. 0=No limit| +|--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval| +|--collector.profile|Enable collecting metrics from profile| |--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC || |--version|Show version and exit| \ No newline at end of file diff --git a/exporter/exporter.go b/exporter/exporter.go index 0bc9781ac..4787abe7a 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -59,6 +59,7 @@ type Opts struct { DisableDefaultRegistry bool DiscoveringMode bool GlobalConnPool bool + ProfileTimeTS int CollectAll bool EnableDBStats bool @@ -69,6 +70,7 @@ type Opts struct { EnableTopMetrics bool EnableIndexStats bool EnableCollStats bool + EnableProfile bool EnableOverrideDescendingIndex bool @@ -172,6 +174,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.opts.EnableReplicasetStatus = true e.opts.EnableIndexStats = true e.opts.EnableCurrentopMetrics = true + e.opts.EnableProfile = true } // arbiter only have isMaster privileges @@ -183,6 +186,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.opts.EnableReplicasetStatus = false e.opts.EnableIndexStats = false e.opts.EnableCurrentopMetrics = false + e.opts.EnableProfile = false } // If we manually set the collection names we want or auto discovery is set. @@ -219,6 +223,12 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol registry.MustRegister(coc) } + if e.opts.EnableProfile && nodeType != typeMongos && limitsOk && requestOpts.EnableProfile && e.opts.ProfileTimeTS != 0 { + pc := newProfileCollector(ctx, client, e.opts.Logger, + e.opts.CompatibleMode, topologyInfo, e.opts.ProfileTimeTS) + registry.MustRegister(pc) + } + if e.opts.EnableTopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableTopMetrics { tc := newTopCollector(ctx, client, e.opts.Logger, e.opts.CompatibleMode, topologyInfo) @@ -303,6 +313,8 @@ func (e *Exporter) Handler() http.Handler { requestOpts.EnableIndexStats = true case "collstats": requestOpts.EnableCollStats = true + case "profile": + requestOpts.EnableProfile = true } } diff --git a/exporter/profile_status_collector.go b/exporter/profile_status_collector.go new file mode 100644 index 000000000..910cd1559 --- /dev/null +++ b/exporter/profile_status_collector.go @@ -0,0 +1,96 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + "time" + + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type profileCollector struct { + ctx context.Context + base *baseCollector + compatibleMode bool + topologyInfo labelsGetter + profiletimets int +} + +// newProfileCollector creates a collector for being processed queries. +func newProfileCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, + compatible bool, topology labelsGetter, profileTimeTS int, +) *profileCollector { + return &profileCollector{ + ctx: ctx, + base: newBaseCollector(client, logger), + compatibleMode: compatible, + topologyInfo: topology, + profiletimets: profileTimeTS, + } +} + +func (d *profileCollector) Describe(ch chan<- *prometheus.Desc) { + d.base.Describe(d.ctx, ch, d.collect) +} + +func (d *profileCollector) Collect(ch chan<- prometheus.Metric) { + d.base.Collect(ch) +} + +func (d *profileCollector) collect(ch chan<- prometheus.Metric) { + defer measureCollectTime(ch, "mongodb", "profile")() + + logger := d.base.logger + client := d.base.client + timeScrape := d.profiletimets + + databases, err := databases(d.ctx, client, nil, nil) + if err != nil { + errors.Wrap(err, "cannot get the database names list") + return + } + + // Now time + '--collector.profile-time-ts' + ts := primitive.NewDateTimeFromTime(time.Now().Add(-time.Duration(time.Second * time.Duration(timeScrape)))) + + labels := d.topologyInfo.baseLabels() + + // Get all slow queries from all databases + cmd := bson.M{"ts": bson.M{"$gte": ts}} + for _, db := range databases { + res, err := client.Database(db).Collection("system.profile").CountDocuments(d.ctx, cmd) + if err != nil { + errors.Wrapf(err, "cannot read system.profile") + break + } + labels["database"] = db + + m := primitive.M{"count": res} + + logger.Debug("profile response from MongoDB:") + debugResult(logger, primitive.M{db: m}) + + for _, metric := range makeMetrics("profile_slow_query", m, labels, d.compatibleMode) { + ch <- metric + } + } +} diff --git a/exporter/profile_status_collector_test.go b/exporter/profile_status_collector_test.go new file mode 100644 index 000000000..df47d5a34 --- /dev/null +++ b/exporter/profile_status_collector_test.go @@ -0,0 +1,69 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + + "github.com/percona/mongodb_exporter/internal/tu" +) + +func TestProfileCollector(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + client := tu.DefaultTestClient(ctx, t) + + database := client.Database("testdb") + database.Drop(ctx) //nolint + + defer func() { + err := database.Drop(ctx) + assert.NoError(t, err) + }() + + // Enable database profiler https://www.mongodb.com/docs/manual/tutorial/manage-the-database-profiler/ + cmd := bson.M{"profile": 2} + _ = database.RunCommand(ctx, cmd) + + ti := labelsGetterMock{} + + c := newProfileCollector(ctx, client, logrus.New(), false, ti, 30) + + expected := strings.NewReader(` + # HELP mongodb_profile_slow_query_count profile_slow_query. + # TYPE mongodb_profile_slow_query_count counter + mongodb_profile_slow_query_count{database="admin"} 0 + mongodb_profile_slow_query_count{database="config"} 0 + mongodb_profile_slow_query_count{database="local"} 0 + mongodb_profile_slow_query_count{database="testdb"} 0` + + "\n") + + filter := []string{ + "mongodb_profile_slow_query_count", + } + + err := testutil.CollectAndCompare(c, expected, filter...) + assert.NoError(t, err) +} diff --git a/main.go b/main.go index 229e4a33a..1e5d13561 100644 --- a/main.go +++ b/main.go @@ -54,6 +54,7 @@ type GlobalFlags struct { EnableCurrentopMetrics bool `name:"collector.currentopmetrics" help:"Enable collecting metrics currentop admin command"` EnableIndexStats bool `name:"collector.indexstats" help:"Enable collecting metrics from $indexStats"` EnableCollStats bool `name:"collector.collstats" help:"Enable collecting metrics from $collStats"` + EnableProfile bool `name:"collector.profile" help:"Enable collecting metrics from profile"` EnableOverrideDescendingIndex bool `name:"metrics.overridedescendingindex" help:"Enable descending index name override to replace -1 with _DESC"` @@ -61,6 +62,8 @@ type GlobalFlags struct { CollStatsLimit int `name:"collector.collstats-limit" help:"Disable collstats, dbstats, topmetrics and indexstats collector if there are more than collections. 0=No limit" default:"0"` + ProfileTimeTS int `name:"collector.profile-time-ts" help:"Set time for scrape slow queries." default:"30"` + DiscoveringMode bool `name:"discovering-mode" help:"Enable autodiscover collections" negatable:""` CompatibleMode bool `name:"compatible-mode" help:"Enable old mongodb-exporter compatible metrics" negatable:""` Version bool `name:"version" help:"Show version and exit"` @@ -146,11 +149,13 @@ func buildExporter(opts GlobalFlags) *exporter.Exporter { EnableDBStatsFreeStorage: opts.EnableDBStatsFreeStorage, EnableIndexStats: opts.EnableIndexStats, EnableCollStats: opts.EnableCollStats, + EnableProfile: opts.EnableProfile, EnableOverrideDescendingIndex: opts.EnableOverrideDescendingIndex, CollStatsLimit: opts.CollStatsLimit, CollectAll: opts.CollectAll, + ProfileTimeTS: opts.ProfileTimeTS, } e := exporter.New(exporterOpts) From 4cbba0ff93435c80b3f303e31feccb9dfac26ce9 Mon Sep 17 00:00:00 2001 From: Trevor North Date: Fri, 29 Sep 2023 14:33:57 +0100 Subject: [PATCH 5/5] Add support for timestamp metrics with seconds precision (#558) * Add support for timestamp metrics with seconds precision This is useful for exporting oplog metrics in order to calculate the oplog window as the difference between mongodb_ss_oplog_latestOptime and mongodb_ss_oplog_earliestOptime * Fix test expectations --- exporter/metrics.go | 4 +++- exporter/metrics_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index 844dab611..e1040d7d7 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -208,7 +208,9 @@ func asFloat64(value interface{}) (*float64, error) { f = v case primitive.DateTime: f = float64(v) - case primitive.A, primitive.ObjectID, primitive.Timestamp, primitive.Binary, string, []uint8, time.Time: + case primitive.Timestamp: + f = float64(v.T) + case primitive.A, primitive.ObjectID, primitive.Binary, string, []uint8, time.Time: return nil, nil default: return nil, errors.Wrapf(errCannotHandleType, "%T", v) diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index 77469dbac..da0a00743 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -141,7 +141,7 @@ func TestMakeRawMetric(t *testing.T) { {value: float32(1.23), wantVal: pointer.ToFloat64(float64(float32(1.23)))}, {value: float64(1.23), wantVal: pointer.ToFloat64(1.23)}, {value: primitive.A{}, wantVal: nil}, - {value: primitive.Timestamp{}, wantVal: nil}, + {value: primitive.Timestamp{T: 123, I: 456}, wantVal: pointer.ToFloat64(123)}, {value: "zapp", wantVal: nil}, {value: []byte{}, wantVal: nil}, {value: time.Date(2020, 6, 15, 0, 0, 0, 0, time.UTC), wantVal: nil},