Export currentOP uptime query metrics #704

REFERENCE.md

@@ -19,6 +19,7 @@
 |--collector.replicasetstatus|Enable collecting metrics from replSetGetStatus|
 |--collector.dbstats|Enable collecting metrics from dbStats||
 |--collector.topmetrics|Enable collecting metrics from top admin command|
+|--collector.currentopmetrics|Enable collecting metrics from currentop admin command|
 |--collector.indexstats|Enable collecting metrics from $indexStats|
 |--collector.collstats|Enable collecting metrics from $collStats|
 |--collect-all|Enable all collectors. Same as specifying all --collector.\<name\>|

exporter/currentop_collector.go

@@ -0,0 +1,144 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type currentopCollector struct {
+	ctx            context.Context
+	base           *baseCollector
+	compatibleMode bool
+	topologyInfo   labelsGetter
+}
+
+var ErrInvalidOrMissingInprogEntry = fmt.Errorf("invalid or misssing inprog entry in currentop results")
+
+// newCurrentopCollector creates a collector for being processed queries.
+func newCurrentopCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger,
+	compatible bool, topology labelsGetter,
+) *currentopCollector {
+	return &currentopCollector{
+		ctx:            ctx,
+		base:           newBaseCollector(client, logger),
+		compatibleMode: compatible,
+		topologyInfo:   topology,
+	}
+}
+
+func (d *currentopCollector) Describe(ch chan<- *prometheus.Desc) {
+	d.base.Describe(d.ctx, ch, d.collect)
+}
+
+func (d *currentopCollector) Collect(ch chan<- prometheus.Metric) {
+	d.base.Collect(ch)
+}
+
+func (d *currentopCollector) collect(ch chan<- prometheus.Metric) {
+	defer measureCollectTime(ch, "mongodb", "currentop")()
+
+	logger := d.base.logger
+	client := d.base.client
+
+	// Get all requests that are being processed except system requests (admin and local).
+	cmd := bson.D{
+		{Key: "currentOp", Value: true},
+		{Key: "active", Value: true},
+		{Key: "microsecs_running", Value: bson.D{
+			{Key: "$exists", Value: true},
+		}},
+		{Key: "op", Value: bson.D{{Key: "$ne", Value: ""}}},
+		{Key: "ns", Value: bson.D{
+			{Key: "$ne", Value: ""},
+			{Key: "$not", Value: bson.D{{Key: "$regex", Value: "^admin.*|^local.*"}}},
+		}},
+	}
+	res := client.Database("admin").RunCommand(d.ctx, cmd)
+
+	var r primitive.M
+	if err := res.Decode(&r); err != nil {
+		ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
+		return
+	}
+
+	logger.Debug("currentop response from MongoDB:")
+	debugResult(logger, r)
+
+	inprog, ok := r["inprog"].(primitive.A)
+
+	if !ok {
+		ch <- prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(ErrInvalidOrMissingInprogEntry),
+			ErrInvalidOrMissingInprogEntry)
+	}
+
+	for _, bsonMap := range inprog {
+
+		bsonMapElement, ok := bsonMap.(primitive.M)
+		if !ok {
+			logger.Errorf("Invalid type primitive.M assertion for bsonMap: %t", ok)
+			break
+		}
+		opid, ok := bsonMapElement["opid"].(int32)
+		if !ok {
+			logger.Errorf("Invalid type int32 assertion for 'opid': %t", ok)
+			break
+		}
+		namespace, ok := bsonMapElement["ns"].(string)
+		if !ok {
+			logger.Errorf("Invalid type string assertion for 'ns': %t", ok)
+			break
+		}
+		db, collection := splitNamespace(namespace)
+		op, ok := bsonMapElement["op"].(string)
+		if !ok {
+			logger.Errorf("Invalid type string assertion for 'op': %t", ok)
+			break
+		}
+		decs, ok := bsonMapElement["desc"].(string)
+		if !ok {
+			logger.Errorf("Invalid type string assertion for 'desc': %t", ok)
+			break
+		}
+		microsecs_running, ok := bsonMapElement["microsecs_running"].(int64)
+		if !ok {
+			logger.Errorf("Invalid type int64 assertion for 'microsecs_running': %t", ok)
+			break
+		}
+
+		labels := d.topologyInfo.baseLabels()
+		labels["opid"] = strconv.Itoa(int(opid))
+		labels["op"] = op
+		labels["decs"] = decs
+		labels["database"] = db
+		labels["collection"] = collection
+		labels["ns"] = namespace
+
+		m := primitive.M{"uptime": microsecs_running}
+
+		for _, metric := range makeMetrics("currentop_query", m, labels, d.compatibleMode) {
+			ch <- metric
+		}
+	}
+}

exporter/currentop_collector_test.go

@@ -0,0 +1,77 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/percona/mongodb_exporter/internal/tu"
+)
+
+func TestCurrentopCollector(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	var wg sync.WaitGroup
+
+	client := tu.DefaultTestClient(ctx, t)
+
+	database := client.Database("testdb")
+	database.Drop(ctx)
+
+	defer func() {
+		err := database.Drop(ctx)
+		assert.NoError(t, err)
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 3; i++ {
+			coll := fmt.Sprintf("testcol_%02d", i)
+			_, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": 1, "f2": "2"})
+			assert.NoError(t, err)
+		}
+	}()
+
+	ti := labelsGetterMock{}
+
+	c := newCurrentopCollector(ctx, client, logrus.New(), false, ti)
+
+	// Filter metrics by reason:
+	// 1. The result will be different on different hardware
+	// 2. Can't check labels like 'decs' and 'opid' because they don't return a known value for comparison
+	// It looks like:
+	// # HELP mongodb_currentop_query_uptime currentop_query.
+	// # TYPE mongodb_currentop_query_uptime untyped
+	// mongodb_currentop_query_uptime{collection="testcol_00",database="testdb",decs="conn6365",ns="testdb.testcol_00",op="insert",opid="448307"} 2524
+
+	filter := []string{
+		"mongodb_currentop_query_uptime",
+	}
+
+	count := testutil.CollectAndCount(c, filter...)
+	assert.True(t, count > 0)
+	wg.Wait()
+}

exporter/exporter.go

@@ -65,6 +65,7 @@ type Opts struct {
 	EnableDBStatsFreeStorage bool
 	EnableDiagnosticData     bool
 	EnableReplicasetStatus   bool
+	EnableCurrentopMetrics   bool
 	EnableTopMetrics         bool
 	EnableIndexStats         bool
 	EnableCollStats          bool
@@ -170,6 +171,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		e.opts.EnableTopMetrics = true
 		e.opts.EnableReplicasetStatus = true
 		e.opts.EnableIndexStats = true
+		e.opts.EnableCurrentopMetrics = true
 	}
 
 	// arbiter only have isMaster privileges
@@ -180,6 +182,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		e.opts.EnableTopMetrics = false
 		e.opts.EnableReplicasetStatus = false
 		e.opts.EnableIndexStats = false
+		e.opts.EnableCurrentopMetrics = false
 	}
 
 	// If we manually set the collection names we want or auto discovery is set.
@@ -210,6 +213,12 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		registry.MustRegister(cc)
 	}
 
+	if e.opts.EnableCurrentopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableCurrentopMetrics {
+		coc := newCurrentopCollector(ctx, client, e.opts.Logger,
+			e.opts.CompatibleMode, topologyInfo)
+		registry.MustRegister(coc)
+	}
+
 	if e.opts.EnableTopMetrics && nodeType != typeMongos && limitsOk && requestOpts.EnableTopMetrics {
 		tc := newTopCollector(ctx, client, e.opts.Logger,
 			e.opts.CompatibleMode, topologyInfo)
@@ -288,6 +297,8 @@ func (e *Exporter) Handler() http.Handler {
 				requestOpts.EnableDBStats = true
 			case "topmetrics":
 				requestOpts.EnableTopMetrics = true
+			case "currentopmetrics":
+				requestOpts.EnableCurrentopMetrics = true
 			case "indexstats":
 				requestOpts.EnableIndexStats = true
 			case "collstats":

main.go

@@ -49,6 +49,7 @@ type GlobalFlags struct {
 	EnableDBStats            bool `name:"collector.dbstats" help:"Enable collecting metrics from dbStats"`
 	EnableDBStatsFreeStorage bool `name:"collector.dbstatsfreestorage" help:"Enable collecting free space metrics from dbStats"`
 	EnableTopMetrics         bool `name:"collector.topmetrics" help:"Enable collecting metrics from top admin command"`
+	EnableCurrentopMetrics   bool `name:"collector.currentopmetrics" help:"Enable collecting metrics currentop admin command"`
 	EnableIndexStats         bool `name:"collector.indexstats" help:"Enable collecting metrics from $indexStats"`
 	EnableCollStats          bool `name:"collector.collstats" help:"Enable collecting metrics from $collStats"`
 
@@ -125,6 +126,7 @@ func buildExporter(opts GlobalFlags) *exporter.Exporter {
 
 		EnableDiagnosticData:     opts.EnableDiagnosticData,
 		EnableReplicasetStatus:   opts.EnableReplicasetStatus,
+		EnableCurrentopMetrics:   opts.EnableCurrentopMetrics,
 		EnableTopMetrics:         opts.EnableTopMetrics,
 		EnableDBStats:            opts.EnableDBStats,
 		EnableDBStatsFreeStorage: opts.EnableDBStatsFreeStorage,