diff --git a/agent/runner/actions/mongodb_explain_action.go b/agent/runner/actions/mongodb_explain_action.go index 373b04df9f1..6f223453928 100644 --- a/agent/runner/actions/mongodb_explain_action.go +++ b/agent/runner/actions/mongodb_explain_action.go @@ -18,9 +18,9 @@ import ( "context" "fmt" "path/filepath" + "strings" "time" - "github.com/percona/percona-toolkit/src/go/mongolib/proto" "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -39,6 +39,15 @@ type mongodbExplainAction struct { dsn string } +type explain struct { + Ns string `json:"ns"` + Op string `json:"op"` + Query bson.D `json:"query,omitempty"` + Command bson.D `json:"command,omitempty"` + OriginatingCommand bson.D `json:"originatingCommand,omitempty"` + UpdateObj bson.D `json:"updateobj,omitempty"` +} + var errCannotExplain = fmt.Errorf("cannot explain this type of query") // NewMongoDBExplainAction creates a MongoDB EXPLAIN query Action. @@ -89,28 +98,165 @@ func (a *mongodbExplainAction) Run(ctx context.Context) ([]byte, error) { } defer client.Disconnect(ctx) //nolint:errcheck - var eq proto.ExampleQuery + return explainForQuery(ctx, client, a.params.Query) +} + +func (a *mongodbExplainAction) sealed() {} + +func (e explain) prepareCommand() (bson.D, error) { + command := e.Command + + switch e.Op { + case "query": + if len(command) == 0 { + command = e.Query + } + + if len(command) == 0 || command[0].Key != "find" { + return bson.D{ + {Key: "find", Value: e.getCollection()}, + {Key: "filter", Value: command}, + }, nil + } + + if len(command) != 0 && command[0].Key == "query" { + return bson.D{ + {Key: "find", Value: e.getCollection()}, + {Key: "filter", Value: command[0].Value}, + }, nil + } + + return dropDBField(command), nil + case "update": + if len(command) == 0 { + command = bson.D{ + {Key: "q", Value: e.Query}, + {Key: "u", Value: e.UpdateObj}, + } + } + + return bson.D{ + {Key: "update", Value: e.getCollection()}, + {Key: "updates", Value: []any{command}}, + }, nil + case "remove": + if len(command) == 0 { + command = bson.D{{Key: "q", Value: e.Query}} + } + + return bson.D{ + {Key: "delete", Value: e.getCollection()}, + {Key: "deletes", Value: []any{command}}, + }, nil + case "getmore": + if len(e.OriginatingCommand) == 0 { + return bson.D{{Key: "getmore", Value: ""}}, nil + } + + command = e.OriginatingCommand + + return dropDBField(command), nil + case "command": + command = dropDBField(command) + + if len(command) == 0 { + return command, nil + } + + switch command[0].Key { + // Not supported commands. + case "dbStats": + return nil, errors.Errorf("command %s is not supported for explain", command[0].Key) + case "group": + default: + return command, nil + } + + return fixReduceField(command), nil + // Not supported operations. + case "insert", "drop": + return nil, errors.Errorf("operation %s is not supported for explain", e.Op) + } + + return command, nil +} + +func (e explain) getDB() string { + s := strings.SplitN(e.Ns, ".", 2) + if len(s) == 2 { + return s[0] + } + + return "" +} + +func (e explain) getCollection() string { + s := strings.SplitN(e.Ns, ".", 2) + if len(s) == 2 { + return s[1] + } + + return "" +} + +// dropDBField remove DB field to be able run explain on all supported types. +// Otherwise it could end up with BSON field 'xxx.$db' is a duplicate field. +func dropDBField(command bson.D) bson.D { + for i := range command { + if command[i].Key != "$db" { + continue + } + + if len(command)-1 == i { + return command[:i] + } + + return append(command[:i], command[i+1:]...) + } + + return command +} + +// fixReduceField fixing nil/empty values after unmarshalling funcs. +func fixReduceField(command bson.D) bson.D { + var group bson.D + var ok bool + if group, ok = command[0].Value.(bson.D); !ok { + return command + } - err = bson.UnmarshalExtJSON([]byte(a.params.Query), true, &eq) + for i := range group { + if group[i].Key == "$reduce" { + group[i].Value = "{}" + command[0].Value = group + break + } + } + + return command +} + +func explainForQuery(ctx context.Context, client *mongo.Client, query string) ([]byte, error) { + var e explain + err := bson.UnmarshalExtJSON([]byte(query), false, &e) if err != nil { - return nil, errors.Wrapf(err, "Query: %s", a.params.Query) + return nil, errors.Wrapf(err, "Query: %s", query) } - database := "admin" - if eq.Db() != "" { - database = eq.Db() + preparedCommand, err := e.prepareCommand() + if err != nil { + return nil, errors.Wrap(errCannotExplain, err.Error()) } - res := client.Database(database).RunCommand(ctx, eq.ExplainCmd()) + command := bson.D{{Key: "explain", Value: preparedCommand}} + res := client.Database(e.getDB()).RunCommand(ctx, command) if res.Err() != nil { return nil, errors.Wrap(errCannotExplain, res.Err().Error()) } - result, err := res.DecodeBytes() + result, err := res.Raw() if err != nil { return nil, err } - // We need it because result + return []byte(result.String()), nil } - -func (a *mongodbExplainAction) sealed() {} diff --git a/agent/runner/actions/mongodb_explain_action_test.go b/agent/runner/actions/mongodb_explain_action_test.go index 8e927b71ec7..41299a58578 100644 --- a/agent/runner/actions/mongodb_explain_action_test.go +++ b/agent/runner/actions/mongodb_explain_action_test.go @@ -16,10 +16,13 @@ package actions import ( "context" + "crypto/rand" "encoding/json" "fmt" + "math/big" "os" "path/filepath" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -33,6 +36,221 @@ import ( "github.com/percona/pmm/version" ) +func TestQueryExplain(t *testing.T) { + database := "testdb" + ctx := context.TODO() + + dsn := tests.GetTestMongoDBDSN(t) + client := tests.OpenTestMongoDB(t, dsn) + t.Cleanup(func() { + defer client.Disconnect(ctx) //nolint:errcheck + defer client.Database(database).Drop(ctx) //nolint:errcheck + }) + + t.Run("Find", func(t *testing.T) { + query := `{ + "ns": "config.collections", + "op": "query", + "command": { + "find": "collections", + "filter": { + "_id": { + "$regex": "^admin.", + "$options": "i" + } + }, + "$db": "config" + } + }` + runExplain(ctx, t, prepareParams(t, query)) + }) + + t.Run("Count", func(t *testing.T) { + query := `{ + "ns": "testdb.collection", + "op": "command", + "command": { + "count": "collection", + "query": { + "a": { + "$numberDouble": "5.0" + }, + "b": { + "$numberDouble": "5.0" + } + }, + "$db": "testdb" + } + }` + runExplain(ctx, t, prepareParams(t, query)) + }) + + t.Run("Count with aggregate", func(t *testing.T) { + query := `{ + "ns": "testdb.collection", + "op": "command", + "command": { + "aggregate": "collection", + "pipeline": [ + { + "$group": { + "_id": null, + "count": { + "$sum": { + "$numberDouble": "1.0" + } + } + } + }, + { + "$project": { + "_id": { + "$numberDouble": "0.0" + } + } + } + ], + "cursor": {}, + "$db": "testdb" + } + }` + runExplain(ctx, t, prepareParams(t, query)) + }) + + t.Run("Update", func(t *testing.T) { + query := `{ + "ns": "testdb.inventory", + "op": "update", + "command": { + "q": { + "item": "paper" + }, + "u": { + "$set": { + "size.uom": "cm", + "status": "P" + }, + "$currentDate": { + "lastModified": true + } + }, + "multi": false, + "upsert": false + } + }` + runExplain(ctx, t, prepareParams(t, query)) + }) + + t.Run("Remove", func(t *testing.T) { + query := `{ + "ns": "testdb.inventory", + "op": "remove", + "command": { + "q": { + "_id": { + "id": { + "$binary": { + "base64": "vN9ImShsRBaCIFJ23YkysA==", + "subType": "04" + } + }, + "uid": { + "$binary": { + "base64": "Y5mrDaxi8gv8RmdTsQ+1j7fmkr7JUsabhNmXAheU0fg=", + "subType": "00" + } + } + } + }, + "limit": { + "$numberInt": "0" + } + } + }` + runExplain(ctx, t, prepareParams(t, query)) + }) + + t.Run("Distinct", func(t *testing.T) { + query := `{ + "ns": "testdb.inventory", + "op": "command", + "command": { + "distinct": "inventory", + "key": "dept", + "query": {} + }, + "$db": "testdb" + } + }` + runExplain(ctx, t, prepareParams(t, query)) + }) + + t.Run("Insert - not supported", func(t *testing.T) { + query := `{ + "ns": "testdb.listingsAndReviews", + "op": "insert", + "command": { + "insert": "listingsAndReviews", + "ordered": true, + "$db": "testdb" + } + }` + runExplainExpectError(ctx, t, prepareParams(t, query)) + }) + + t.Run("Drop - not supported", func(t *testing.T) { + query := `{ + "ns": "testdb.listingsAndReviews", + "op": "command", + "command": { + "drop": "listingsAndReviews", + "$db": "testdb" + } + }` + runExplainExpectError(ctx, t, prepareParams(t, query)) + }) + + t.Run("PMM-12451", func(t *testing.T) { + // Query from customer to prevent wrong date/time, timestamp parsing in future and prevent regression. + query := `{"ns":"testdb.testDoc","op":"query","command":{"find":"testDoc","filter":{"$and":[{"c23":{"$ne":""}},{"c23":{"$ne":null},"delete":{"$ne":true}},{"$and":[{"c23":"985662747"},{"c15":{"$gte":{"$date":"2023-09-19T22:00:00.000Z"}}},{"c15":{"$lte":{"$date":"2023-10-20T21:59:59.000Z"}}},{"c8":{"$in":["X1118630710X","X1118630720X","X1118630730X","X1118630740X","X1118630750X","X1118630760X"]},"c22":{"$in":["X1118630710X","X1118630710XA","X1118630710XB","X1118630710XC","X1118630710XD","X1118630710XE"]},"c34":{"$in":["X1118630710X","X1118630710Y","X1118630710Z","X1118630710U","X1118630710V","X1118630710W"]}},{"c2":"xxxxxxx"},{"c29":{"$in":["X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X","X1118630710X"]}}]}]},"lsid":{"id":{"$binary":{"base64":"n/f5RI2jTTCoyt0y+8D9Cw==","subType":"04"}}},"$db":"testdb"}}` + runExplain(ctx, t, prepareParams(t, query)) + }) +} + +func prepareParams(t *testing.T, query string) *agentpb.StartActionRequest_MongoDBExplainParams { + t.Helper() + + return &agentpb.StartActionRequest_MongoDBExplainParams{ + Dsn: tests.GetTestMongoDBDSN(t), + Query: query, + } +} + +func runExplain(ctx context.Context, t *testing.T, params *agentpb.StartActionRequest_MongoDBExplainParams) { + t.Helper() + + big, err := rand.Int(rand.Reader, big.NewInt(27)) + require.NoError(t, err) + id := strconv.FormatUint(big.Uint64(), 10) + ex, err := NewMongoDBExplainAction(id, 0, params, os.TempDir()) + require.NoError(t, err) + res, err := ex.Run(ctx) + require.NoError(t, err) + require.NotEmpty(t, string(res)) +} + +func runExplainExpectError(ctx context.Context, t *testing.T, params *agentpb.StartActionRequest_MongoDBExplainParams) { + t.Helper() + + big, err := rand.Int(rand.Reader, big.NewInt(27)) + require.NoError(t, err) + id := strconv.FormatUint(big.Uint64(), 10) + ex, err := NewMongoDBExplainAction(id, 0, params, os.TempDir()) + require.NoError(t, err) + _, err = ex.Run(ctx) + require.Error(t, err) +} + func TestMongoDBExplain(t *testing.T) { database := "test" collection := "test_col" diff --git a/docs/api/pmm-server-config/troubleshooting/logs.md b/docs/api/pmm-server-config/troubleshooting/logs.md index 23ad6c2b9db..d1bc6186f02 100644 --- a/docs/api/pmm-server-config/troubleshooting/logs.md +++ b/docs/api/pmm-server-config/troubleshooting/logs.md @@ -3,12 +3,26 @@ title: Logs slug: "logs" category: 626badcabbc59c02acc1a540 --- +Download the logs and components configuration to troubleshoot any issues with the PMM Server. -Sometimes users need to troubleshoot an issue. PMM Server offers an ability to download the logs as well as configuration of its components. +## Accessing logs -You can download the logs either by calling this endpoint or by visiting a dedicated URL (ex: https://pmmdemo.percona.com/logs.zip) or via the **Settings UI** as explained in the [Troubleshooting](https://docs.percona.com/percona-monitoring-and-management/how-to/troubleshoot.html#client-server-connections) section of our docs. +PMM Server offers three ways to access and download logs: + +1. Through direct URL, by visiting `https:///logs.zip`. +2. By calling the Logs endpoint. This method enables you to customize the log content using the `line-count` parameter: For example: + + - Default 50,000 lines: `https:///logs.zip` + - Custom number of lines: `https:///logs.zip?line-count=10000` + - Unlimited, full log: `https:///logs.zip?line-count=-1` +3. Through the UI, by selecting the **Help > PMM Logs** option from the main menu. + If you need to share logs with Percona Support via an SFTP server, you can also use the **PMM Dump** option from the Help menu to generate a compressed tarball file with an export of your PMM metrics and QAN data. + For more information, see [Export PMM data with PMM Dump](https://docs.percona.com/percona-monitoring-and-management/how-to/PMM_dump.html) topic in the product documentation. + +## Log structure + +The downloaded logs package contains the following structure: -The structure of the logs is as follows: [block:code] { "codes": [ diff --git a/docs/release-notes/2.43.0.md b/docs/release-notes/2.43.0.md index f9ae4aca04b..1386c934c69 100644 --- a/docs/release-notes/2.43.0.md +++ b/docs/release-notes/2.43.0.md @@ -14,6 +14,28 @@ This release introduces this and that ## Release highlights +### Repository changes for PMM: Transition to dedicated `pmm2-client` repository + + +Effective July 1, 2024, Percona has discontinued updates for the original Percona (https://repo.percona.com/percona/) and tools (https://repo.percona.com/tools/) repositories in favor of dedicated product repositories. This change streamlines our repository management and improves efficiency across our product ecosystem. + +For PMM users, this means you can now access the most up-to-date and secure versions of PMM exclusively from the dedicated [repo.percona.com/pmm2-client](`pmm2-client`) repository. To facilitate a smooth transition to this repo, we have developed an automated script that: + +1. Checks installed packages and their repositories +2. Identifies PMM-related packages that should be installed from the `pmm2-client` repository +3. Provides instructions for enabling the correct repository + +To ensure your PMM installation remains secure and up-to-date, [download this script from GitHub](https://raw.githubusercontent.com/Percona-Lab/release-aux/main/scripts/check_percona_packages.py) and run it on your system. + +For detailed instructions and more information about this change, check out [our recent blog post](https://www.percona.com/blog/ensure-the-correct-repositories-are-enabled-for-percona-packages/) and [updated documentation](). + +### Official ARM support for PMM Client +With the growing adoption of ARM in data centers and cloud environments, we are excited to announce official support for ARM architecture in PMM Clients. This enables you to seamlessly monitor ARM-based infrastructure, including popular cloud instances and emerging ARM-powered servers. + +Starting with version 2.43, PMM now includes pre-built binaries for ARM architecture. With this update, PMM Client features, such as `node_exporter` and `mysqld_exporter`, are fully supported and optimized for ARM platforms. Additionally, ARM-based PMM Clients seamlessly integrate with existing PMM Server installations, enabling unified monitoring across both x86 and ARM architectures. + +#### Upgrading and getting started +If you have been compiling PMM on ARM manually, you can now upgrade to the 2.43 release for official support. The [Set up PMM Client topic](../setting-up/client/index.md) now also includes ARM-specific instructions for new installations. ### New MongoDB collector: CurrentOp @@ -33,6 +55,7 @@ Specifying a limit with `--max-collections` for this collector is not necessary. For more information on MongoDB collectors and metrics, see the [pmm-admin commands documentation](../use/commamds/pmm-admin.md). + ## Improvements - [PMM-13133](https://perconadev.atlassian.net/browse/PMM-13133) - @@ -67,3 +90,6 @@ If you're looking to upgrade, you can easily [install the latest version of Perc While these vulnerabilities did not directly impact PMM's core functionality, fixing them enhanced the overall security of the PMM Server environment. +- [PMM-12451](https://perconadev.atlassian.net/browse/PMM-12451) and [PMM-13017](https://perconadev.atlassian.net/browse/PMM-13017) - Resolved an issue with parsing JSON objects into the correct data types and aligned the explain functionality with the official MongoDB client. These updates enhance consistency with MongoDB's native tools and provide improved performance insights. + +- [PMM-13071](https://perconadev.atlassian.net/browse/PMM-13071) - The **Explain** tab on the Query Analytics (QAN) page now properly handles unsupported MongoDB query types. For operations like `INSERT`, which don’t support explain functionality, you will now see a clear message saying that the operation is not explainable. This replaces the previous, confusing error message about duplicate BSON fields, offering more accurate feedback in QAN. diff --git a/go.mod b/go.mod index fcf00c09481..241b5545fda 100644 --- a/go.mod +++ b/go.mod @@ -75,7 +75,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/objx v0.5.2 github.com/stretchr/testify v1.9.0 - go.mongodb.org/mongo-driver v1.16.0 + go.mongodb.org/mongo-driver v1.16.1 go.starlark.net v0.0.0-20230717150657-8a3343210976 golang.org/x/crypto v0.26.0 golang.org/x/sync v0.8.0 diff --git a/go.sum b/go.sum index a10a28b660c..046e96af539 100644 --- a/go.sum +++ b/go.sum @@ -804,8 +804,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.mongodb.org/mongo-driver v1.6.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= -go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= +go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=