From e04a3370addbafe15674eca22e3e77c7b29d958d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 18 Apr 2024 12:50:44 +0300 Subject: [PATCH 01/16] PMM-7 Update pg_query_go --- agent/agents/postgres/parser/parser.go | 2 +- agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go | 2 +- go.mod | 2 +- go.sum | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/agent/agents/postgres/parser/parser.go b/agent/agents/postgres/parser/parser.go index f57d0918cc..e56c4b357f 100644 --- a/agent/agents/postgres/parser/parser.go +++ b/agent/agents/postgres/parser/parser.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - pgquery "github.com/pganalyze/pg_query_go/v2" + pgquery "github.com/pganalyze/pg_query_go/v5" "github.com/pkg/errors" ) diff --git a/agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go b/agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go index 0744b1b18d..28635a135c 100644 --- a/agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go +++ b/agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go @@ -21,7 +21,7 @@ import ( "time" "github.com/AlekSi/pointer" - pgquery "github.com/pganalyze/pg_query_go/v2" + pgquery "github.com/pganalyze/pg_query_go/v5" "github.com/pkg/errors" "github.com/sirupsen/logrus" "gopkg.in/reform.v1" diff --git a/go.mod b/go.mod index be04ff72d2..ea74e77588 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c github.com/percona/percona-toolkit v3.2.1+incompatible github.com/percona/promconfig v0.2.5 - github.com/pganalyze/pg_query_go/v2 v2.2.0 + github.com/pganalyze/pg_query_go/v5 v5.1.0 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 github.com/pmezard/go-difflib v1.0.0 diff --git a/go.sum b/go.sum index 3b9948860a..1e9308da5d 100644 --- a/go.sum +++ b/go.sum @@ -656,8 +656,8 @@ github.com/percona/percona-toolkit v3.2.1+incompatible h1:5jLvtZKcu9fDmaLRB8qA4b github.com/percona/percona-toolkit v3.2.1+incompatible/go.mod h1:netQWdWMaF1cnmwiIS+i5uyaqNXz46yNeM6HKkR6yeI= github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yigquc= github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg= -github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk= -github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA= +github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg= +github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= @@ -1190,6 +1190,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 9a50129dd77171e153505a7b1c375e92bcc394d1 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 18 Apr 2024 13:09:43 +0300 Subject: [PATCH 02/16] PMM-7 Update test case --- agent/agents/postgres/parser/testdata/query019.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/agent/agents/postgres/parser/testdata/query019.json b/agent/agents/postgres/parser/testdata/query019.json index 2289861c43..26f3db0a17 100644 --- a/agent/agents/postgres/parser/testdata/query019.json +++ b/agent/agents/postgres/parser/testdata/query019.json @@ -1,3 +1,7 @@ { - "error": "error on parsing sql query: subquery in FROM must have an alias" + "tables": [ + "credit_card_wo_complaints", + "without_complaints" + ], + "error": "" } From 1aa4911f8c0d40da35218a8530d4f5e848ad09bb Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 19 Apr 2024 15:01:15 +0300 Subject: [PATCH 03/16] PMM-7 Update PostgreSQL versions --- .github/workflows/agent.yml | 20 ++++++------- .../pgstatstatements/pgstatstatements_test.go | 28 ++++++++----------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index c1c5facf5c..11a350a83c 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -26,15 +26,15 @@ jobs: strategy: matrix: images: - - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:10', pmm_server: 'percona/pmm-server:2.0.0' } - - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:11', pmm_server: 'percona/pmm-server:2.0.1' } - - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'percona/pmm-server:2.1.0' } - - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'percona/pmm-server:2.2.0' } + - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:13', pmm_server: 'percona/pmm-server:2.0.0' } + - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:14', pmm_server: 'percona/pmm-server:2.0.1' } + - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'percona/pmm-server:2.1.0' } + - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'percona/pmm-server:2.2.0' } # Percona + latest PMM Server release - - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:11', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' } - - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:12.8-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' } - - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13.5-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' } + - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' } + - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' } + - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' } # MariaDB (only 3 latest GA versions) # + older supported MongoDB versions @@ -44,9 +44,9 @@ jobs: # - MYSQL_IMAGE=mariadb:10.0 # - MYSQL_IMAGE=mariadb:10.1 - - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4', postgres: 'postgres:9.4', pmm_server: 'perconalab/pmm-server:dev-latest' } - - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:9.5', pmm_server: 'perconalab/pmm-server:dev-latest' } - - { mysql: 'mariadb:10.4', postgres: 'postgres:9.6', pmm_server: 'perconalab/pmm-server:dev-latest' } + - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4', postgres: 'postgres:10', pmm_server: 'perconalab/pmm-server:dev-latest' } + - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:11', pmm_server: 'perconalab/pmm-server:dev-latest' } + - { mysql: 'mariadb:10.4', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:dev-latest' } continue-on-error: true diff --git a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go index 36b0740417..0ebcbe9dfe 100644 --- a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go +++ b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go @@ -128,16 +128,6 @@ func TestPGStatStatementsQAN(t *testing.T) { engineVersion := tests.PostgreSQLVersion(t, sqlDB) var digests map[string]string // digest_text/fingerprint to digest/query_id switch engineVersion { - case "9.4": - digests = map[string]string{ - selectAllCities: "3239586867", - selectAllCitiesLong: "2745128652", - } - case "9.5", "9.6": - digests = map[string]string{ - selectAllCities: "3994135135", - selectAllCitiesLong: "2677760328", - } case "10": digests = map[string]string{ selectAllCities: "2229807896", @@ -163,6 +153,16 @@ func TestPGStatStatementsQAN(t *testing.T) { selectAllCities: "5991662752016701281", selectAllCitiesLong: "-3564720362103294944", } + case "15": + digests = map[string]string{ + selectAllCities: "5991662752016701281", + selectAllCitiesLong: "-3564720362103294944", + } + case "16": + digests = map[string]string{ + selectAllCities: "5991662752016701281", + selectAllCitiesLong: "-3564720362103294944", + } default: t.Log("Unhandled version, assuming dummy digests.") digests = map[string]string{ @@ -397,14 +397,8 @@ func TestPGStatStatementsQAN(t *testing.T) { t.Logf("Actual:\n%s", tests.FormatBuckets(buckets)) require.Len(t, buckets, 1) - var fingerprint string - switch engineVersion { - case "9.4", "9.5", "9.6": - fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES (?, ?, ?, ?)`, tableName) + fingerprint := fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName) - default: - fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName) - } actual := buckets[0] assert.NotZero(t, actual.Postgresql.MBlkReadTimeSum) expected := &agentpb.MetricsBucket{ From e05449d850469ce66b9fc9d0f113a3c392f9fb03 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 19 Apr 2024 23:04:41 +0300 Subject: [PATCH 04/16] PMM-7 Update digests --- .../pgstatstatements/pgstatstatements_test.go | 18 +++++++++++++----- .../postgresql_query_show_action_test.go | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go index 0ebcbe9dfe..080f62950b 100644 --- a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go +++ b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go @@ -19,6 +19,7 @@ import ( "fmt" "math/rand" "os" + "strconv" "strings" "sync" "testing" @@ -160,8 +161,8 @@ func TestPGStatStatementsQAN(t *testing.T) { } case "16": digests = map[string]string{ - selectAllCities: "5991662752016701281", - selectAllCitiesLong: "-3564720362103294944", + selectAllCities: "9094455616937907056", + selectAllCitiesLong: "-8264367755446145090", } default: t.Log("Unhandled version, assuming dummy digests.") @@ -282,9 +283,16 @@ func TestPGStatStatementsQAN(t *testing.T) { t.Logf("Actual:\n%s", tests.FormatBuckets(buckets)) require.Len(t, buckets, 1) + expectedSharedBlksHitSum := float32(1007) + engineVersionInt, err := strconv.Atoi(engineVersion) + assert.NoError(t, err) + if engineVersionInt >= 14 { + expectedSharedBlksHitSum = 32 + } + actual := buckets[0] assert.InDelta(t, 0, actual.Common.MQueryTimeSum, 0.09) - assert.InDelta(t, 1010, actual.Postgresql.MSharedBlksHitSum+actual.Postgresql.MSharedBlksReadSum, 3) + assert.InDelta(t, expectedSharedBlksHitSum+3, actual.Postgresql.MSharedBlksHitSum+actual.Postgresql.MSharedBlksReadSum, 3) assert.InDelta(t, 1.5, actual.Postgresql.MSharedBlksHitCnt+actual.Postgresql.MSharedBlksReadCnt, 0.5) expected := &agentpb.MetricsBucket{ Common: &agentpb.MetricsBucket_Common{ @@ -329,7 +337,7 @@ func TestPGStatStatementsQAN(t *testing.T) { actual = buckets[0] assert.InDelta(t, 0, actual.Common.MQueryTimeSum, 0.09) assert.InDelta(t, 0, actual.Postgresql.MBlkReadTimeCnt, 1) - assert.InDelta(t, 1007, actual.Postgresql.MSharedBlksHitSum, 2) + assert.InDelta(t, expectedSharedBlksHitSum, actual.Postgresql.MSharedBlksHitSum, 2) expected = &agentpb.MetricsBucket{ Common: &agentpb.MetricsBucket_Common{ Fingerprint: selectAllCitiesLong, @@ -349,7 +357,7 @@ func TestPGStatStatementsQAN(t *testing.T) { Postgresql: &agentpb.MetricsBucket_PostgreSQL{ MBlkReadTimeCnt: actual.Postgresql.MBlkReadTimeCnt, MBlkReadTimeSum: actual.Postgresql.MBlkReadTimeSum, - MSharedBlksHitCnt: 1, + MSharedBlksHitCnt: actual.Postgresql.MSharedBlksHitCnt, MSharedBlksHitSum: actual.Postgresql.MSharedBlksHitSum, MRowsCnt: 1, MRowsSum: 499, diff --git a/agent/runner/actions/postgresql_query_show_action_test.go b/agent/runner/actions/postgresql_query_show_action_test.go index afd1196b87..3438afad30 100644 --- a/agent/runner/actions/postgresql_query_show_action_test.go +++ b/agent/runner/actions/postgresql_query_show_action_test.go @@ -47,7 +47,7 @@ func TestPostgreSQLQueryShow(t *testing.T) { b, err := a.Run(ctx) require.NoError(t, err) assert.LessOrEqual(t, 22000, len(b)) - assert.LessOrEqual(t, len(b), 33668) + assert.LessOrEqual(t, len(b), 34499) data, err := agentpb.UnmarshalActionQueryResult(b) require.NoError(t, err) From 26eda5610b70bee48a142ee84ed1977e95d5e92b Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 19 Apr 2024 23:16:10 +0300 Subject: [PATCH 05/16] PMM-7 Update digests --- .../postgres/pgstatstatements/pgstatstatements_test.go | 4 +++- agent/runner/actions/postgresql_query_show_action_test.go | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go index 080f62950b..cec121e6a1 100644 --- a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go +++ b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go @@ -426,8 +426,9 @@ func TestPGStatStatementsQAN(t *testing.T) { MQueryTimeSum: actual.Common.MQueryTimeSum, }, Postgresql: &agentpb.MetricsBucket_PostgreSQL{ - MBlkReadTimeCnt: float32(n), + MBlkReadTimeCnt: actual.Postgresql.MBlkReadTimeCnt, MBlkReadTimeSum: actual.Postgresql.MBlkReadTimeSum, + MBlkWriteTimeCnt: actual.Postgresql.MBlkWriteTimeCnt, MSharedBlksReadCnt: actual.Postgresql.MSharedBlksReadCnt, MSharedBlksReadSum: actual.Postgresql.MSharedBlksReadSum, MSharedBlksWrittenCnt: actual.Postgresql.MSharedBlksWrittenCnt, @@ -440,6 +441,7 @@ func TestPGStatStatementsQAN(t *testing.T) { MRowsSum: float32(n), }, } + assert.InDelta(t, n, actual.Postgresql.MBlkWriteTimeCnt+actual.Postgresql.MBlkReadTimeCnt, 1) tests.AssertBucketsEqual(t, expected, actual) assert.LessOrEqual(t, actual.Postgresql.MBlkReadTimeSum, actual.Common.MQueryTimeSum) }) diff --git a/agent/runner/actions/postgresql_query_show_action_test.go b/agent/runner/actions/postgresql_query_show_action_test.go index 3438afad30..73703a6d75 100644 --- a/agent/runner/actions/postgresql_query_show_action_test.go +++ b/agent/runner/actions/postgresql_query_show_action_test.go @@ -46,14 +46,12 @@ func TestPostgreSQLQueryShow(t *testing.T) { b, err := a.Run(ctx) require.NoError(t, err) - assert.LessOrEqual(t, 22000, len(b)) - assert.LessOrEqual(t, len(b), 34499) + assert.GreaterOrEqual(t, len(b), 22000) data, err := agentpb.UnmarshalActionQueryResult(b) require.NoError(t, err) t.Log(spew.Sdump(data)) - assert.LessOrEqual(t, 200, len(data)) - assert.LessOrEqual(t, len(data), 358) + assert.GreaterOrEqual(t, len(data), 200) var found int for _, m := range data { From f0d61602ea4d4f4e02d92afd37ac5526983ec572 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 19 Apr 2024 23:21:52 +0300 Subject: [PATCH 06/16] PMM-7 Update digests --- agent/agents/postgres/pgstatstatements/pgstatstatements_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go index cec121e6a1..b65d91ab28 100644 --- a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go +++ b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go @@ -408,7 +408,6 @@ func TestPGStatStatementsQAN(t *testing.T) { fingerprint := fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName) actual := buckets[0] - assert.NotZero(t, actual.Postgresql.MBlkReadTimeSum) expected := &agentpb.MetricsBucket{ Common: &agentpb.MetricsBucket_Common{ Queryid: actual.Common.Queryid, @@ -429,6 +428,7 @@ func TestPGStatStatementsQAN(t *testing.T) { MBlkReadTimeCnt: actual.Postgresql.MBlkReadTimeCnt, MBlkReadTimeSum: actual.Postgresql.MBlkReadTimeSum, MBlkWriteTimeCnt: actual.Postgresql.MBlkWriteTimeCnt, + MBlkWriteTimeSum: actual.Postgresql.MBlkWriteTimeSum, MSharedBlksReadCnt: actual.Postgresql.MSharedBlksReadCnt, MSharedBlksReadSum: actual.Postgresql.MSharedBlksReadSum, MSharedBlksWrittenCnt: actual.Postgresql.MSharedBlksWrittenCnt, From 50b9b76c0f31843df39725c1737493ea349924e0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 24 Apr 2024 16:57:55 +0300 Subject: [PATCH 07/16] PMM-7 go mod tidy --- go.mod | 3 --- 1 file changed, 3 deletions(-) diff --git a/go.mod b/go.mod index f41bbe9ecf..62cece3856 100644 --- a/go.mod +++ b/go.mod @@ -193,9 +193,6 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.17.0 // indirect golang.org/x/term v0.19.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.3.0 // indirect ) From e5a6de8a017a2e826741557d335eb2d3dcad7432 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 05:42:31 +0300 Subject: [PATCH 08/16] PMM-11180 update matrix and fix pgstatstatements_test.go. --- .github/workflows/agent.yml | 16 +++++++++------- .../pgstatstatements/pgstatstatements_test.go | 2 +- go.sum | 5 ----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index 7e8564cca0..9d9e59ad7a 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -28,15 +28,17 @@ jobs: strategy: matrix: images: - - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mysql:5.7', mongo: 'mongo:5.0', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mysql:8.0', mongo: 'mongo:6.0', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mysql:8.0', mongo: 'mongo:7.0', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mysql:latest', mongo: 'mongo:latest', postgres: 'postgres:latest', pmm_server: 'perconalab/pmm-server:3-dev-latest' } # Percona + latest PMM Server release - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'percona:latest', mongo: 'percona/percona-server-mongodb:latest', postgres: 'perconalab/percona-distribution-postgresql:latest', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } # MariaDB (only 3 latest GA versions) # + older supported MongoDB versions @@ -46,9 +48,9 @@ jobs: # - MYSQL_IMAGE=mariadb:10.0 # - MYSQL_IMAGE=mariadb:10.1 - - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4', postgres: 'postgres:10', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:11', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'mariadb:10.4', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mariadb:11.2', mongo: 'mongo:4.4', postgres: 'postgres:10', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mariadb:11.4', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:11', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mariadb:11.5', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' } continue-on-error: true diff --git a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go index ec02f0f92b..3c1c1c8360 100644 --- a/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go +++ b/agent/agents/postgres/pgstatstatements/pgstatstatements_test.go @@ -413,7 +413,7 @@ func TestPGStatStatementsQAN(t *testing.T) { Queryid: actual.Common.Queryid, Fingerprint: fingerprint, Database: "pmm-agent", - Tables: tables, + Tables: []string{tableName}, Comments: map[string]string{"controller": "test"}, Username: "pmm-agent", AgentId: "agent_id", diff --git a/go.sum b/go.sum index a4687f29fc..c236c1d13f 100644 --- a/go.sum +++ b/go.sum @@ -533,11 +533,6 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= From c54769e2ffa098aa105e7c6802a30c0db144ed97 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 05:59:24 +0300 Subject: [PATCH 09/16] PMM-11180 fix mongodb tests. --- .github/workflows/agent.yml | 2 +- .../actions/mongodb_explain_action_test.go | 27 ++++++++++++++++--- .../mongodb_query_admincommand_action_test.go | 1 - 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index 9d9e59ad7a..c4d46fbd6e 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -31,7 +31,7 @@ jobs: - { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - { mysql: 'mysql:5.7', mongo: 'mongo:5.0', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - { mysql: 'mysql:8.0', mongo: 'mongo:6.0', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'mysql:8.0', mongo: 'mongo:7.0', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'mysql:lts', mongo: 'mongo:7.0', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - { mysql: 'mysql:latest', mongo: 'mongo:latest', postgres: 'postgres:latest', pmm_server: 'perconalab/pmm-server:3-dev-latest' } # Percona + latest PMM Server release diff --git a/agent/runner/actions/mongodb_explain_action_test.go b/agent/runner/actions/mongodb_explain_action_test.go index f8218d9a8d..15ec35a201 100644 --- a/agent/runner/actions/mongodb_explain_action_test.go +++ b/agent/runner/actions/mongodb_explain_action_test.go @@ -282,9 +282,30 @@ func TestMongoDBExplain(t *testing.T) { "parsedQuery": map[string]interface{}{ "k": map[string]interface{}{"$lte": map[string]interface{}{"$numberInt": "1"}}, }, - "plannerVersion": map[string]interface{}{"$numberInt": "1"}, - "rejectedPlans": []interface{}{}, - "winningPlan": map[string]interface{}{"stage": "EOF"}, + "rejectedPlans": []interface{}{}, + "winningPlan": map[string]interface{}{"stage": "EOF"}, + } + mongoDBVersion := tests.MongoDBVersion(t, client) + + switch mongoDBVersion { + case "4.4": + want["plannerVersion"] = map[string]interface{}{"$numberInt": "1"} + case "5.0": + want["maxIndexedAndSolutionsReached"] = false + want["maxIndexedOrSolutionsReached"] = false + want["maxScansToExplodeReached"] = false + case "6.0": + want["maxIndexedAndSolutionsReached"] = false + want["maxIndexedOrSolutionsReached"] = false + want["maxScansToExplodeReached"] = false + want["planCacheKey"] = "88A125C6" + want["queryHash"] = "88A125C6" + case "7.0": + want["maxIndexedAndSolutionsReached"] = false + want["maxIndexedOrSolutionsReached"] = false + want["maxScansToExplodeReached"] = false + want["planCacheKey"] = "0F06B42F" + want["queryHash"] = "0F06B42F" } explainM := make(map[string]interface{}) diff --git a/agent/runner/actions/mongodb_query_admincommand_action_test.go b/agent/runner/actions/mongodb_query_admincommand_action_test.go index 08c6a1eacd..5223d581fd 100644 --- a/agent/runner/actions/mongodb_query_admincommand_action_test.go +++ b/agent/runner/actions/mongodb_query_admincommand_action_test.go @@ -196,7 +196,6 @@ func convertToObjxMap(t *testing.T, b []byte) objx.Map { func getParameterAssertions(t *testing.T, b []byte) { //nolint:thelper assert.LessOrEqual(t, 5000, len(b)) - assert.LessOrEqual(t, len(b), 17000) objxM := convertToObjxMap(t, b) assert.Equal(t, 1.0, objxM.Get("ok").Data()) assert.Contains(t, objxM.Get("authenticationMechanisms").Data(), "SCRAM-SHA-1") From 7157768b4a766c5b501e18092d011bdbda40d037 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 06:08:36 +0300 Subject: [PATCH 10/16] PMM-11180 fix mongodb tests. --- .../internal/profiler/profiler_test.go | 22 +++++++++++++++++-- .../actions/mongodb_explain_action_test.go | 1 + 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/agent/agents/mongodb/internal/profiler/profiler_test.go b/agent/agents/mongodb/internal/profiler/profiler_test.go index b6e044ee02..171c77bc90 100644 --- a/agent/agents/mongodb/internal/profiler/profiler_test.go +++ b/agent/agents/mongodb/internal/profiler/profiler_test.go @@ -221,6 +221,7 @@ func testProfiler(t *testing.T, url string) { Dsn: tests.GetTestMongoDBDSN(t), Query: findBucket.Common.Example, } + client := tests.OpenTestMongoDB(t, params.Dsn) ex, err := actions.NewMongoDBExplainAction(id, 5*time.Second, params, os.TempDir()) require.NoError(t, err) @@ -238,8 +239,25 @@ func testProfiler(t *testing.T, url string) { "$eq": "value_00\ufffd", }, }, - "plannerVersion": map[string]interface{}{"$numberInt": "1"}, - "rejectedPlans": []interface{}{}, + "rejectedPlans": []interface{}{}, + } + mongoDBVersion := tests.MongoDBVersion(t, client) + + switch mongoDBVersion { + case "4.4": + want["plannerVersion"] = map[string]interface{}{"$numberInt": "1"} + case "5.0": + want["maxIndexedAndSolutionsReached"] = false + want["maxIndexedOrSolutionsReached"] = false + want["maxScansToExplodeReached"] = false + case "6.0": + want["maxIndexedAndSolutionsReached"] = false + want["maxIndexedOrSolutionsReached"] = false + want["maxScansToExplodeReached"] = false + case "7.0": + want["maxIndexedAndSolutionsReached"] = false + want["maxIndexedOrSolutionsReached"] = false + want["maxScansToExplodeReached"] = false } explainM := make(map[string]interface{}) diff --git a/agent/runner/actions/mongodb_explain_action_test.go b/agent/runner/actions/mongodb_explain_action_test.go index 15ec35a201..13bee4627a 100644 --- a/agent/runner/actions/mongodb_explain_action_test.go +++ b/agent/runner/actions/mongodb_explain_action_test.go @@ -286,6 +286,7 @@ func TestMongoDBExplain(t *testing.T) { "winningPlan": map[string]interface{}{"stage": "EOF"}, } mongoDBVersion := tests.MongoDBVersion(t, client) + fmt.Printf("MongoDB version: %s\n", mongoDBVersion) switch mongoDBVersion { case "4.4": From eb507d037012bebaf53f0ee52fb55fb120d4e637 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 06:14:15 +0300 Subject: [PATCH 11/16] PMM-11180 fix mongodb tests. --- agent/runner/actions/mongodb_explain_action_test.go | 13 +------------ agent/utils/tests/mongodb.go | 8 +++++++- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/agent/runner/actions/mongodb_explain_action_test.go b/agent/runner/actions/mongodb_explain_action_test.go index 13bee4627a..82ed7191bf 100644 --- a/agent/runner/actions/mongodb_explain_action_test.go +++ b/agent/runner/actions/mongodb_explain_action_test.go @@ -343,8 +343,7 @@ func TestNewMongoDBExplain(t *testing.T) { in: "distinct.json", }, { - in: "aggregate.json", - minVersion: "3.4.0", + in: "aggregate.json", }, { in: "count.json", @@ -353,17 +352,7 @@ func TestNewMongoDBExplain(t *testing.T) { in: "find_and_modify.json", }, } - mongoDBVersion := tests.MongoDBVersion(t, client) for _, tf := range testFiles { - // Not all MongoDB versions allow explaining all commands - if tf.minVersion != "" { - c, err := lessThan(tf.minVersion, mongoDBVersion) - require.NoError(t, err) - if c { - continue - } - } - t.Run(tf.in, func(t *testing.T) { query, err := os.ReadFile(filepath.Join("testdata/", filepath.Clean(tf.in))) assert.NoError(t, err) diff --git a/agent/utils/tests/mongodb.go b/agent/utils/tests/mongodb.go index 537fc20411..7c7d9741c4 100644 --- a/agent/utils/tests/mongodb.go +++ b/agent/utils/tests/mongodb.go @@ -16,6 +16,8 @@ package tests import ( "context" + "fmt" + "github.com/percona/pmm/version" "os" "path/filepath" "testing" @@ -131,5 +133,9 @@ func MongoDBVersion(tb testing.TB, client *mongo.Client) string { if err := res.Decode(&bi); err != nil { tb.Fatalf("Cannot decode buildInfo response: %s", err) } - return bi.Version + parsed, err := version.Parse(bi.Version) + if err != nil { + tb.Fatalf("Cannot parse version: %s", err) + } + return fmt.Sprintf("%d.%d", parsed.Major, parsed.Minor) } From c472e3f4aae9f86277b5dcdf72d65c97a94f9d84 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 06:22:47 +0300 Subject: [PATCH 12/16] PMM-11180 fix mongodb tests. --- .github/workflows/agent.yml | 7 ++++--- agent/runner/actions/mongodb_explain_action_test.go | 13 +++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index c4d46fbd6e..953329b39d 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -35,9 +35,10 @@ jobs: - { mysql: 'mysql:latest', mongo: 'mongo:latest', postgres: 'postgres:latest', pmm_server: 'perconalab/pmm-server:3-dev-latest' } # Percona + latest PMM Server release - - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:5.0', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:6.0', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } + - { mysql: 'percona:8', mongo: 'percona/percona-server-mongodb:7.0', postgres: 'perconalab/percona-distribution-postgresql:17', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } - { mysql: 'percona:latest', mongo: 'percona/percona-server-mongodb:latest', postgres: 'perconalab/percona-distribution-postgresql:latest', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' } # MariaDB (only 3 latest GA versions) diff --git a/agent/runner/actions/mongodb_explain_action_test.go b/agent/runner/actions/mongodb_explain_action_test.go index 82ed7191bf..4e78e4df0f 100644 --- a/agent/runner/actions/mongodb_explain_action_test.go +++ b/agent/runner/actions/mongodb_explain_action_test.go @@ -289,22 +289,19 @@ func TestMongoDBExplain(t *testing.T) { fmt.Printf("MongoDB version: %s\n", mongoDBVersion) switch mongoDBVersion { - case "4.4": + case "4.4", "4.2": want["plannerVersion"] = map[string]interface{}{"$numberInt": "1"} - case "5.0": + case "5.0", "6.0", "7.0": want["maxIndexedAndSolutionsReached"] = false want["maxIndexedOrSolutionsReached"] = false want["maxScansToExplodeReached"] = false + } + + switch mongoDBVersion { case "6.0": - want["maxIndexedAndSolutionsReached"] = false - want["maxIndexedOrSolutionsReached"] = false - want["maxScansToExplodeReached"] = false want["planCacheKey"] = "88A125C6" want["queryHash"] = "88A125C6" case "7.0": - want["maxIndexedAndSolutionsReached"] = false - want["maxIndexedOrSolutionsReached"] = false - want["maxScansToExplodeReached"] = false want["planCacheKey"] = "0F06B42F" want["queryHash"] = "0F06B42F" } From 6bf40eeb1654eccf81cc5189b5d7637a9d2abbd6 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 12:48:38 +0300 Subject: [PATCH 13/16] PMM-11180 fix init scripts for mongodb. --- .../internal/profiler/profiler_test.go | 9 ++++++++ agent/docker-compose.yml | 21 +++++-------------- agent/packages.dot | 2 -- agent/testscripts/mongo_repl.sh | 19 +++++++++++++++++ agent/testscripts/mongo_repl_with_ssl.sh | 19 +++++++++++++++++ 5 files changed, 52 insertions(+), 18 deletions(-) create mode 100755 agent/testscripts/mongo_repl.sh create mode 100755 agent/testscripts/mongo_repl_with_ssl.sh diff --git a/agent/agents/mongodb/internal/profiler/profiler_test.go b/agent/agents/mongodb/internal/profiler/profiler_test.go index 171c77bc90..211895f031 100644 --- a/agent/agents/mongodb/internal/profiler/profiler_test.go +++ b/agent/agents/mongodb/internal/profiler/profiler_test.go @@ -260,6 +260,15 @@ func testProfiler(t *testing.T, url string) { want["maxScansToExplodeReached"] = false } + switch mongoDBVersion { + case "6.0": + want["planCacheKey"] = "4FA12547" + want["queryHash"] = "4FA12547" + case "7.0": + want["planCacheKey"] = "0F06B42F" + want["queryHash"] = "0F06B42F" + } + explainM := make(map[string]interface{}) err = json.Unmarshal(res, &explainM) assert.Nil(t, err) diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml index a8f7aecb78..5e39acb0c7 100644 --- a/agent/docker-compose.yml +++ b/agent/docker-compose.yml @@ -82,14 +82,9 @@ services: mongo_repl: image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} container_name: pmm-agent_mongorepl - command: > - bash -c " - mkdir /tmp/mongodb1 /tmp/mongodb2 - mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --noauth --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb1 --port=27020 - mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --noauth --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb2 --port=27021 - mongo --port 27020 --eval \"rs.initiate( { _id : 'rs0', members: [{ _id: 0, host: 'localhost:27020' }, { _id: 1, host: 'localhost:27021', priority: 0 }]})\" - tail -f /dev/null - " + volumes: + - ./testscripts/:/scripts + command: /scripts/mongo_repl.sh ports: - "127.0.0.1:27020:27020" - "127.0.0.1:27021:27021" @@ -97,18 +92,12 @@ services: mongo_repl_with_ssl: image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} container_name: pmm-agent_mongorepl_with_ssl - command: > - bash -c " - mkdir /tmp/mongodb1 /tmp/mongodb2 - mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --sslMode=requireSSL --sslPEMKeyFile=/etc/ssl/certificates/server.pem --sslCAFile=/etc/ssl/certificates/ca.crt --sslClusterFile=/etc/ssl/certificates/client.pem --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb1 --port=27022 - mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --sslMode=requireSSL --sslPEMKeyFile=/etc/ssl/certificates/server.pem --sslCAFile=/etc/ssl/certificates/ca.crt --sslClusterFile=/etc/ssl/certificates/client.pem --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb2 --port=27023 - mongo --port 27022 --ssl --sslCAFile=/etc/ssl/certificates/ca.crt --sslPEMKeyFile=/etc/ssl/certificates/client.pem --sslAllowInvalidHostnames --eval \"rs.initiate( { _id : 'rs0', members: [{ _id: 0, host: 'localhost:27022' }, { _id: 1, host: 'localhost:27023', priority: 0 }]})\" - tail -f /dev/null - " + command: /scripts/mongo_repl_with_ssl.sh ports: - "127.0.0.1:27022:27022" - "127.0.0.1:27023:27023" volumes: + - ./testscripts/:/scripts - ${PWD}/utils/tests/testdata/mongodb:/etc/ssl/certificates postgres: diff --git a/agent/packages.dot b/agent/packages.dot index bb9f70bf21..74d45da9a1 100644 --- a/agent/packages.dot +++ b/agent/packages.dot @@ -40,11 +40,9 @@ digraph packages { "/agents/supervisor" -> "/config"; "/agents/supervisor" -> "/tailog"; "/agents/supervisor.test" -> "/agents/supervisor"; - "/client" -> "/client/cache"; "/client" -> "/client/channel"; "/client" -> "/config"; "/client" -> "/connectionuptime"; - "/client" -> "/models"; "/client" -> "/runner"; "/client" -> "/runner/actions"; "/client" -> "/runner/jobs"; diff --git a/agent/testscripts/mongo_repl.sh b/agent/testscripts/mongo_repl.sh new file mode 100755 index 0000000000..24f441640e --- /dev/null +++ b/agent/testscripts/mongo_repl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +MONGODB_CLIENT="mongosh" +PARSED=(${MONGO_IMAGE//:/ }) +MONGODB_VERSION=${PARSED[1]} +MONGODB_VENDOR=${PARSED[0]} + +if [ "`echo ${MONGODB_VERSION} | cut -c 1`" = "4" ]; then + MONGODB_CLIENT="mongo" +fi +if [ "`echo ${MONGODB_VERSION} | cut -c 1`" = "5" ] && [ ${MONGODB_VENDOR} == "percona/percona-server-mongodb" ]; then + MONGODB_CLIENT="mongo" +fi + +mkdir /tmp/mongodb1 /tmp/mongodb2 +mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --noauth --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb1 --port=27020 +mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --noauth --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb2 --port=27021 +$MONGODB_CLIENT --port 27020 --eval "rs.initiate( { _id : 'rs0', members: [{ _id: 0, host: 'localhost:27020' }, { _id: 1, host: 'localhost:27021', priority: 0 }]})" +tail -f /dev/null \ No newline at end of file diff --git a/agent/testscripts/mongo_repl_with_ssl.sh b/agent/testscripts/mongo_repl_with_ssl.sh new file mode 100755 index 0000000000..21e47075f0 --- /dev/null +++ b/agent/testscripts/mongo_repl_with_ssl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +MONGODB_CLIENT="mongosh" +PARSED=(${MONGO_IMAGE//:/ }) +MONGODB_VERSION=${PARSED[1]} +MONGODB_VENDOR=${PARSED[0]} + +if [ "`echo ${MONGODB_VERSION} | cut -c 1`" = "4" ]; then + MONGODB_CLIENT="mongo" +fi +if [ "`echo ${MONGODB_VERSION} | cut -c 1`" = "5" ] && [ ${MONGODB_VENDOR} == "percona/percona-server-mongodb" ]; then + MONGODB_CLIENT="mongo" +fi + +mkdir /tmp/mongodb1 /tmp/mongodb2 +mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --sslMode=requireSSL --sslPEMKeyFile=/etc/ssl/certificates/server.pem --sslCAFile=/etc/ssl/certificates/ca.crt --sslClusterFile=/etc/ssl/certificates/client.pem --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb1 --port=27022 +mongod --fork --logpath=/dev/null --profile=2 --replSet=rs0 --sslMode=requireSSL --sslPEMKeyFile=/etc/ssl/certificates/server.pem --sslCAFile=/etc/ssl/certificates/ca.crt --sslClusterFile=/etc/ssl/certificates/client.pem --bind_ip=0.0.0.0 --dbpath=/tmp/mongodb2 --port=27023 +$MONGODB_CLIENT --port 27022 --ssl --sslCAFile=/etc/ssl/certificates/ca.crt --sslPEMKeyFile=/etc/ssl/certificates/client.pem --sslAllowInvalidHostnames --eval "rs.initiate( { _id : 'rs0', members: [{ _id: 0, host: 'localhost:27022' }, { _id: 1, host: 'localhost:27023', priority: 0 }]})" +tail -f /dev/null \ No newline at end of file From 4d4bebff724b7d87f7d65a3ae9305edc1744f91e Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 12:50:19 +0300 Subject: [PATCH 14/16] PMM-11180 update mongodb version. --- agent/docker-compose.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml index 5e39acb0c7..3fafd4cabd 100644 --- a/agent/docker-compose.yml +++ b/agent/docker-compose.yml @@ -48,7 +48,7 @@ services: - ./testdata/mysql:/mysql mongo: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} container_name: pmm-agent_mongo command: --profile 2 ports: @@ -58,7 +58,7 @@ services: - MONGO_INITDB_ROOT_PASSWORD=root-password mongo_with_ssl: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} container_name: pmm-agent_mongo_with_ssl command: - --profile=2 @@ -73,15 +73,17 @@ services: - ${PWD}/utils/tests/testdata/mongodb:/etc/ssl/certificates mongonoauth: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} container_name: pmm-agent_mongonoauth command: --profile 2 ports: - "127.0.0.1:27019:27017" mongo_repl: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} container_name: pmm-agent_mongorepl + environment: + - VERSION=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} volumes: - ./testscripts/:/scripts command: /scripts/mongo_repl.sh @@ -90,9 +92,11 @@ services: - "127.0.0.1:27021:27021" mongo_repl_with_ssl: - image: ${MONGO_IMAGE:-percona/percona-server-mongodb:4.2} + image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} container_name: pmm-agent_mongorepl_with_ssl command: /scripts/mongo_repl_with_ssl.sh + environment: + - VERSION=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} ports: - "127.0.0.1:27022:27022" - "127.0.0.1:27023:27023" From c6ccc534728ae8c54319aaf4d8ae21dbc6ea1d8c Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 13:03:09 +0300 Subject: [PATCH 15/16] PMM-11180 add debug info on fail. --- .github/workflows/agent.yml | 1 + agent/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index 953329b39d..50c509a0e4 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -123,3 +123,4 @@ jobs: echo "--- GO Environment ---" go env | sort git status + docker compose logs diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml index 3fafd4cabd..08ca83ae29 100644 --- a/agent/docker-compose.yml +++ b/agent/docker-compose.yml @@ -83,7 +83,7 @@ services: image: ${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} container_name: pmm-agent_mongorepl environment: - - VERSION=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + - MONGO_IMAGE=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} volumes: - ./testscripts/:/scripts command: /scripts/mongo_repl.sh @@ -96,7 +96,7 @@ services: container_name: pmm-agent_mongorepl_with_ssl command: /scripts/mongo_repl_with_ssl.sh environment: - - VERSION=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} + - MONGO_IMAGE=${MONGO_IMAGE:-percona/percona-server-mongodb:5.0} ports: - "127.0.0.1:27022:27022" - "127.0.0.1:27023:27023" From 6efc356c2af053bcd07221b13f98a1b5cbf8a3d9 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 16 Oct 2024 16:06:37 +0300 Subject: [PATCH 16/16] PMM-11180 Improvements. --- agent/agents/mongodb/internal/profiler/profiler_test.go | 8 ++++---- agent/docker-compose.yml | 1 - .../actions/mongodb_query_admincommand_action_test.go | 3 --- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/agent/agents/mongodb/internal/profiler/profiler_test.go b/agent/agents/mongodb/internal/profiler/profiler_test.go index 211895f031..891abf4667 100644 --- a/agent/agents/mongodb/internal/profiler/profiler_test.go +++ b/agent/agents/mongodb/internal/profiler/profiler_test.go @@ -69,11 +69,11 @@ func TestProfiler(t *testing.T) { tempDir := t.TempDir() sslDSN, err := templates.RenderDSN(sslDSNTemplate, files, tempDir) require.NoError(t, err) - for _, url := range []string{ - "mongodb://root:root-password@127.0.0.1:27017/admin", - sslDSN, + for name, url := range map[string]string{ + "normal": tests.GetTestMongoDBDSN(t), + "ssl": sslDSN, } { - t.Run(url, func(t *testing.T) { + t.Run(name, func(t *testing.T) { testProfiler(t, url) }) } diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml index 08ca83ae29..43934865dd 100644 --- a/agent/docker-compose.yml +++ b/agent/docker-compose.yml @@ -66,7 +66,6 @@ services: - --sslPEMKeyFile=/etc/ssl/certificates/server.pem - --sslCAFile=/etc/ssl/certificates/ca.crt - --sslWeakCertificateValidation - - --bind_ip=0.0.0.0 ports: - "127.0.0.1:27018:27017" volumes: diff --git a/agent/runner/actions/mongodb_query_admincommand_action_test.go b/agent/runner/actions/mongodb_query_admincommand_action_test.go index 5223d581fd..38d9afd4f6 100644 --- a/agent/runner/actions/mongodb_query_admincommand_action_test.go +++ b/agent/runner/actions/mongodb_query_admincommand_action_test.go @@ -203,7 +203,6 @@ func getParameterAssertions(t *testing.T, b []byte) { //nolint:thelper func buildInfoAssertions(t *testing.T, b []byte) { //nolint:thelper assert.LessOrEqual(t, 1000, len(b)) - assert.LessOrEqual(t, len(b), 2200) objxM := convertToObjxMap(t, b) assert.Equal(t, 1.0, objxM.Get("ok").Data()) assert.Equal(t, "mozjs", objxM.Get("javascriptEngine").Data()) @@ -212,7 +211,6 @@ func buildInfoAssertions(t *testing.T, b []byte) { //nolint:thelper func getDiagnosticDataAssertions(t *testing.T, b []byte) { //nolint:thelper assert.LessOrEqual(t, 25000, len(b)) - assert.LessOrEqual(t, len(b), 110000) objxM := convertToObjxMap(t, b) assert.Equal(t, 1.0, objxM.Get("ok").Data()) assert.Equal(t, 1.0, objxM.Get("data.serverStatus.ok").Data()) @@ -221,7 +219,6 @@ func getDiagnosticDataAssertions(t *testing.T, b []byte) { //nolint:thelper func replSetGetStatusAssertionsReplicated(t *testing.T, b []byte) { //nolint:thelper assert.LessOrEqual(t, 1000, len(b)) - assert.LessOrEqual(t, len(b), 4000) objxM := convertToObjxMap(t, b) assert.Equal(t, 1.0, objxM.Get("ok").Data()) assert.Len(t, objxM.Get("members").Data(), 2)