Merge branch 'main' into PMM-7-build-devcontainer-with-actions

.devcontainer/setup.py

@@ -1,16 +1,15 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 # See CONTRIBUTING.md.
 
-from __future__ import print_function, unicode_literals
 import os
 import subprocess
 import time
 
 
 GO_VERSION = os.getenv("GO_VERSION")
 if GO_VERSION is None:
-    raise "GO_VERSION is not set"
+    raise RuntimeError("GO_VERSION is not set")
 
 
 def run_commands(commands):
@@ -25,21 +24,16 @@ def install_packages():
     """Installs required and useful RPM packages."""
 
     run_commands([
-        # to install man pages
-        "sed -i '/nodocs/d' /etc/yum.conf",
-
-        # reinstall with man pages
-        "yum reinstall -y yum rpm",
-
-        "yum install -y gcc git make pkgconfig glibc-static \
+        "dnf install -y gcc git make pkgconfig \
             vim \
-            ansible-lint \
             mc tmux psmisc lsof which iproute \
-            bash-completion bash-completion-extras \
+            bash-completion \
             man man-pages \
-            dh-autoreconf \
             openssl-devel \
-            wget"
+            wget",
+
+        "dnf install -y ansible-lint glibc-static --enablerepo=ol9_codeready_builder"
+
     ])
 
 
@@ -51,7 +45,7 @@ def install_go():
         "chmod +x /usr/local/bin/gimme"
     ])
 
-    go_version = str(subprocess.check_output("gimme -r " + GO_VERSION, shell=True).strip())
+    go_version = str(subprocess.check_output("gimme -r " + GO_VERSION, shell=True).strip().decode())
 
     if GO_VERSION == "tip":
         run_commands([
@@ -105,7 +99,8 @@ def main():
     make_init()
 
     # do basic setup
-    setup()
+    # TODO: fix the setup and revert
+    # setup()
 
 
 MARKER = "/tmp/devcontainer-setup-done"

.github/dependabot.yml

@@ -5,16 +5,25 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
 
   - package-ecosystem: "gomod"
     directory: "/tools"
     schedule:
       interval: "daily"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
 
   - package-ecosystem: "gomod"
     directory: "/api-tests/tools"
     schedule:
       interval: "daily"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
 
   - package-ecosystem: "docker"
     directory: "/"
@@ -45,3 +54,6 @@ updates:
     directory: "/cli-tests"
     schedule:
       interval: "daily"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]

.github/workflows/main.yml

@@ -133,7 +133,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           interval: 45
           timeout: 1200
-          ignored: "WhiteSource License Check"
+          ignored: "WhiteSource License Check, WhiteSource Security Check"
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
   workflow_success:

.github/workflows/managed.yml

@@ -9,14 +9,14 @@ on:
 
   pull_request:
     paths-ignore:
-      - "admin/**"
-      - "agent/**"
-      - "api-tests/**"
-      - "cli-tests/**"
-      - "docs/**"
-      - "qan-api2/**"
-      - "update/**"
-      - "vmproxy/**"
+      - 'admin/**'
+      - 'agent/**'
+      - 'api-tests/**'
+      - 'cli-tests/**'
+      - 'docs/**'
+      - 'qan-api2/**'
+      - 'update/**'
+      - 'vmproxy/**'
 
 jobs:
   test:
@@ -63,26 +63,29 @@ jobs:
           popd        && go mod download -x
 
       - name: Initialize CI environment
-        run: make env-compose-up
+        run: make env-compose-up # the container workdir is /root/go/src/github.com/percona/pmm
 
       - name: Restore Go build cache
         if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
         continue-on-error: true
-        run: docker cp ~/.cache/go-build pmm-managed-server:/root/.cache/go-build
+        run: docker cp ~/.cache/go-build pmm-server:/root/.cache/go-build
 
       - name: Restore Go modules cache
         if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
         continue-on-error: true
-        run: docker cp ~/go/pkg/mod pmm-managed-server:/root/go/pkg/mod
+        run: docker cp ~/go/pkg/mod pmm-server:/root/go/pkg/mod
+
+      - name: Mark the root directory of pmm as safe
+        run: docker exec -i pmm-server git config --global --add safe.directory /root/go/src/github.com/percona/pmm
 
       - name: Update binaries
-        run: docker exec -i --workdir=/root/go/src/github.com/percona/pmm pmm-managed-server make run-managed-ci run-agent run-vmproxy
+        run: docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy
 
       - name: Run tests
-        run: docker exec -i --workdir=/root/go/src/github.com/percona/pmm/managed pmm-managed-server make test-cover
+        run: docker exec -i pmm-server make -C managed test-cover
 
       - name: Run PMM server update test
-        run: docker exec -i --workdir=/root/go/src/github.com/percona/pmm/managed pmm-managed-server make test-update
+        run: docker exec -i pmm-server make -C managed test-update
 
       - name: Upload coverage results
         uses: codecov/codecov-action@v3
@@ -95,11 +98,11 @@ jobs:
       - name: Cache
         if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
         run: |
-          docker exec pmm-managed-server go clean -testcache
-          docker exec --workdir=/root/go/src/github.com/percona/pmm/managed pmm-managed-server find . -type d -name fuzzdata -exec rm -r {} +
+          docker exec pmm-server go clean -testcache
+          docker exec pmm-server find ./managed -type d -name fuzzdata -exec rm -r {} +
           rm -fr ~/.cache/go-build
           mkdir -p ~/.cache
-          docker cp pmm-managed-server:/root/.cache/go-build ~/.cache/go-build
+          docker cp pmm-server:/root/.cache/go-build ~/.cache/go-build
 
       - name: Run debug commands on failure
         if: ${{ failure() }}
@@ -109,4 +112,4 @@ jobs:
           go env
           pwd
           git status
-          kubectl version
+          kubectl version --short --output json

.github/workflows/update.yml

@@ -45,7 +45,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Docker Up
-        run: docker-compose up -d
+        run: docker compose up -d
 
       - name: Install dev tools in container
         run: docker exec pmm-update-server /root/go/src/github.com/percona/pmm/update/.devcontainer/install-dev-tools.sh

.golangci.yml

@@ -29,6 +29,11 @@ linters-settings:
 
   godot:
     capital: true
+    period: false
+    scope: toplevel
+    exclude:
+      - go-sumtype:decl 
+      - pmm-managed
 
   goimports:
     local-prefixes: github.com/percona/pmm
@@ -45,11 +50,15 @@ linters-settings:
       - stdlib
       - github.com/charmbracelet/bubbletea.Model
       - github.com/percona/pmm/admin/commands.Result
+      - github.com/percona/pmm/agent/runner/actions.Action
 
   lll:
     line-length: 170
     tab-width: 4
 
+  maintidx:
+    under: 20
+
   nestif:
     min-complexity: 7
 
@@ -84,6 +93,7 @@ linters:
     - gochecknoglobals # mostly useless
     - gochecknoinits   # we use init functions
     - gocyclo          # using cyclop with the max 30 instead
+    - godox            # we sometimes leave TODOS right in the code
     - goerr113         # extra work & poor benefit
     - golint           # unmaintained, replaced by revive
     - gomnd            # we are using numbers in many cases
@@ -92,28 +102,18 @@ linters:
     - interfacer       # deprecated
     - maligned         # deprecated
     - nlreturn         # too annoying
+    - nosnakecase      # deprecated
     - scopelint        # too many false positives
     - varnamelen       # useless
     - wrapcheck        # we do not use wrapping everywhere
     - wsl              # too annoying
 
     # TODO: carefully review all the rules below and either fix the code
     # or leave disabled and provide a reason why
-    - paralleltest
     - tagliatelle
-    - tparallel
-    - gocritic
-    - godot
-    - godox
     - revive
-    - nosnakecase
     - paralleltest
-    - ireturn
     - gocognit
-    - maintidx
-    - interfacebloat
-    - gosimple
-    - forbidigo
     - errcheck
     # ENDTODO
 
@@ -143,7 +143,9 @@ issues:
         - funlen           # tests may be long
         - gocognit         # triggered by subtests
         - gomnd            # tests are full of magic numbers
+        - ireturn          # we have exceptions, so need to silence them in tests
         - lll              # tests often require long lines
-        - nonamedreturns   # it's not critical for tests, albeit desirable
+        - maintidx         # not critical for tests
+        - nonamedreturns   # not critical for tests, albeit desirable
         - testpackage      # senseless
         - unused           # very annoying false positive: https://github.com/golangci/golangci-lint/issues/791

Makefile

@@ -23,7 +23,7 @@ env-compose-up: env-update-image
 	docker compose up --detach --renew-anon-volumes --remove-orphans
 
 env-devcontainer:
-	docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-managed-server .devcontainer/setup.py
+	docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server .devcontainer/setup.py
 
 env-down:							## Stop devcontainer
 	COMPOSE_PROFILES=$(PROFILES) \
@@ -37,7 +37,7 @@ TARGET ?= _bash
 
 env:								## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-managed-server make $(TARGET)
+	docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server make $(TARGET)
 
 update-dbaas-catalog: 				## Update the DBaaS catalog from the latest production branch (percona-platform).
 	wget https://raw.githubusercontent.com/percona/dbaas-catalog/percona-platform/percona-dbaas-catalog.yaml -O managed/data/crds/olm/percona-dbaas-catalog.yaml

admin/commands/inventory/add_agent_qan_mysql_perfschema_agent.go

@@ -60,6 +60,7 @@ type AddAgentQANMySQLPerfSchemaAgentCommand struct {
 	Password             string            `help:"MySQL password for scraping metrics"`
 	CustomLabels         map[string]string `mapsep:"," help:"Custom user-assigned labels"`
 	SkipConnectionCheck  bool              `help:"Skip connection check"`
+	CommentsParsing      string            `enum:"on,off" default:"off" help:"Enable/disable parsing comments from queries. One of: [on, off]"`
 	MaxQueryLength       int32             `placeholder:"NUMBER" help:"Limit query length in QAN (default: server-defined; -1: no limit)"`
 	DisableQueryExamples bool              `name:"disable-queryexamples" help:"Disable collection of query examples"`
 	TLS                  bool              `help:"Use TLS to connect to the database"`
@@ -94,22 +95,28 @@ func (cmd *AddAgentQANMySQLPerfSchemaAgentCommand) RunCmd() (commands.Result, er
 		}
 	}
 
+	disableCommentsParsing := true
+	if cmd.CommentsParsing == "on" {
+		disableCommentsParsing = false
+	}
+
 	params := &agents.AddQANMySQLPerfSchemaAgentParams{
 		Body: agents.AddQANMySQLPerfSchemaAgentBody{
-			PMMAgentID:           cmd.PMMAgentID,
-			ServiceID:            cmd.ServiceID,
-			Username:             cmd.Username,
-			Password:             cmd.Password,
-			CustomLabels:         customLabels,
-			SkipConnectionCheck:  cmd.SkipConnectionCheck,
-			MaxQueryLength:       cmd.MaxQueryLength,
-			DisableQueryExamples: cmd.DisableQueryExamples,
-			TLS:                  cmd.TLS,
-			TLSSkipVerify:        cmd.TLSSkipVerify,
-			TLSCa:                tlsCa,
-			TLSCert:              tlsCert,
-			TLSKey:               tlsKey,
-			LogLevel:             &cmd.LogLevel,
+			PMMAgentID:             cmd.PMMAgentID,
+			ServiceID:              cmd.ServiceID,
+			Username:               cmd.Username,
+			Password:               cmd.Password,
+			CustomLabels:           customLabels,
+			SkipConnectionCheck:    cmd.SkipConnectionCheck,
+			DisableCommentsParsing: disableCommentsParsing,
+			MaxQueryLength:         cmd.MaxQueryLength,
+			DisableQueryExamples:   cmd.DisableQueryExamples,
+			TLS:                    cmd.TLS,
+			TLSSkipVerify:          cmd.TLSSkipVerify,
+			TLSCa:                  tlsCa,
+			TLSCert:                tlsCert,
+			TLSKey:                 tlsKey,
+			LogLevel:               &cmd.LogLevel,
 		},
 		Context: commands.Ctx,
 	}

admin/commands/inventory/add_agent_qan_mysql_slowlog_agent.go

@@ -72,6 +72,7 @@ type AddAgentQANMySQLSlowlogAgentCommand struct {
 	Password             string            `help:"MySQL password for scraping metrics"`
 	CustomLabels         map[string]string `mapsep:"," help:"Custom user-assigned labels"`
 	SkipConnectionCheck  bool              `help:"Skip connection check"`
+	CommentsParsing      string            `enum:"on,off" default:"off" help:"Enable/disable parsing comments from queries. One of: [on, off]"`
 	MaxQueryLength       int32             `placeholder:"NUMBER" help:"Limit query length in QAN (default: server-defined; -1: no limit)"`
 	DisableQueryExamples bool              `name:"disable-queryexamples" help:"Disable collection of query examples"`
 	MaxSlowlogFileSize   units.Base2Bytes  `name:"size-slow-logs" placeholder:"size" help:"Rotate slow log file at this size (default: 0; 0 or negative value disables rotation). Ex.: 1GiB"`
@@ -107,23 +108,29 @@ func (cmd *AddAgentQANMySQLSlowlogAgentCommand) RunCmd() (commands.Result, error
 		}
 	}
 
+	disableCommentsParsing := true
+	if cmd.CommentsParsing == "on" {
+		disableCommentsParsing = false
+	}
+
 	params := &agents.AddQANMySQLSlowlogAgentParams{
 		Body: agents.AddQANMySQLSlowlogAgentBody{
-			PMMAgentID:           cmd.PMMAgentID,
-			ServiceID:            cmd.ServiceID,
-			Username:             cmd.Username,
-			Password:             cmd.Password,
-			CustomLabels:         customLabels,
-			SkipConnectionCheck:  cmd.SkipConnectionCheck,
-			MaxQueryLength:       cmd.MaxQueryLength,
-			DisableQueryExamples: cmd.DisableQueryExamples,
-			MaxSlowlogFileSize:   strconv.FormatInt(int64(cmd.MaxSlowlogFileSize), 10),
-			TLS:                  cmd.TLS,
-			TLSSkipVerify:        cmd.TLSSkipVerify,
-			TLSCa:                tlsCa,
-			TLSCert:              tlsCert,
-			TLSKey:               tlsKey,
-			LogLevel:             &cmd.LogLevel,
+			PMMAgentID:             cmd.PMMAgentID,
+			ServiceID:              cmd.ServiceID,
+			Username:               cmd.Username,
+			Password:               cmd.Password,
+			CustomLabels:           customLabels,
+			SkipConnectionCheck:    cmd.SkipConnectionCheck,
+			DisableCommentsParsing: disableCommentsParsing,
+			MaxQueryLength:         cmd.MaxQueryLength,
+			DisableQueryExamples:   cmd.DisableQueryExamples,
+			MaxSlowlogFileSize:     strconv.FormatInt(int64(cmd.MaxSlowlogFileSize), 10),
+			TLS:                    cmd.TLS,
+			TLSSkipVerify:          cmd.TLSSkipVerify,
+			TLSCa:                  tlsCa,
+			TLSCert:                tlsCert,
+			TLSKey:                 tlsKey,
+			LogLevel:               &cmd.LogLevel,
 		},
 		Context: commands.Ctx,
 	}