Merge branch 'main' into dependabot/npm_and_yarn/cli-tests/typescript…

…-eslint/eslint-plugin-6.21.0
percona · Feb 7, 2024 · 3d593f8 · 3d593f8
2 parents 6d12d04 + b3e935e
commit 3d593f8
Show file tree

Hide file tree

Showing 4 changed files with 141 additions and 52 deletions.
diff --git a/.whitesource b/.whitesource
diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/url"
 	"os"
 	"os/exec"
@@ -47,9 +48,17 @@ import (
 )
 
 const (
-	defaultClickhouseDatabase       = "pmm"
-	defaultClickhouseAddr           = "127.0.0.1:9000"
-	defaultClickhouseDataSourceAddr = "127.0.0.1:8123"
+	defaultClickhouseDatabase           = "pmm"
+	defaultClickhouseAddr               = "127.0.0.1:9000"
+	defaultClickhouseDataSourceAddr     = "127.0.0.1:8123"
+	defaultVMSearchMaxQueryLen          = "1MB"
+	defaultVMSearchLatencyOffset        = "5s"
+	defaultVMSearchMaxUniqueTimeseries  = "100000000"
+	defaultVMSearchMaxSamplesPerQuery   = "1500000000"
+	defaultVMSearchMaxQueueDuration     = "30s"
+	defaultVMSearchMaxQueryDuration     = "90s"
+	defaultVMSearchLogSlowQueryDuration = "30s"
+	defaultVMPromscrapeStreamParse      = "true"
 )
 
 // Service is responsible for interactions with Supervisord via supervisorctl.
@@ -255,7 +264,7 @@ func (s *Service) StartUpdate() (uint32, error) {
 
 	// remove existing log file
 	err := os.Remove(pmmUpdatePerformLog)
-	if err != nil && os.IsNotExist(err) {
+	if err != nil && errors.Is(err, fs.ErrNotExist) {
 		err = nil
 	}
 	if err != nil {
@@ -290,7 +299,7 @@ func (s *Service) StartUpdate() (uint32, error) {
 			s.l.Warnf("Unexpected log file size: %+v", fi)
 		}
 		offset = uint32(fi.Size())
-	case os.IsNotExist(err):
+	case errors.Is(err, fs.ErrNotExist):
 		// that's expected as we remove this file above
 	default:
 		s.l.Warn(err)
@@ -419,23 +428,40 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin
 	clickhousePoolSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE", "")
 	clickhouseBlockSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE", "")
 	clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2)
+	vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.VictoriaMetrics.CacheEnabled))
+	vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen)
+	vmSearchLatencyOffset := getValueFromENV("VM_search_latencyOffset", defaultVMSearchLatencyOffset)
+	vmSearchMaxUniqueTimeseries := getValueFromENV("VM_search_maxUniqueTimeseries", defaultVMSearchMaxUniqueTimeseries)
+	vmSearchMaxSamplesPerQuery := getValueFromENV("VM_search_maxSamplesPerQuery", defaultVMSearchMaxSamplesPerQuery)
+	vmSearchMaxQueueDuration := getValueFromENV("VM_search_maxQueueDuration", defaultVMSearchMaxQueueDuration)
+	vmSearchMaxQueryDuration := getValueFromENV("VM_search_maxQueryDuration", defaultVMSearchMaxQueryDuration)
+	vmSearchLogSlowQueryDuration := getValueFromENV("VM_search_logSlowQueryDuration", defaultVMSearchLogSlowQueryDuration)
+	vmPromscrapeStreamParse := getValueFromENV("VM_promscrape_streamParse", defaultVMPromscrapeStreamParse)
 
 	templateParams := map[string]interface{}{
-		"DataRetentionHours":       int(settings.DataRetention.Hours()),
-		"DataRetentionDays":        int(settings.DataRetention.Hours() / 24),
-		"VMAlertFlags":             s.vmParams.VMAlertFlags,
-		"VMDBCacheDisable":         !settings.VictoriaMetrics.CacheEnabled,
-		"VMURL":                    s.vmParams.URL(),
-		"ExternalVM":               s.vmParams.ExternalVM(),
-		"PerconaTestDbaas":         settings.DBaaS.Enabled,
-		"InterfaceToBind":          envvars.GetInterfaceToBind(),
-		"ClickhouseAddr":           clickhouseAddr,
-		"ClickhouseDataSourceAddr": clickhouseDataSourceAddr,
-		"ClickhouseDatabase":       clickhouseDatabase,
-		"ClickhousePoolSize":       clickhousePoolSize,
-		"ClickhouseBlockSize":      clickhouseBlockSize,
-		"ClickhouseHost":           clickhouseAddrPair[0],
-		"ClickhousePort":           clickhouseAddrPair[1],
+		"DataRetentionHours":           int(settings.DataRetention.Hours()),
+		"DataRetentionDays":            int(settings.DataRetention.Hours() / 24),
+		"VMAlertFlags":                 s.vmParams.VMAlertFlags,
+		"VMSearchDisableCache":         vmSearchDisableCache,
+		"VMSearchMaxQueryLen":          vmSearchMaxQueryLen,
+		"VMSearchLatencyOffset":        vmSearchLatencyOffset,
+		"VMSearchMaxUniqueTimeseries":  vmSearchMaxUniqueTimeseries,
+		"VMSearchMaxSamplesPerQuery":   vmSearchMaxSamplesPerQuery,
+		"VMSearchMaxQueueDuration":     vmSearchMaxQueueDuration,
+		"VMSearchMaxQueryDuration":     vmSearchMaxQueryDuration,
+		"VMSearchLogSlowQueryDuration": vmSearchLogSlowQueryDuration,
+		"VMPromscrapeStreamParse":      vmPromscrapeStreamParse,
+		"VMURL":                        s.vmParams.URL(),
+		"ExternalVM":                   s.vmParams.ExternalVM(),
+		"PerconaTestDbaas":             settings.DBaaS.Enabled,
+		"InterfaceToBind":              envvars.GetInterfaceToBind(),
+		"ClickhouseAddr":               clickhouseAddr,
+		"ClickhouseDataSourceAddr":     clickhouseDataSourceAddr,
+		"ClickhouseDatabase":           clickhouseDatabase,
+		"ClickhousePoolSize":           clickhousePoolSize,
+		"ClickhouseBlockSize":          clickhouseBlockSize,
+		"ClickhouseHost":               clickhouseAddrPair[0],
+		"ClickhousePort":               clickhouseAddrPair[1],
 	}
 
 	s.addPostgresParams(templateParams)
@@ -547,7 +573,7 @@ func (s *Service) saveConfigAndReload(name string, cfg []byte) (bool, error) {
 	// read existing content
 	path := filepath.Join(s.configDir, name+".ini")
 	oldCfg, err := os.ReadFile(path) //nolint:gosec
-	if os.IsNotExist(err) {
+	if errors.Is(err, fs.ErrNotExist) {
 		err = nil
 	}
 	if err != nil {
@@ -685,15 +711,15 @@ command =
 		--retentionPeriod={{ .DataRetentionDays }}d
 		--storageDataPath=/srv/victoriametrics/data
 		--httpListenAddr={{ .InterfaceToBind }}:9090
-		--search.disableCache={{ .VMDBCacheDisable }}
-		--search.maxQueryLen=1MB
-		--search.latencyOffset=5s
-		--search.maxUniqueTimeseries=100000000
-		--search.maxSamplesPerQuery=1500000000
-		--search.maxQueueDuration=30s
-		--search.logSlowQueryDuration=30s
-		--search.maxQueryDuration=90s
-		--promscrape.streamParse=true
+		--search.disableCache={{ .VMSearchDisableCache }}
+		--search.maxQueryLen={{ .VMSearchMaxQueryLen }}
+		--search.latencyOffset={{ .VMSearchLatencyOffset }}
+		--search.maxUniqueTimeseries={{ .VMSearchMaxUniqueTimeseries }}
+		--search.maxSamplesPerQuery={{ .VMSearchMaxSamplesPerQuery }}
+		--search.maxQueueDuration={{ .VMSearchMaxQueueDuration }}
+		--search.logSlowQueryDuration={{ .VMSearchLogSlowQueryDuration }}
+		--search.maxQueryDuration={{ .VMSearchMaxQueryDuration }}
+		--promscrape.streamParse={{ .VMPromscrapeStreamParse }}
 		--prometheusDataPath=/srv/prometheus/data
 		--http.pathPrefix=/prometheus
 		--envflag.enable

diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go
@@ -30,9 +30,10 @@ import (
 	"github.com/percona/pmm/managed/models"
 )
 
+const gRPCMessageMaxSize = uint32(100 * 1024 * 1024)
+
 func TestConfig(t *testing.T) {
 	t.Parallel()
-	gRPCMessageMaxSize := uint32(100 * 1024 * 1024)
 
 	pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs"))
 	configDir := filepath.Join("..", "..", "testdata", "supervisord.d")
@@ -73,9 +74,57 @@ func TestConfig(t *testing.T) {
 	}
 }
 
+func TestConfigVictoriaMetricsEnvvars(t *testing.T) {
+	pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs"))
+	configDir := filepath.Join("..", "..", "testdata", "supervisord.d")
+	vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL)
+	require.NoError(t, err)
+	pgParams := &models.PGParams{
+		Addr:        "127.0.0.1:5432",
+		DBName:      "postgres",
+		DBUsername:  "db_username",
+		DBPassword:  "db_password",
+		SSLMode:     "verify",
+		SSLCAPath:   "path-to-CA-cert",
+		SSLKeyPath:  "path-to-key",
+		SSLCertPath: "path-to-cert",
+	}
+	s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: pgParams, HAParams: &models.HAParams{}}, gRPCMessageMaxSize)
+	settings := &models.Settings{
+		DataRetention:   30 * 24 * time.Hour,
+		AlertManagerURL: "https://external-user:passw!,ord@external-alertmanager:6443/alerts",
+	}
+	settings.VictoriaMetrics.CacheEnabled = false
+
+	// Test environment variables being passed to VictoriaMetrics.
+	t.Setenv("VM_search_maxQueryLen", "2MB")
+	t.Setenv("VM_search_latencyOffset", "10s")
+	t.Setenv("VM_search_maxUniqueTimeseries", "500000000")
+	t.Setenv("VM_search_maxSamplesPerQuery", "1600000000")
+	t.Setenv("VM_search_maxQueueDuration", "100s")
+	t.Setenv("VM_search_logSlowQueryDuration", "300s")
+	t.Setenv("VM_search_maxQueryDuration", "9s")
+	t.Setenv("VM_promscrape_streamParse", "false")
+
+	for _, tmpl := range templates.Templates() {
+		n := tmpl.Name()
+		if n != "victoriametrics" { // just test the VM template
+			continue
+		}
+
+		tmpl := tmpl
+		t.Run(tmpl.Name(), func(t *testing.T) {
+			expected, err := os.ReadFile(filepath.Join(configDir, tmpl.Name()+"_envvars.ini")) //nolint:gosec
+			require.NoError(t, err)
+			actual, err := s.marshalConfig(tmpl, settings, nil)
+			require.NoError(t, err)
+			assert.Equal(t, string(expected), string(actual))
+		})
+	}
+}
+
 func TestDBaaSController(t *testing.T) {
 	t.Parallel()
-	gRPCMessageMaxSize := uint32(100 * 1024 * 1024)
 
 	pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs"))
 	configDir := filepath.Join("..", "..", "testdata", "supervisord.d")

diff --git a/managed/testdata/supervisord.d/victoriametrics_envvars.ini b/managed/testdata/supervisord.d/victoriametrics_envvars.ini
@@ -0,0 +1,34 @@
+; Managed by pmm-managed. DO NOT EDIT.
+
+[program:victoriametrics]
+priority = 7
+command =
+	/usr/sbin/victoriametrics
+		--promscrape.config=/etc/victoriametrics-promscrape.yml
+		--retentionPeriod=30d
+		--storageDataPath=/srv/victoriametrics/data
+		--httpListenAddr=127.0.0.1:9090
+		--search.disableCache=true
+		--search.maxQueryLen=2MB
+		--search.latencyOffset=10s
+		--search.maxUniqueTimeseries=500000000
+		--search.maxSamplesPerQuery=1600000000
+		--search.maxQueueDuration=100s
+		--search.logSlowQueryDuration=300s
+		--search.maxQueryDuration=9s
+		--promscrape.streamParse=false
+		--prometheusDataPath=/srv/prometheus/data
+		--http.pathPrefix=/prometheus
+		--envflag.enable
+		--envflag.prefix=VM_
+user = pmm
+autorestart = true
+autostart = true
+startretries = 10
+startsecs = 1
+stopsignal = INT
+stopwaitsecs = 300
+stdout_logfile = /srv/logs/victoriametrics.log
+stdout_logfile_maxbytes = 10MB
+stdout_logfile_backups = 3
+redirect_stderr = true