From f0176ed54871738c778e14fd7ddc94cc10d6b937 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 6 Feb 2024 13:59:50 +0300 Subject: [PATCH 1/2] PMM-7 remove .whitesource config (#2813) * PMM-7 remove .whitesource config * PMM-7 remove the config file --- .whitesource | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .whitesource diff --git a/.whitesource b/.whitesource deleted file mode 100644 index 541b4c8fbf..0000000000 --- a/.whitesource +++ /dev/null @@ -1,20 +0,0 @@ -{ - "scanSettings": { - "configMode": "AUTO", - "configExternalURL": "", - "projectToken": "", - "baseBranches": [], - "enableLicenseViolations": true - }, - "checkRunSettings": { - "vulnerableCheckRunConclusionLevel": "failure", - "displayMode": "diff" - }, - "issueSettings": { - "minSeverityLevel": "NONE" - }, - "remediateSettings": { - "enableRenovate": false, - "dependencyDashboard": false - } -} From b3e935e94cf0206e9a031d373f3334eb80967044 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 6 Feb 2024 14:55:47 +0300 Subject: [PATCH 2/2] PMM-12806 VictoriaMetrics to give preference to environment vars (#2793) * PMM-12806 VictoriaMetrics to give preference to enviroment vars * PMM-12806 supress linter errors * PMM-12806 test VictoriaMetrics template separately * PMM-12806 fix a typo * PMM-12806 add more variables to test --- managed/services/supervisord/supervisord.go | 86 ++++++++++++------- .../services/supervisord/supervisord_test.go | 53 +++++++++++- .../supervisord.d/victoriametrics_envvars.ini | 34 ++++++++ 3 files changed, 141 insertions(+), 32 deletions(-) create mode 100644 managed/testdata/supervisord.d/victoriametrics_envvars.ini diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 33826cd245..fa3d564f3d 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "io" + "io/fs" "net/url" "os" "os/exec" @@ -47,9 +48,17 @@ import ( ) const ( - defaultClickhouseDatabase = "pmm" - defaultClickhouseAddr = "127.0.0.1:9000" - defaultClickhouseDataSourceAddr = "127.0.0.1:8123" + defaultClickhouseDatabase = "pmm" + defaultClickhouseAddr = "127.0.0.1:9000" + defaultClickhouseDataSourceAddr = "127.0.0.1:8123" + defaultVMSearchMaxQueryLen = "1MB" + defaultVMSearchLatencyOffset = "5s" + defaultVMSearchMaxUniqueTimeseries = "100000000" + defaultVMSearchMaxSamplesPerQuery = "1500000000" + defaultVMSearchMaxQueueDuration = "30s" + defaultVMSearchMaxQueryDuration = "90s" + defaultVMSearchLogSlowQueryDuration = "30s" + defaultVMPromscrapeStreamParse = "true" ) // Service is responsible for interactions with Supervisord via supervisorctl. @@ -255,7 +264,7 @@ func (s *Service) StartUpdate() (uint32, error) { // remove existing log file err := os.Remove(pmmUpdatePerformLog) - if err != nil && os.IsNotExist(err) { + if err != nil && errors.Is(err, fs.ErrNotExist) { err = nil } if err != nil { @@ -290,7 +299,7 @@ func (s *Service) StartUpdate() (uint32, error) { s.l.Warnf("Unexpected log file size: %+v", fi) } offset = uint32(fi.Size()) - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): // that's expected as we remove this file above default: s.l.Warn(err) @@ -419,23 +428,40 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin clickhousePoolSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE", "") clickhouseBlockSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE", "") clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2) + vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.VictoriaMetrics.CacheEnabled)) + vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen) + vmSearchLatencyOffset := getValueFromENV("VM_search_latencyOffset", defaultVMSearchLatencyOffset) + vmSearchMaxUniqueTimeseries := getValueFromENV("VM_search_maxUniqueTimeseries", defaultVMSearchMaxUniqueTimeseries) + vmSearchMaxSamplesPerQuery := getValueFromENV("VM_search_maxSamplesPerQuery", defaultVMSearchMaxSamplesPerQuery) + vmSearchMaxQueueDuration := getValueFromENV("VM_search_maxQueueDuration", defaultVMSearchMaxQueueDuration) + vmSearchMaxQueryDuration := getValueFromENV("VM_search_maxQueryDuration", defaultVMSearchMaxQueryDuration) + vmSearchLogSlowQueryDuration := getValueFromENV("VM_search_logSlowQueryDuration", defaultVMSearchLogSlowQueryDuration) + vmPromscrapeStreamParse := getValueFromENV("VM_promscrape_streamParse", defaultVMPromscrapeStreamParse) templateParams := map[string]interface{}{ - "DataRetentionHours": int(settings.DataRetention.Hours()), - "DataRetentionDays": int(settings.DataRetention.Hours() / 24), - "VMAlertFlags": s.vmParams.VMAlertFlags, - "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, - "VMURL": s.vmParams.URL(), - "ExternalVM": s.vmParams.ExternalVM(), - "PerconaTestDbaas": settings.DBaaS.Enabled, - "InterfaceToBind": envvars.GetInterfaceToBind(), - "ClickhouseAddr": clickhouseAddr, - "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, - "ClickhouseDatabase": clickhouseDatabase, - "ClickhousePoolSize": clickhousePoolSize, - "ClickhouseBlockSize": clickhouseBlockSize, - "ClickhouseHost": clickhouseAddrPair[0], - "ClickhousePort": clickhouseAddrPair[1], + "DataRetentionHours": int(settings.DataRetention.Hours()), + "DataRetentionDays": int(settings.DataRetention.Hours() / 24), + "VMAlertFlags": s.vmParams.VMAlertFlags, + "VMSearchDisableCache": vmSearchDisableCache, + "VMSearchMaxQueryLen": vmSearchMaxQueryLen, + "VMSearchLatencyOffset": vmSearchLatencyOffset, + "VMSearchMaxUniqueTimeseries": vmSearchMaxUniqueTimeseries, + "VMSearchMaxSamplesPerQuery": vmSearchMaxSamplesPerQuery, + "VMSearchMaxQueueDuration": vmSearchMaxQueueDuration, + "VMSearchMaxQueryDuration": vmSearchMaxQueryDuration, + "VMSearchLogSlowQueryDuration": vmSearchLogSlowQueryDuration, + "VMPromscrapeStreamParse": vmPromscrapeStreamParse, + "VMURL": s.vmParams.URL(), + "ExternalVM": s.vmParams.ExternalVM(), + "PerconaTestDbaas": settings.DBaaS.Enabled, + "InterfaceToBind": envvars.GetInterfaceToBind(), + "ClickhouseAddr": clickhouseAddr, + "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, + "ClickhouseDatabase": clickhouseDatabase, + "ClickhousePoolSize": clickhousePoolSize, + "ClickhouseBlockSize": clickhouseBlockSize, + "ClickhouseHost": clickhouseAddrPair[0], + "ClickhousePort": clickhouseAddrPair[1], } s.addPostgresParams(templateParams) @@ -547,7 +573,7 @@ func (s *Service) saveConfigAndReload(name string, cfg []byte) (bool, error) { // read existing content path := filepath.Join(s.configDir, name+".ini") oldCfg, err := os.ReadFile(path) //nolint:gosec - if os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { err = nil } if err != nil { @@ -685,15 +711,15 @@ command = --retentionPeriod={{ .DataRetentionDays }}d --storageDataPath=/srv/victoriametrics/data --httpListenAddr={{ .InterfaceToBind }}:9090 - --search.disableCache={{ .VMDBCacheDisable }} - --search.maxQueryLen=1MB - --search.latencyOffset=5s - --search.maxUniqueTimeseries=100000000 - --search.maxSamplesPerQuery=1500000000 - --search.maxQueueDuration=30s - --search.logSlowQueryDuration=30s - --search.maxQueryDuration=90s - --promscrape.streamParse=true + --search.disableCache={{ .VMSearchDisableCache }} + --search.maxQueryLen={{ .VMSearchMaxQueryLen }} + --search.latencyOffset={{ .VMSearchLatencyOffset }} + --search.maxUniqueTimeseries={{ .VMSearchMaxUniqueTimeseries }} + --search.maxSamplesPerQuery={{ .VMSearchMaxSamplesPerQuery }} + --search.maxQueueDuration={{ .VMSearchMaxQueueDuration }} + --search.logSlowQueryDuration={{ .VMSearchLogSlowQueryDuration }} + --search.maxQueryDuration={{ .VMSearchMaxQueryDuration }} + --promscrape.streamParse={{ .VMPromscrapeStreamParse }} --prometheusDataPath=/srv/prometheus/data --http.pathPrefix=/prometheus --envflag.enable diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index 343d818db4..17678ca96f 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -30,9 +30,10 @@ import ( "github.com/percona/pmm/managed/models" ) +const gRPCMessageMaxSize = uint32(100 * 1024 * 1024) + func TestConfig(t *testing.T) { t.Parallel() - gRPCMessageMaxSize := uint32(100 * 1024 * 1024) pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") @@ -73,9 +74,57 @@ func TestConfig(t *testing.T) { } } +func TestConfigVictoriaMetricsEnvvars(t *testing.T) { + pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) + configDir := filepath.Join("..", "..", "testdata", "supervisord.d") + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + pgParams := &models.PGParams{ + Addr: "127.0.0.1:5432", + DBName: "postgres", + DBUsername: "db_username", + DBPassword: "db_password", + SSLMode: "verify", + SSLCAPath: "path-to-CA-cert", + SSLKeyPath: "path-to-key", + SSLCertPath: "path-to-cert", + } + s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: pgParams, HAParams: &models.HAParams{}}, gRPCMessageMaxSize) + settings := &models.Settings{ + DataRetention: 30 * 24 * time.Hour, + AlertManagerURL: "https://external-user:passw!,ord@external-alertmanager:6443/alerts", + } + settings.VictoriaMetrics.CacheEnabled = false + + // Test environment variables being passed to VictoriaMetrics. + t.Setenv("VM_search_maxQueryLen", "2MB") + t.Setenv("VM_search_latencyOffset", "10s") + t.Setenv("VM_search_maxUniqueTimeseries", "500000000") + t.Setenv("VM_search_maxSamplesPerQuery", "1600000000") + t.Setenv("VM_search_maxQueueDuration", "100s") + t.Setenv("VM_search_logSlowQueryDuration", "300s") + t.Setenv("VM_search_maxQueryDuration", "9s") + t.Setenv("VM_promscrape_streamParse", "false") + + for _, tmpl := range templates.Templates() { + n := tmpl.Name() + if n != "victoriametrics" { // just test the VM template + continue + } + + tmpl := tmpl + t.Run(tmpl.Name(), func(t *testing.T) { + expected, err := os.ReadFile(filepath.Join(configDir, tmpl.Name()+"_envvars.ini")) //nolint:gosec + require.NoError(t, err) + actual, err := s.marshalConfig(tmpl, settings, nil) + require.NoError(t, err) + assert.Equal(t, string(expected), string(actual)) + }) + } +} + func TestDBaaSController(t *testing.T) { t.Parallel() - gRPCMessageMaxSize := uint32(100 * 1024 * 1024) pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") diff --git a/managed/testdata/supervisord.d/victoriametrics_envvars.ini b/managed/testdata/supervisord.d/victoriametrics_envvars.ini new file mode 100644 index 0000000000..78c56d93ed --- /dev/null +++ b/managed/testdata/supervisord.d/victoriametrics_envvars.ini @@ -0,0 +1,34 @@ +; Managed by pmm-managed. DO NOT EDIT. + +[program:victoriametrics] +priority = 7 +command = + /usr/sbin/victoriametrics + --promscrape.config=/etc/victoriametrics-promscrape.yml + --retentionPeriod=30d + --storageDataPath=/srv/victoriametrics/data + --httpListenAddr=127.0.0.1:9090 + --search.disableCache=true + --search.maxQueryLen=2MB + --search.latencyOffset=10s + --search.maxUniqueTimeseries=500000000 + --search.maxSamplesPerQuery=1600000000 + --search.maxQueueDuration=100s + --search.logSlowQueryDuration=300s + --search.maxQueryDuration=9s + --promscrape.streamParse=false + --prometheusDataPath=/srv/prometheus/data + --http.pathPrefix=/prometheus + --envflag.enable + --envflag.prefix=VM_ +user = pmm +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = INT +stopwaitsecs = 300 +stdout_logfile = /srv/logs/victoriametrics.log +stdout_logfile_maxbytes = 10MB +stdout_logfile_backups = 3 +redirect_stderr = true