From d5fe25c60fb78caca9158ad3c386eeb3ebf8bb8e Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 2 Mar 2023 01:00:22 +0300 Subject: [PATCH 01/21] PMM-9374 support of external victoria metrics. --- .../rpm/server/SPECS/victoriametrics.spec | 8 +- docker-compose.yml | 36 ++++++++- managed/cmd/pmm-managed/main.go | 27 ++++--- ...iametrics.go => victoriametrics_params.go} | 18 ++++- ...test.go => victoriametrics_params_test.go} | 4 +- managed/services/agents/state.go | 18 +++-- managed/services/agents/vmagent.go | 28 +++++-- managed/services/config/pmm-managed.yaml | 1 - managed/services/supervisord/supervisord.go | 34 +++++++- .../victoriametrics/victoriametrics.go | 77 +++++++++++++------ .../victoriametrics/victoriametrics_test.go | 6 +- managed/utils/envvars/parser.go | 6 ++ update/ansible/playbook/tasks/files/pmm.ini | 1 - 13 files changed, 201 insertions(+), 63 deletions(-) rename managed/models/{victoriametrics.go => victoriametrics_params.go} (81%) rename managed/models/{victoriametrics_test.go => victoriametrics_params_test.go} (90%) diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index 84f2b0a2c2..f8ebcb5366 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -35,11 +35,13 @@ export USER=builder make victoria-metrics-pure make vmalert-pure +make vmagent-pure %install install -D -p -m 0755 ./bin/victoria-metrics-pure %{buildroot}%{_sbindir}/victoriametrics install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert +install -D -p -m 0755 ./bin/vmagent-pure %{buildroot}%{_sbindir}/vmagent %files @@ -47,10 +49,14 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert %doc README.md %{_sbindir}/victoriametrics %{_sbindir}/vmalert +%{_sbindir}/vmagent %changelog -* Thu Oct 20 2022 Michal Kralik - 1.82.1 +* Thu Mar 2 2023 Nurlan Moldomurov - 1.82.1-2 +- add vmagent to PMM Server + +* Thu Oct 20 2022 Michal Kralik - 1.82.1-1 - upgrade victoriametrics to 1.82.1 release * Thu May 11 2022 Michael Okoko - 1.77.1 diff --git a/docker-compose.yml b/docker-compose.yml index ba9d0a0e99..a07fe19f11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,20 +73,46 @@ services: - go-modules:/root/go/pkg/mod - root-cache:/root/.cache - # PMM with external ClickHouse DB + # PMM with external DBs ch: profiles: - - pmm-ch + - pmm-external-dbs image: ${CH_IMAGE:-clickhouse/clickhouse-server:22.6.9.11-alpine} platform: linux/amd64 hostname: ${CH_HOSTNAME:-ch} ports: - ${CH_PORT:-9000}:9000 + networks: + - ${NETWORK:-default} + victoriametrics: + profiles: + - pmm-external-dbs + hostname: ${VM_HOSTNAME:-victoriametrics} + image: victoriametrics/victoria-metrics:v1.88.1 + ports: + - 8428:8428 + - 8089:8089 + - 8089:8089/udp + - 2003:2003 + - 2003:2003/udp + - 4242:4242 + volumes: + - vmdata:/storage + command: + - "--storageDataPath=/storage" + - "--graphiteListenAddr=:2003" + - "--opentsdbListenAddr=:4242" + - "--httpListenAddr=:8428" + - "--influxListenAddr=:8089" + - "--vmalert.proxyURL=http://vmalert:8880" + networks: + - ${NETWORK:-default} pmm-managed-server-ch: profiles: - - pmm-ch + - pmm-external-dbs depends_on: - ch + - victoriametrics image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-managed-server hostname: pmm-managed-server @@ -109,7 +135,8 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 -# - PMM_DEBUG=1 + - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} + - PMM_DEBUG=1 - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest extra_hosts: @@ -151,6 +178,7 @@ services: volumes: go-modules: + vmdata: {} root-cache: networks: diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index fae384b25e..a8f1727767 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -646,9 +646,9 @@ func main() { kingpin.Version(version.FullInfo()) kingpin.HelpFlag.Short('h') - victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL"). - Default("http://127.0.0.1:9090/prometheus/").String() - victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL"). + victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL").Envar("PMM_VM_URL"). + Default(models.VMBaseURL).String() + victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL").Envar("PMM_VM_ALERT_URL"). Default("http://127.0.0.1:8880/").String() victoriaMetricsConfigF := kingpin.Flag("victoriametrics-config", "VictoriaMetrics scrape configuration file path"). Default("/etc/victoriametrics-promscrape.yml").String() @@ -699,8 +699,8 @@ func main() { l.Panicf("Failed to load config: %+v", err) } ds := cfg.Config.Services.Telemetry.DataSources - pmmdb := ds.PmmDBSelect + pmmdb := ds.PmmDBSelect pmmdb.Credentials.Username = *postgresDBUsernameF pmmdb.Credentials.Password = *postgresDBPasswordF pmmdb.DSN.Scheme = "postgres" // TODO: should be configurable @@ -711,9 +711,17 @@ func main() { pmmdb.DSN.Params = q.Encode() clickhousedb := ds.QanDBSelect - clickhousedb.DSN = "tcp://" + *clickhouseAddrF + "/" + *clickHouseDatabaseF + ds.VM.Address = *victoriaMetricsURLF + + vmParams, err := models.NewVictoriaMetricsParams( + models.BasePrometheusConfigPath, + *victoriaMetricsURLF) + if err != nil { + l.Panicf("cannot load victoriametrics params problem: %+v", err) + } + sqlDB, err := models.OpenDB(*postgresAddrF, *postgresDBNameF, *postgresDBUsernameF, *postgresDBPasswordF) if err != nil { l.Panicf("Failed to connect to database: %+v", err) @@ -735,12 +743,7 @@ func main() { cleaner := clean.New(db) externalRules := vmalert.NewExternalRules() - - vmParams, err := models.NewVictoriaMetricsParams(victoriametrics.BasePrometheusConfigPath) - if err != nil { - l.Panicf("cannot load victoriametrics params problem: %+v", err) - } - vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, *victoriaMetricsURLF, vmParams) + vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, vmParams) if err != nil { l.Panicf("VictoriaMetrics service problem: %+v", err) } @@ -792,7 +795,7 @@ func main() { prom.MustRegister(grafanaClient) jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService) - agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb) + agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams) agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) actionsService := agents.NewActionsService(qanClient, agentsRegistry) diff --git a/managed/models/victoriametrics.go b/managed/models/victoriametrics_params.go similarity index 81% rename from managed/models/victoriametrics.go rename to managed/models/victoriametrics_params.go index d8b07caf27..8cda5ee576 100644 --- a/managed/models/victoriametrics.go +++ b/managed/models/victoriametrics_params.go @@ -23,18 +23,28 @@ import ( "gopkg.in/yaml.v3" ) +const ( + // BasePrometheusConfigPath - basic path with prometheus config, + // that user can mount to container. + BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml" + VMBaseURL = "http://127.0.0.1:9090/prometheus/" +) + // VictoriaMetricsParams - defines flags and settings for victoriametrics. type VictoriaMetricsParams struct { // VMAlertFlags additional flags for VMAlert. VMAlertFlags []string // BaseConfigPath defines path for basic prometheus config. BaseConfigPath string + // URL defines URL of Victoria Metrics + URL string } // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. -func NewVictoriaMetricsParams(basePath string) (*VictoriaMetricsParams, error) { +func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, + URL: vmURL, } if err := vmp.UpdateParams(); err != nil { return vmp, err @@ -59,7 +69,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { if !os.IsNotExist(err) { return errors.Wrap(err, "cannot read baseConfigPath for VMAlertParams") } - // fast return if users configuration doesn't exists with path + // fast return if users configuration doesn't exist with path // /srv/prometheus/prometheus.base.yml, // its maybe mounted into container by user. return nil @@ -79,3 +89,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { return nil } + +func (vmp *VictoriaMetricsParams) ExternalVM() bool { + return vmp.URL != VMBaseURL +} diff --git a/managed/models/victoriametrics_test.go b/managed/models/victoriametrics_params_test.go similarity index 90% rename from managed/models/victoriametrics_test.go rename to managed/models/victoriametrics_params_test.go index 02ab59b036..c52c349800 100644 --- a/managed/models/victoriametrics_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -23,11 +23,11 @@ import ( func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "", "", "") require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "", "", "") require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index 9ffddde95a..b784b2d540 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -41,17 +41,19 @@ const ( // StateUpdater handles updating status of agents. type StateUpdater struct { - db *reform.DB - r *Registry - vmdb prometheusService + db *reform.DB + r *Registry + vmdb prometheusService + vmParams *models.VictoriaMetricsParams } // NewStateUpdater creates new agent state updater. -func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService) *StateUpdater { +func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams *models.VictoriaMetricsParams) *StateUpdater { return &StateUpdater{ - db: db, - r: r, - vmdb: vmdb, + db: db, + r: r, + vmdb: vmdb, + vmParams: vmParams, } } @@ -181,7 +183,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI if err != nil { return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.id) } - agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg)) + agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg), u.vmParams) case models.NodeExporterType: node, err := models.FindNodeByID(u.db.Querier, pointer.GetString(row.NodeID)) diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index cb04ce25e7..7688ecd472 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -16,16 +16,25 @@ package agents import ( + "fmt" + "github.com/percona/pmm/managed/utils/envvars" + "os" "sort" + "strings" "github.com/percona/pmm/api/agentpb" "github.com/percona/pmm/api/inventorypb" + "github.com/percona/pmm/managed/models" ) // vmAgentConfig returns desired configuration of vmagent process. -func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { +func vmAgentConfig(scrapeCfg string, params *models.VictoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { + serverURL := "{{.server_url}}/victoriametrics/" + if params.ExternalVM() { + serverURL = params.URL + } args := []string{ - "-remoteWrite.url={{.server_url}}/victoriametrics/api/v1/write", + fmt.Sprintf("-remoteWrite.url=%sapi/v1/write", serverURL), "-remoteWrite.tlsInsecureSkipVerify={{.server_insecure}}", "-remoteWrite.tmpDataPath={{.tmp_dir}}/vmagent-temp-dir", "-promscrape.config={{.TextFiles.vmagentscrapecfg}}", @@ -35,13 +44,22 @@ func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { "-httpListenAddr=127.0.0.1:{{.listen_port}}", // needed for login/password at client side. "-envflag.enable=true", + "-envflag.prefix=VMAGENT_", } sort.Strings(args) - envs := []string{ - "remoteWrite_basicAuth_username={{.server_username}}", - "remoteWrite_basicAuth_password={{.server_password}}", + var envs []string + if !params.ExternalVM() { + envs = []string{ + "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}", + "VMAGENT_remoteWrite_basicAuth_password={{.server_password}}", + } + } + for _, env := range os.Environ() { + if strings.HasPrefix(env, envvars.ENVvmAgentPrefix) { + envs = append(envs, env) + } } sort.Strings(envs) diff --git a/managed/services/config/pmm-managed.yaml b/managed/services/config/pmm-managed.yaml index 8068096020..0ad29456b9 100644 --- a/managed/services/config/pmm-managed.yaml +++ b/managed/services/config/pmm-managed.yaml @@ -5,7 +5,6 @@ services: VM: enabled: true timeout: 2s - address: http://localhost:9090/prometheus QANDB_SELECT: enabled: true timeout: 2s diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 87a486becb..afae6f53b8 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -428,6 +428,8 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "DataRetentionDays": int(settings.DataRetention.Hours() / 24), "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, + "VMURL": s.vmParams.URL, + "EnableVMAgent": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, "ClickhouseAddr": clickhouseAddr, "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, @@ -613,7 +615,36 @@ stdout_logfile_backups = 3 redirect_stderr = true {{end}} +{{define "vmagent"}} +{{- if .EnableVMAgent }} +[program:vmagent] +priority = 7 +command = + /usr/sbin/vmagent + -remoteWrite.url={{ .VMURL }}api/v1/write + -remoteWrite.maxDiskUsagePerURL=1073741824 + -remoteWrite.tmpDataPath=/srv/vmagent/data + -promscrape.config=/etc/victoriametrics-promscrape.yml + -httpListenAddr=127.0.0.1:9091 + -loggerLevel=INFO + -envflag.enable + -envflag.prefix=VMAGENT_ +user = pmm +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = INT +stopwaitsecs = 300 +stdout_logfile = /srv/logs/vmagent.log +stdout_logfile_maxbytes = 10MB +stdout_logfile_backups = 3 +redirect_stderr = true +{{end}} +{{end}} + {{define "victoriametrics"}} +{{- if not .EnableVMAgent }} [program:victoriametrics] priority = 7 command = @@ -647,6 +678,7 @@ stdout_logfile_maxbytes = 10MB stdout_logfile_backups = 3 redirect_stderr = true {{end}} +{{end}} {{define "vmalert"}} [program:vmalert] @@ -685,7 +717,7 @@ redirect_stderr = true priority = 9 command = /usr/sbin/vmproxy - --target-url=http://127.0.0.1:9090/ + --target-url={{ .VMURL }} --listen-port=8430 --listen-address=127.0.0.1 --header-name=X-Proxy-Filter diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 1456d2cbad..43b536287f 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -46,10 +46,7 @@ const ( updateBatchDelay = time.Second configurationUpdateTimeout = 3 * time.Second - // BasePrometheusConfigPath - basic path with prometheus config, - // that user can mount to container. - BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml" - + vmagentDir = "/srv/vmagent" victoriametricsDir = "/srv/victoriametrics" victoriametricsDataDir = "/srv/victoriametrics/data" dirPerm = os.FileMode(0o775) @@ -64,15 +61,19 @@ type Service struct { baseURL *url.URL client *http.Client - baseConfigPath string // for testing + params *models.VictoriaMetricsParams l *logrus.Entry reloadCh chan struct{} } // NewVictoriaMetrics creates new VictoriaMetrics service. -func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, baseURL string, params *models.VictoriaMetricsParams) (*Service, error) { - u, err := url.Parse(baseURL) +func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { + internalVMURL := params.URL + if params.ExternalVM() { + internalVMURL = "http://127.0.0.1:9091/" + } + u, err := url.Parse(internalVMURL) if err != nil { return nil, errors.WithStack(err) } @@ -82,7 +83,7 @@ func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, baseURL string, db: db, baseURL: u, client: &http.Client{}, // TODO instrument with utils/irt; see vmalert package https://jira.percona.com/browse/PMM-7229 - baseConfigPath: params.BaseConfigPath, + params: params, l: logrus.WithField("component", "victoriametrics"), reloadCh: make(chan struct{}, 1), }, nil @@ -95,12 +96,17 @@ func (svc *Service) Run(ctx context.Context) { svc.l.Info("Starting...") defer svc.l.Info("Done.") - - if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } - if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) + if svc.params.ExternalVM() { + if err := dir.CreateDataDir(vmagentDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } + } else { + if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } + if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } } // reloadCh, configuration update loop, and RequestConfigurationUpdate method ensure that configuration @@ -181,18 +187,18 @@ func (svc *Service) reload(ctx context.Context) error { return errors.WithStack(err) } - if resp.StatusCode != http.StatusNoContent { - return errors.Errorf("expected 204, got %d", resp.StatusCode) + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return errors.Errorf("expected 200 or 204, got %d", resp.StatusCode) } return nil } // loadBaseConfig returns parsed base configuration file, or empty configuration on error. func (svc *Service) loadBaseConfig() *config.Config { - buf, err := os.ReadFile(svc.baseConfigPath) + buf, err := os.ReadFile(svc.params.BaseConfigPath) if err != nil { if !os.IsNotExist(err) { - svc.l.Errorf("Failed to load base VictoriaMetrics config %s: %s", svc.baseConfigPath, err) + svc.l.Errorf("Failed to load base VictoriaMetrics config %s: %s", svc.params.BaseConfigPath, err) } return &config.Config{} @@ -200,7 +206,7 @@ func (svc *Service) loadBaseConfig() *config.Config { var cfg config.Config if err := yaml.Unmarshal(buf, &cfg); err != nil { - svc.l.Errorf("Failed to parse base VictoriaMetrics config %s: %s.", svc.baseConfigPath, err) + svc.l.Errorf("Failed to parse base VictoriaMetrics config %s: %s.", svc.params.BaseConfigPath, err) return &config.Config{} } @@ -330,7 +336,14 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR)) + u, err := url.Parse(svc.params.URL) + if err != nil { + return err + } + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u.Host)) + if svc.params.ExternalVM() { + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) + } cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVMAlert(s.HR)) AddInternalServicesToScrape(cfg, s, settings.DBaaS.Enabled) return AddScrapeConfigs(svc.l, cfg, tx.Querier, &s, nil, false) @@ -338,7 +351,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration) *config.ScrapeConfig { +func scrapeConfigForVictoriaMetrics(interval time.Duration, target string) *config.ScrapeConfig { return &config.ScrapeConfig{ JobName: "victoriametrics", ScrapeInterval: config.Duration(interval), @@ -347,8 +360,26 @@ func scrapeConfigForVictoriaMetrics(interval time.Duration) *config.ScrapeConfig ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ StaticConfigs: []*config.Group{ { - Targets: []string{"127.0.0.1:9090"}, - Labels: map[string]string{"instance": "pmm-server"}, + Targets: []string{target}, + Labels: map[string]string{"instance": models.PMMServerAgentID}, + }, + }, + }, + } +} + +// scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. +func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { + return &config.ScrapeConfig{ + JobName: "vmagent", + ScrapeInterval: config.Duration(interval), + ScrapeTimeout: ScrapeTimeout(interval), + MetricsPath: "/metrics", + ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ + StaticConfigs: []*config.Group{ + { + Targets: []string{target}, + Labels: map[string]string{"instance": models.PMMServerAgentID}, }, }, }, diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 811f1b6d83..de824cb29c 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,8 +43,8 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml"} - svc, err := NewVictoriaMetrics(configPath, db, "http://127.0.0.1:9090/prometheus/", vmParams) + vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml", URL: "http://127.0.0.1:9090/prometheus/"} + svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) original, err := os.ReadFile(configPath) @@ -801,7 +801,7 @@ func TestBaseConfig(t *testing.T) { db, svc, original := setup(t) defer teardown(t, db, svc, original) - svc.baseConfigPath = "../../testdata/victoriametrics/promscrape.base.yml" + svc.params.BaseConfigPath = "../../testdata/victoriametrics/promscrape.base.yml" expected := strings.TrimSpace(` # Managed by pmm-managed. DO NOT EDIT. diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index ea4112798a..2cbb0a53b6 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -41,6 +41,7 @@ const ( envEnableAccessControl = "ENABLE_RBAC" envPlatformAPITimeout = "PERCONA_PLATFORM_API_TIMEOUT" defaultPlatformAPITimeout = 30 * time.Second + ENVvmAgentPrefix = "VMAGENT_" ) // InvalidDurationError invalid duration error. @@ -202,6 +203,11 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs continue } + // skip VM Agents environment variables + if strings.HasPrefix(k, ENVvmAgentPrefix) { + continue + } + // skip supervisord environment variables if strings.HasPrefix(k, "SUPERVISOR_") { continue diff --git a/update/ansible/playbook/tasks/files/pmm.ini b/update/ansible/playbook/tasks/files/pmm.ini index e82339638f..e9c467e498 100644 --- a/update/ansible/playbook/tasks/files/pmm.ini +++ b/update/ansible/playbook/tasks/files/pmm.ini @@ -96,7 +96,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --postgres-name=pmm-managed --postgres-username=pmm-managed --postgres-password=pmm-managed From 8400da1eccfa38db1d9e9747c9cca3d3f0e7cb17 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 7 Mar 2023 01:22:02 +0600 Subject: [PATCH 02/21] PMM-9374 update version number of victoria metrics. --- build/packages/rpm/server/SPECS/victoriametrics.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index f8ebcb5366..050f87e970 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -13,7 +13,7 @@ Name: percona-victoriametrics Version: 1.82.1 -Release: 1%{?dist} +Release: 2%{?dist} Summary: VictoriaMetrics monitoring solution and time series database License: Apache-2.0 URL: https://%{provider} From c1ceff0e62b3ff150f769196c34e9b386fafd9ff Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 13 Mar 2023 18:54:03 +0600 Subject: [PATCH 03/21] PMM-9374 vmalert. --- docker-compose.yml | 1 - managed/services/supervisord/logs.go | 8 ++++++-- managed/services/supervisord/supervisord.go | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a07fe19f11..9229b902f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,7 +104,6 @@ services: - "--opentsdbListenAddr=:4242" - "--httpListenAddr=:8428" - "--influxListenAddr=:8089" - - "--vmalert.proxyURL=http://vmalert:8880" networks: - ${NETWORK:-default} pmm-managed-server-ch: diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index c6d94a0258..9efb941108 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -25,6 +25,7 @@ import ( "io" "mime" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -37,6 +38,7 @@ import ( "golang.org/x/sys/unix" "gopkg.in/yaml.v3" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/logger" pprofUtils "github.com/percona/pmm/managed/utils/pprof" "github.com/percona/pmm/utils/pdeathsig" @@ -59,14 +61,16 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker + vmParams *models.VictoriaMetricsParams } // NewLogs creates a new Logs service. // n is a number of last lines of log to read. -func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs { +func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams *models.VictoriaMetricsParams) *Logs { return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, + vmParams: vmParams, } } @@ -213,7 +217,7 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten Data: b, Err: err, }) - + url.Parse(l.vmParams.URL) // add VictoriaMetrics targets b, err = readURL(ctx, "http://127.0.0.1:9090/prometheus/api/v1/targets") files = append(files, fileContent{ diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index afae6f53b8..63944ed36e 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -688,11 +688,11 @@ command = --notifier.url="{{ .AlertmanagerURL }}" --notifier.basicAuth.password='{{ .AlertManagerPassword }}' --notifier.basicAuth.username="{{ .AlertManagerUser }}" - --external.url=http://localhost:9090/prometheus - --datasource.url=http://127.0.0.1:9090/prometheus - --remoteRead.url=http://127.0.0.1:9090/prometheus + --external.url={{ .VMURL }} + --datasource.url={{ .VMURL }} + --remoteRead.url={{ .VMURL }} --remoteRead.ignoreRestoreErrors=false - --remoteWrite.url=http://127.0.0.1:9090/prometheus + --remoteWrite.url={{ .VMURL }} --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml --httpListenAddr=127.0.0.1:8880 From ed017ba8502b3d68de9b10c2eb939c6ffd01ca44 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 15 Mar 2023 17:57:12 +0600 Subject: [PATCH 04/21] PMM-9374 use pmm-clients vmagent instead of a separate one. --- .../rpm/server/SPECS/victoriametrics.spec | 10 +--- managed/cmd/pmm-managed/main.go | 4 +- managed/services/agents/registry.go | 20 ++++--- managed/services/supervisord/logs.go | 16 +++++- managed/services/supervisord/supervisord.go | 28 ---------- .../victoriametrics/victoriametrics.go | 52 +++++++++++-------- 6 files changed, 59 insertions(+), 71 deletions(-) diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index 050f87e970..84f2b0a2c2 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -13,7 +13,7 @@ Name: percona-victoriametrics Version: 1.82.1 -Release: 2%{?dist} +Release: 1%{?dist} Summary: VictoriaMetrics monitoring solution and time series database License: Apache-2.0 URL: https://%{provider} @@ -35,13 +35,11 @@ export USER=builder make victoria-metrics-pure make vmalert-pure -make vmagent-pure %install install -D -p -m 0755 ./bin/victoria-metrics-pure %{buildroot}%{_sbindir}/victoriametrics install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert -install -D -p -m 0755 ./bin/vmagent-pure %{buildroot}%{_sbindir}/vmagent %files @@ -49,14 +47,10 @@ install -D -p -m 0755 ./bin/vmagent-pure %{buildroot}%{_sbindir}/vmagent %doc README.md %{_sbindir}/victoriametrics %{_sbindir}/vmalert -%{_sbindir}/vmagent %changelog -* Thu Mar 2 2023 Nurlan Moldomurov - 1.82.1-2 -- add vmagent to PMM Server - -* Thu Oct 20 2022 Michal Kralik - 1.82.1-1 +* Thu Oct 20 2022 Michal Kralik - 1.82.1 - upgrade victoriametrics to 1.82.1 release * Thu May 11 2022 Michael Okoko - 1.77.1 diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index a8f1727767..34c8df55ad 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -757,7 +757,7 @@ func main() { qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) - agentsRegistry := agents.NewRegistry(db) + agentsRegistry := agents.NewRegistry(db, vmParams.ExternalVM()) backupRemovalService := backup.NewRemovalService(db, minioClient) pitrTimerangeService := backup.NewPITRTimerangeService(minioClient) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) @@ -772,7 +772,7 @@ func main() { pmmUpdateCheck := supervisord.NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker")) - logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck) + logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck, vmParams) supervisord := supervisord.New(*supervisordConfigDirF, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) platformAddress, err := envvars.GetPlatformAddress() diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 28ea5965f6..00caf4f348 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -85,10 +85,12 @@ type Registry struct { mRoundTrip prom.Summary mClockDrift prom.Summary mAgents prom.GaugeFunc + + isExternalVM bool } // NewRegistry creates a new registry with given database connection. -func NewRegistry(db *reform.DB) *Registry { +func NewRegistry(db *reform.DB, isExternalVM bool) *Registry { agents := make(map[string]*pmmAgentInfo) r := &Registry{ db: db, @@ -123,6 +125,8 @@ func NewRegistry(db *reform.DB) *Registry { Help: "Clock drift.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }), + + isExternalVM: isExternalVM, } r.mAgents = prom.NewGaugeFunc(prom.GaugeOpts{ @@ -160,7 +164,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, } var node *models.Node err = r.db.InTransaction(func(tx *reform.TX) error { - node, err = authenticate(agentMD, tx.Querier) + node, err = r.authenticate(agentMD, tx.Querier) if err != nil { return err } @@ -204,7 +208,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, return agent, nil } -func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) { +func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) { if md.ID == "" { return nil, status.Error(codes.PermissionDenied, "Empty Agent ID.") } @@ -233,7 +237,7 @@ func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models. return nil, status.Errorf(codes.InvalidArgument, "Can't parse 'version' for pmm-agent with ID %q.", md.ID) } - if err := addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil { + if err := r.addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil { return nil, err } @@ -296,18 +300,18 @@ func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error { // addOrRemoveVMAgent - creates vmAgent agentType if pmm-agent's version supports it and agent not exists yet, // otherwise ensures that vmAgent not exist for pmm-agent and pmm-agent's agents don't have push_metrics mode, // removes it if needed. -func addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error { +func (r *Registry) addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error { if pmmAgentVersion.Less(models.PMMAgentWithPushMetricsSupport) { // ensure that vmagent not exists and agents dont have push_metrics. return removeVMAgentFromPMMAgent(q, pmmAgentID) } - return addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID) + return r.addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID) } -func addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error { +func (r *Registry) addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error { // TODO remove it after fix // https://jira.percona.com/browse/PMM-4420 - if runsOnNodeID == "pmm-server" { + if runsOnNodeID == "pmm-server" && !r.isExternalVM { return nil } vmAgentType := models.VMAgentType diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 9efb941108..207d5121ee 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -217,9 +217,9 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten Data: b, Err: err, }) - url.Parse(l.vmParams.URL) + // add VictoriaMetrics targets - b, err = readURL(ctx, "http://127.0.0.1:9090/prometheus/api/v1/targets") + b, err = l.victoriaMetricsTargets(ctx) files = append(files, fileContent{ Name: "victoriametrics_targets.json", Data: b, @@ -284,6 +284,18 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten return files } +func (l *Logs) victoriaMetricsTargets(ctx context.Context) ([]byte, error) { + vmURL, err := url.Parse(l.vmParams.URL) + if err != nil { + return nil, err + } + targetsURL, err := vmURL.Parse("api/v1/targets") + if err != nil { + return nil, err + } + return readURL(ctx, targetsURL.String()) +} + // readLog reads last lines (up to given number of lines and bytes) from given file, // and returns them together with modification time. func readLog(name string, maxLines int, maxBytes int64) ([]byte, time.Time, error) { diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 63944ed36e..a7e5adfcc2 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -615,34 +615,6 @@ stdout_logfile_backups = 3 redirect_stderr = true {{end}} -{{define "vmagent"}} -{{- if .EnableVMAgent }} -[program:vmagent] -priority = 7 -command = - /usr/sbin/vmagent - -remoteWrite.url={{ .VMURL }}api/v1/write - -remoteWrite.maxDiskUsagePerURL=1073741824 - -remoteWrite.tmpDataPath=/srv/vmagent/data - -promscrape.config=/etc/victoriametrics-promscrape.yml - -httpListenAddr=127.0.0.1:9091 - -loggerLevel=INFO - -envflag.enable - -envflag.prefix=VMAGENT_ -user = pmm -autorestart = true -autostart = true -startretries = 10 -startsecs = 1 -stopsignal = INT -stopwaitsecs = 300 -stdout_logfile = /srv/logs/vmagent.log -stdout_logfile_maxbytes = 10MB -stdout_logfile_backups = 3 -redirect_stderr = true -{{end}} -{{end}} - {{define "victoriametrics"}} {{- if not .EnableVMAgent }} [program:victoriametrics] diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 43b536287f..f4347b6a4d 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -46,7 +46,6 @@ const ( updateBatchDelay = time.Second configurationUpdateTimeout = 3 * time.Second - vmagentDir = "/srv/vmagent" victoriametricsDir = "/srv/victoriametrics" victoriametricsDataDir = "/srv/victoriametrics/data" dirPerm = os.FileMode(0o775) @@ -69,11 +68,7 @@ type Service struct { // NewVictoriaMetrics creates new VictoriaMetrics service. func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { - internalVMURL := params.URL - if params.ExternalVM() { - internalVMURL = "http://127.0.0.1:9091/" - } - u, err := url.Parse(internalVMURL) + u, err := url.Parse(params.URL) if err != nil { return nil, errors.WithStack(err) } @@ -96,17 +91,12 @@ func (svc *Service) Run(ctx context.Context) { svc.l.Info("Starting...") defer svc.l.Info("Done.") - if svc.params.ExternalVM() { - if err := dir.CreateDataDir(vmagentDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } - } else { - if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } - if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } + + if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } + if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) } // reloadCh, configuration update loop, and RequestConfigurationUpdate method ensure that configuration @@ -151,6 +141,9 @@ func (svc *Service) RequestConfigurationUpdate() { // updateConfiguration updates VictoriaMetrics configuration. func (svc *Service) updateConfiguration(ctx context.Context) error { + if svc.params.ExternalVM() { + return nil + } start := time.Now() defer func() { if dur := time.Since(start); dur > time.Second { @@ -158,8 +151,7 @@ func (svc *Service) updateConfiguration(ctx context.Context) error { } }() - base := svc.loadBaseConfig() - cfg, err := svc.marshalConfig(base) + cfg, err := svc.buildVMConfig() if err != nil { return err } @@ -167,6 +159,11 @@ func (svc *Service) updateConfiguration(ctx context.Context) error { return svc.configAndReload(ctx, cfg) } +func (svc *Service) buildVMConfig() ([]byte, error) { + base := svc.loadBaseConfig() + return svc.marshalConfig(base) +} + // reload asks VictoriaMetrics to reload configuration. func (svc *Service) reload(ctx context.Context) error { u := *svc.baseURL @@ -340,7 +337,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if err != nil { return err } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u.Host)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u)) if svc.params.ExternalVM() { cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) } @@ -351,16 +348,18 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration, target string) *config.ScrapeConfig { +func scrapeConfigForVictoriaMetrics(interval time.Duration, target *url.URL) *config.ScrapeConfig { + target, _ = target.Parse("metrics") + return &config.ScrapeConfig{ JobName: "victoriametrics", ScrapeInterval: config.Duration(interval), ScrapeTimeout: ScrapeTimeout(interval), - MetricsPath: "/prometheus/metrics", + MetricsPath: target.Path, ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ StaticConfigs: []*config.Group{ { - Targets: []string{target}, + Targets: []string{target.Host}, Labels: map[string]string{"instance": models.PMMServerAgentID}, }, }, @@ -406,6 +405,9 @@ func scrapeConfigForVMAlert(interval time.Duration) *config.ScrapeConfig { // BuildScrapeConfigForVMAgent builds scrape configuration for given pmm-agent. func (svc *Service) BuildScrapeConfigForVMAgent(pmmAgentID string) ([]byte, error) { + if pmmAgentID == models.PMMServerAgentID { + return svc.buildVMConfig() + } var cfg config.Config e := svc.db.InTransaction(func(tx *reform.TX) error { settings, err := models.GetSettings(tx) @@ -424,6 +426,10 @@ func (svc *Service) BuildScrapeConfigForVMAgent(pmmAgentID string) ([]byte, erro // IsReady verifies that VictoriaMetrics works. func (svc *Service) IsReady(ctx context.Context) error { + if svc.params.ExternalVM() { + svc.l.Debugf("External VM is used, VM healthcheck is skipped") + return nil + } u := *svc.baseURL u.Path = path.Join(u.Path, "health") req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) From 36182b45acc626784565a91fa35361be346e9603 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 29 Mar 2023 15:16:27 +0300 Subject: [PATCH 05/21] PMM-9374 Fix linters. --- managed/services/agents/vmagent.go | 1 - managed/services/supervisord/logs.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index 0934a29634..1740cac088 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -17,7 +17,6 @@ package agents import ( "fmt" - "github.com/percona/pmm/managed/utils/envvars" "os" "sort" "strings" diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 7e58274e14..9748cdf695 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -61,7 +61,7 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker - vmParams *models.VictoriaMetricsParams + vmParams *models.VictoriaMetricsParams } // NewLogs creates a new Logs service. @@ -70,7 +70,7 @@ func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams *mo return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, - vmParams: vmParams, + vmParams: vmParams, } } From b46395af41ff7f2350aa6b6987b2dfbdc99a9eb0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 29 Mar 2023 16:02:17 +0300 Subject: [PATCH 06/21] PMM-9374 Fix tests. --- managed/models/victoriametrics_params_test.go | 4 ++-- managed/services/agents/vmagent_test.go | 10 ++++++++-- managed/services/supervisord/logs_test.go | 9 +++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index c52c349800..6362299128 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -23,11 +23,11 @@ import ( func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "", "", "") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "") require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "", "", "") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "") require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 2adc5fded8..3a1431ee1b 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -16,6 +16,8 @@ package agents import ( + "github.com/percona/pmm/managed/models" + "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/assert" @@ -23,13 +25,17 @@ import ( func TestMaxScrapeSize(t *testing.T) { t.Run("by default 64MiB", func(t *testing.T) { - actual := vmAgentConfig("") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) + actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+maxScrapeSizeDefault) }) t.Run("overridden with ENV", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) newValue := "16MiB" t.Setenv(maxScrapeSizeEnv, newValue) - actual := vmAgentConfig("") + actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+newValue) }) } diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index 265e17c112..02f9f52ff7 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "github.com/percona/pmm/managed/models" "os" "path/filepath" "sort" @@ -122,7 +123,9 @@ func TestAddAdminSummary(t *testing.T) { func TestFiles(t *testing.T) { checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - l := NewLogs("2.4.5", checker) + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) + l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) files := l.files(ctx, nil) @@ -160,7 +163,9 @@ func TestZip(t *testing.T) { t.Skip("FIXME") checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - l := NewLogs("2.4.5", checker) + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) + l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) var buf bytes.Buffer From 7fe77aa55fc64683c99f0bb5b7ab43a031474f6d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 29 Mar 2023 22:59:42 +0300 Subject: [PATCH 07/21] PMM-9374 Improve work with VM Proxy. --- docker-compose.yml | 7 ------- managed/services/agents/vmagent_test.go | 9 +++++---- managed/services/supervisord/logs_test.go | 4 ++-- managed/services/supervisord/supervisord.go | 2 +- managed/services/supervisord/supervisord_test.go | 4 ++-- managed/testdata/supervisord.d/vmalert.ini | 8 ++++---- managed/testdata/supervisord.d/vmproxy.ini | 2 +- update/ansible/playbook/tasks/files/datasources.yml | 2 +- vmproxy/Makefile | 2 +- vmproxy/proxy/proxy.go | 7 +++++++ 10 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fda1cb6962..5eb978112e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,13 +164,6 @@ services: - ${PMM_PORT_HTTPS:-443}:443 # For headless delve - ${PMM_PORT_DELVE:-2345}:2345 - # PG - - ${PMM_PORT_PG:-15432}:5432 - # VM - - ${PMM_PORT_VM:-9090}:9090 - # CH - - ${PMM_PORT_CH_TCP:-11000}:9000 - - ${PMM_PORT_CH_HTTP:-11123}:8123 volumes: - ./:/root/go/src/github.com/percona/pmm # - "../grafana/public:/usr/share/grafana/public" diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 3a1431ee1b..6522629ab4 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -16,22 +16,23 @@ package agents import ( - "github.com/percona/pmm/managed/models" - "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/percona/pmm/managed/models" ) func TestMaxScrapeSize(t *testing.T) { t.Run("by default 64MiB", func(t *testing.T) { - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+maxScrapeSizeDefault) }) t.Run("overridden with ENV", func(t *testing.T) { - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) newValue := "16MiB" t.Setenv(maxScrapeSizeEnv, newValue) diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index 02f9f52ff7..f9b2d19f7a 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "fmt" - "github.com/percona/pmm/managed/models" "os" "path/filepath" "sort" @@ -32,6 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/logger" ) @@ -163,7 +163,7 @@ func TestZip(t *testing.T) { t.Skip("FIXME") checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 31b56a951b..1a1c854b52 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -654,7 +654,7 @@ stdout_logfile = /srv/logs/victoriametrics.log stdout_logfile_maxbytes = 10MB stdout_logfile_backups = 3 redirect_stderr = true -{{end}} +{{end -}} {{end}} {{define "vmalert"}} diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index aa2b2e90d6..86578c5d1e 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -36,7 +36,7 @@ func TestConfig(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{} + vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, @@ -67,7 +67,7 @@ func TestDBaaSController(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{} + vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) var tp *template.Template diff --git a/managed/testdata/supervisord.d/vmalert.ini b/managed/testdata/supervisord.d/vmalert.ini index e784596bcb..40f61b1d83 100644 --- a/managed/testdata/supervisord.d/vmalert.ini +++ b/managed/testdata/supervisord.d/vmalert.ini @@ -7,11 +7,11 @@ command = --notifier.url="http://127.0.0.1:9093/alertmanager,https://external-alertmanager:6443/alerts" --notifier.basicAuth.password=',"passw!,ord"' --notifier.basicAuth.username=",external-user" - --external.url=http://localhost:9090/prometheus - --datasource.url=http://127.0.0.1:9090/prometheus - --remoteRead.url=http://127.0.0.1:9090/prometheus + --external.url=http://127.0.0.1:9090/prometheus/ + --datasource.url=http://127.0.0.1:9090/prometheus/ + --remoteRead.url=http://127.0.0.1:9090/prometheus/ --remoteRead.ignoreRestoreErrors=false - --remoteWrite.url=http://127.0.0.1:9090/prometheus + --remoteWrite.url=http://127.0.0.1:9090/prometheus/ --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml --httpListenAddr=127.0.0.1:8880 diff --git a/managed/testdata/supervisord.d/vmproxy.ini b/managed/testdata/supervisord.d/vmproxy.ini index 72acc05eaf..5654b85bc4 100644 --- a/managed/testdata/supervisord.d/vmproxy.ini +++ b/managed/testdata/supervisord.d/vmproxy.ini @@ -4,7 +4,7 @@ priority = 9 command = /usr/sbin/vmproxy - --target-url=http://127.0.0.1:9090/ + --target-url=http://127.0.0.1:9090/prometheus/ --listen-port=8430 --listen-address=127.0.0.1 --header-name=X-Proxy-Filter diff --git a/update/ansible/playbook/tasks/files/datasources.yml b/update/ansible/playbook/tasks/files/datasources.yml index 733ea6679d..2145e92684 100644 --- a/update/ansible/playbook/tasks/files/datasources.yml +++ b/update/ansible/playbook/tasks/files/datasources.yml @@ -5,7 +5,7 @@ datasources: orgId: 1 type: prometheus access: proxy - url: http://127.0.0.1:8430/prometheus/ + url: http://127.0.0.1:8430/ isDefault: true jsonData: httpMethod: POST diff --git a/vmproxy/Makefile b/vmproxy/Makefile index 458aea2543..de22f04d71 100644 --- a/vmproxy/Makefile +++ b/vmproxy/Makefile @@ -16,7 +16,7 @@ ifeq ($(GOBIN),) endif LD_FLAGS = -ldflags " \ - -X 'github.com/percona/pmm/version.ProjectName=pmm-admin' \ + -X 'github.com/percona/pmm/version.ProjectName=vmproxy' \ -X 'github.com/percona/pmm/version.Version=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.PMMVersion=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.Timestamp=$(PMM_RELEASE_TIMESTAMP)' \ diff --git a/vmproxy/proxy/proxy.go b/vmproxy/proxy/proxy.go index 907043345b..8d7aec1c8a 100644 --- a/vmproxy/proxy/proxy.go +++ b/vmproxy/proxy/proxy.go @@ -24,6 +24,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "strings" "time" "github.com/sirupsen/logrus" @@ -83,6 +84,12 @@ func director(target *url.URL, headerName string) func(*http.Request) { req.URL.Scheme = target.Scheme req.URL.Host = target.Host + rp, err := target.Parse(strings.TrimPrefix(req.URL.Path, "/")) + if err != nil { + logrus.Error(err) + } + req.URL.Path = rp.Path + // Replace extra filters if present if filters := req.Header.Get(headerName); filters != "" { q := req.URL.Query() From 2ec6403b1b4da855f658091bbc11e5dd283e35c7 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 11:54:33 +0300 Subject: [PATCH 08/21] PMM-9374 Fix tests. --- managed/models/victoriametrics_params_test.go | 4 ++-- managed/services/supervisord/devcontainer_test.go | 2 +- managed/services/supervisord/logs_test.go | 2 +- managed/services/victoriametrics/victoriametrics_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index 6362299128..4c9b80a3f3 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -23,11 +23,11 @@ import ( func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", VMBaseURL) require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", VMBaseURL) require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index 0a7604714c..c902faf6c5 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -112,7 +112,7 @@ func TestDevContainer(t *testing.T) { t.Run("UpdateConfiguration", func(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - vmParams := &models.VictoriaMetricsParams{} + vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} s := New("/etc/supervisord.d", checker, vmParams, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index f9b2d19f7a..2f51bc4097 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -123,7 +123,7 @@ func TestAddAdminSummary(t *testing.T) { func TestFiles(t *testing.T) { checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index de824cb29c..780fad011d 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,7 +43,7 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml", URL: "http://127.0.0.1:9090/prometheus/"} + vmParams := &models.VictoriaMetricsParams{BaseConfigPath: models.BasePrometheusConfigPath, URL: models.VMBaseURL} svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) From 91c83c4afa208bfae3108a3d613d6d0c900dc2d2 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 15:14:50 +0300 Subject: [PATCH 09/21] PMM-9374 Fix tests. --- managed/models/victoriametrics_params.go | 3 +- managed/services/supervisord/supervisord.go | 15 +++++----- .../victoriametrics/victoriametrics.go | 30 +++++++++++++------ .../victoriametrics/victoriametrics_test.go | 30 ++++++++++++++++--- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 8cda5ee576..1693fa6a6b 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -17,6 +17,7 @@ package models import ( "os" + "strings" config "github.com/percona/promconfig" "github.com/pkg/errors" @@ -91,5 +92,5 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { } func (vmp *VictoriaMetricsParams) ExternalVM() bool { - return vmp.URL != VMBaseURL + return !strings.HasPrefix(vmp.URL, "http://127.0.0.1") } diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 1a1c854b52..f4fadab40c 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -430,8 +430,9 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, "VMURL": s.vmParams.URL, - "EnableVMAgent": s.vmParams.ExternalVM(), + "ExternalVM": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, + "InterfaceToBind": envvars.GetInterfaceToBind(), "ClickhouseAddr": clickhouseAddr, "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, "ClickhouseDatabase": clickhouseDatabase, @@ -583,8 +584,6 @@ func (s *Service) RestartSupervisedService(serviceName string) error { return err } -var interfaceToBind = envvars.GetInterfaceToBind() - //nolint:lll var templates = template.Must(template.New("").Option("missingkey=error").Parse(` {{define "dbaas-controller"}} @@ -621,7 +620,7 @@ redirect_stderr = true {{end}} {{define "victoriametrics"}} -{{- if not .EnableVMAgent }} +{{- if not .ExternalVM }} [program:victoriametrics] priority = 7 command = @@ -629,7 +628,7 @@ command = --promscrape.config=/etc/victoriametrics-promscrape.yml --retentionPeriod={{ .DataRetentionDays }}d --storageDataPath=/srv/victoriametrics/data - --httpListenAddr=` + interfaceToBind + `:9090 + --httpListenAddr={{ .InterfaceToBind }}:9090 --search.disableCache={{ .VMDBCacheDisable }} --search.maxQueryLen=1MB --search.latencyOffset=5s @@ -672,7 +671,7 @@ command = --remoteWrite.url={{ .VMURL }} --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml - --httpListenAddr=` + interfaceToBind + `:8880 + --httpListenAddr={{ .InterfaceToBind }}:8880 {{- range $index, $param := .VMAlertFlags }} {{ $param }} {{- end }} @@ -696,7 +695,7 @@ command = /usr/sbin/vmproxy --target-url={{ .VMURL }} --listen-port=8430 - --listen-address=` + interfaceToBind + ` + --listen-address={{ .InterfaceToBind }} --header-name=X-Proxy-Filter user = pmm autorestart = true @@ -720,7 +719,7 @@ command = --storage.path=/srv/alertmanager/data --data.retention={{ .DataRetentionHours }}h --web.external-url=http://localhost:9093/alertmanager/ - --web.listen-address=` + interfaceToBind + `:9093 + --web.listen-address={{ .InterfaceToBind }}:9093 --cluster.listen-address="" user = pmm autorestart = true diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 6e32927f5f..c627843425 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -25,6 +25,7 @@ import ( "os/exec" "path" "regexp" + "strings" "time" "github.com/AlekSi/pointer" @@ -184,8 +185,8 @@ func (svc *Service) reload(ctx context.Context) error { return errors.WithStack(err) } - if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { - return errors.Errorf("expected 200 or 204, got %d", resp.StatusCode) + if resp.StatusCode != http.StatusNoContent { + return errors.Errorf("expected 204, got %d", resp.StatusCode) } return nil } @@ -333,11 +334,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - u, err := url.Parse(svc.params.URL) - if err != nil { - return err - } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params.URL)) if svc.params.ExternalVM() { cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) } @@ -348,8 +345,12 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration, target *url.URL) *config.ScrapeConfig { - target, _ = target.Parse("metrics") +func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, baseURL string) *config.ScrapeConfig { + target, err := relativePath(baseURL, "metrics") + if err != nil { + l.Errorf("couldn't parse relative path to victoria metrics: %q", err) + return nil + } return &config.ScrapeConfig{ JobName: "victoriametrics", @@ -367,6 +368,17 @@ func scrapeConfigForVictoriaMetrics(interval time.Duration, target *url.URL) *co } } +func relativePath(baseURL string, path string) (*url.URL, error) { + if !strings.HasSuffix(baseURL, "/") { + baseURL = baseURL + "/" + } + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + return u.Parse(path) +} + // scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { return &config.ScrapeConfig{ diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 780fad011d..b262d6fcf6 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -18,10 +18,7 @@ package victoriametrics import ( "context" "database/sql" - "os" - "strings" - "testing" - + "fmt" "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,6 +26,9 @@ import ( "google.golang.org/grpc/status" "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" + "os" + "strings" + "testing" "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/testdb" @@ -893,3 +893,25 @@ scrape_configs: assert.NoError(t, err) assert.Equal(t, expected, string(newcfg), "actual:\n%s", newcfg) } + +func Test_relativePath(t *testing.T) { + tests := []struct { + baseURL string + path string + want string + }{ + { + "http://127.0.0.1:9090/prometheus", + "metrics", + "http://127.0.0.1:9090/prometheus/metrics", + }, + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s", tt.want), func(t *testing.T) { + got, err := relativePath(tt.baseURL, tt.path) + assert.NoError(t, err) + assert.Equalf(t, tt.want, got.String(), "relativePath(%v, %v)", tt.baseURL, tt.path) + }) + } +} From 64b37396fb8a8e2f7cff3cc64ff157439423fd33 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 15:15:21 +0300 Subject: [PATCH 10/21] PMM-9374 Fix tests. --- managed/services/victoriametrics/victoriametrics_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index b262d6fcf6..8d03365820 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -905,7 +905,11 @@ func Test_relativePath(t *testing.T) { "metrics", "http://127.0.0.1:9090/prometheus/metrics", }, - // TODO: Add test cases. + { + "http://127.0.0.1:9090/prometheus/", + "metrics", + "http://127.0.0.1:9090/prometheus/metrics", + }, } for _, tt := range tests { t.Run(fmt.Sprintf("%s", tt.want), func(t *testing.T) { From 71ce8123c3e4f256cccee8fba27b722c488f0359 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 15:38:12 +0300 Subject: [PATCH 11/21] PMM-9374 Fix linters. --- managed/services/victoriametrics/victoriametrics_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 8d03365820..6d83a5d4a2 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -19,6 +19,10 @@ import ( "context" "database/sql" "fmt" + "os" + "strings" + "testing" + "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,9 +30,6 @@ import ( "google.golang.org/grpc/status" "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" - "os" - "strings" - "testing" "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/testdb" From abe1dd798e5fce1f1513b4ba438ad2aacd4d30b4 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 16:42:00 +0300 Subject: [PATCH 12/21] PMM-9374 Add new tests. --- managed/models/victoriametrics_params.go | 3 +++ managed/services/agents/vmagent_test.go | 19 +++++++++++++ .../victoriametrics/victoriametrics.go | 4 --- .../victoriametrics/victoriametrics_test.go | 27 ------------------- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 1693fa6a6b..841ed16b16 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -43,6 +43,9 @@ type VictoriaMetricsParams struct { // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { + if !strings.HasSuffix(vmURL, "/") { + vmURL = vmURL + "/" + } vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, URL: vmURL, diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 6522629ab4..43402e56ad 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -39,4 +39,23 @@ func TestMaxScrapeSize(t *testing.T) { actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+newValue) }) + t.Run("VMAGENT_ ENV variables", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + t.Setenv("VMAGENT_promscrape_maxScrapeSize", "16MiB") + t.Setenv("VM_remoteWrite_basicAuth_password", "password") + actual := vmAgentConfig("", params) + assert.Contains(t, actual.Env, "VMAGENT_promscrape_maxScrapeSize=16MiB") + assert.Contains(t, actual.Env, "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}") + assert.NotContains(t, actual.Env, "VM_remoteWrite_basicAuth_password=password") + }) + t.Run("External Victoria Metrics ENV variables", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "http://victoriametrics:8428") + require.NoError(t, err) + t.Setenv("VMAGENT_promscrape_maxScrapeSize", "16MiB") + actual := vmAgentConfig("", params) + assert.Contains(t, actual.Args, "-remoteWrite.url=http://victoriametrics:8428/api/v1/write") + assert.Contains(t, actual.Env, "VMAGENT_promscrape_maxScrapeSize=16MiB") + assert.NotContains(t, actual.Env, "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}") + }) } diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index c627843425..4f3c06c238 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -25,7 +25,6 @@ import ( "os/exec" "path" "regexp" - "strings" "time" "github.com/AlekSi/pointer" @@ -369,9 +368,6 @@ func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, bas } func relativePath(baseURL string, path string) (*url.URL, error) { - if !strings.HasSuffix(baseURL, "/") { - baseURL = baseURL + "/" - } u, err := url.Parse(baseURL) if err != nil { return nil, err diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 6d83a5d4a2..780fad011d 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -18,7 +18,6 @@ package victoriametrics import ( "context" "database/sql" - "fmt" "os" "strings" "testing" @@ -894,29 +893,3 @@ scrape_configs: assert.NoError(t, err) assert.Equal(t, expected, string(newcfg), "actual:\n%s", newcfg) } - -func Test_relativePath(t *testing.T) { - tests := []struct { - baseURL string - path string - want string - }{ - { - "http://127.0.0.1:9090/prometheus", - "metrics", - "http://127.0.0.1:9090/prometheus/metrics", - }, - { - "http://127.0.0.1:9090/prometheus/", - "metrics", - "http://127.0.0.1:9090/prometheus/metrics", - }, - } - for _, tt := range tests { - t.Run(fmt.Sprintf("%s", tt.want), func(t *testing.T) { - got, err := relativePath(tt.baseURL, tt.path) - assert.NoError(t, err) - assert.Equalf(t, tt.want, got.String(), "relativePath(%v, %v)", tt.baseURL, tt.path) - }) - } -} From c5d6afef4044a83b0420bd91266902e32993f8ee Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 01:00:18 +0300 Subject: [PATCH 13/21] PMM-9374 use interfaces instead of real object. --- managed/cmd/pmm-managed/main.go | 2 +- managed/models/victoriametrics_params.go | 25 ++++++++++++++++--- managed/services/agents/deps.go | 9 +++++++ managed/services/agents/registry.go | 4 +-- managed/services/agents/state.go | 4 +-- managed/services/agents/vmagent.go | 5 ++-- managed/services/supervisord/deps.go | 25 +++++++++++++++++++ .../services/supervisord/devcontainer_test.go | 3 ++- managed/services/supervisord/logs.go | 12 +++------ managed/services/supervisord/supervisord.go | 2 +- .../services/supervisord/supervisord_test.go | 6 +++-- .../victoriametrics/victoriametrics.go | 8 +++--- .../victoriametrics/victoriametrics_test.go | 3 ++- 13 files changed, 78 insertions(+), 30 deletions(-) create mode 100644 managed/services/supervisord/deps.go diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index b85c45d343..ca75a08627 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -781,7 +781,7 @@ func main() { qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) - agentsRegistry := agents.NewRegistry(db, vmParams.ExternalVM()) + agentsRegistry := agents.NewRegistry(db, vmParams) backupRemovalService := backup.NewRemovalService(db, minioClient) pitrTimerangeService := backup.NewPITRTimerangeService(minioClient) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 841ed16b16..b44d4ffa17 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -16,6 +16,7 @@ package models import ( + "net/url" "os" "strings" @@ -37,8 +38,8 @@ type VictoriaMetricsParams struct { VMAlertFlags []string // BaseConfigPath defines path for basic prometheus config. BaseConfigPath string - // URL defines URL of Victoria Metrics - URL string + // url defines url of Victoria Metrics + url *url.URL } // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. @@ -46,9 +47,14 @@ func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsPa if !strings.HasSuffix(vmURL, "/") { vmURL = vmURL + "/" } + + URL, err := url.Parse(vmURL) + if err != nil { + return nil, err + } vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, - URL: vmURL, + url: URL, } if err := vmp.UpdateParams(); err != nil { return vmp, err @@ -95,5 +101,16 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { } func (vmp *VictoriaMetricsParams) ExternalVM() bool { - return !strings.HasPrefix(vmp.URL, "http://127.0.0.1") + return vmp.url.Hostname() == "127.0.0.1" +} + +func (vmp *VictoriaMetricsParams) URL() string { + return vmp.url.String() +} + +func (vmp *VictoriaMetricsParams) URLFor(path string) (*url.URL, error) { + if path == "" { + return vmp.url, nil + } + return vmp.url.Parse(path) } diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index e05582ae11..b540e6f915 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -17,6 +17,7 @@ package agents import ( "context" + "net/url" "github.com/sirupsen/logrus" @@ -53,3 +54,11 @@ type jobsService interface { handleJobResult(ctx context.Context, l *logrus.Entry, result *agentpb.JobResult) handleJobProgress(ctx context.Context, progress *agentpb.JobProgress) } + +// victoriaMetricsParams is a subset of methods of models.VMParams used by this package. +// We use it instead of real type to avoid dependency cycle. +type victoriaMetricsParams interface { + ExternalVM() bool + URLFor(path string) (*url.URL, error) + URL() string +} diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 00caf4f348..84691cb739 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -90,7 +90,7 @@ type Registry struct { } // NewRegistry creates a new registry with given database connection. -func NewRegistry(db *reform.DB, isExternalVM bool) *Registry { +func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry { agents := make(map[string]*pmmAgentInfo) r := &Registry{ db: db, @@ -126,7 +126,7 @@ func NewRegistry(db *reform.DB, isExternalVM bool) *Registry { Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }), - isExternalVM: isExternalVM, + isExternalVM: externalVMChecker.ExternalVM(), } r.mAgents = prom.NewGaugeFunc(prom.GaugeOpts{ diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index b784b2d540..5d86f39c6e 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -44,11 +44,11 @@ type StateUpdater struct { db *reform.DB r *Registry vmdb prometheusService - vmParams *models.VictoriaMetricsParams + vmParams victoriaMetricsParams } // NewStateUpdater creates new agent state updater. -func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams *models.VictoriaMetricsParams) *StateUpdater { +func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams victoriaMetricsParams) *StateUpdater { return &StateUpdater{ db: db, r: r, diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index 1740cac088..b5586df710 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -23,7 +23,6 @@ import ( "github.com/percona/pmm/api/agentpb" "github.com/percona/pmm/api/inventorypb" - "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/envvars" ) @@ -33,10 +32,10 @@ var ( ) // vmAgentConfig returns desired configuration of vmagent process. -func vmAgentConfig(scrapeCfg string, params *models.VictoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { +func vmAgentConfig(scrapeCfg string, params victoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { serverURL := "{{.server_url}}/victoriametrics/" if params.ExternalVM() { - serverURL = params.URL + serverURL = params.URL() } maxScrapeSize := maxScrapeSizeDefault if space := os.Getenv(maxScrapeSizeEnv); space != "" { diff --git a/managed/services/supervisord/deps.go b/managed/services/supervisord/deps.go new file mode 100644 index 0000000000..8dea19e974 --- /dev/null +++ b/managed/services/supervisord/deps.go @@ -0,0 +1,25 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package supervisord + +import "net/url" + +// victoriaMetricsParams is a subset of methods of models.VMParams used by this package. +// We use it instead of real type to avoid dependency cycle. +type victoriaMetricsParams interface { + ExternalVM() bool + URLFor(path string) (*url.URL, error) +} diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index c902faf6c5..ae0ca51959 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -112,7 +112,8 @@ func TestDevContainer(t *testing.T) { t.Run("UpdateConfiguration", func(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New("/etc/supervisord.d", checker, vmParams, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 9748cdf695..1b78998bd0 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -25,7 +25,6 @@ import ( "io" "mime" "net/http" - "net/url" "os" "os/exec" "path/filepath" @@ -38,7 +37,6 @@ import ( "golang.org/x/sys/unix" "gopkg.in/yaml.v3" - "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/logger" pprofUtils "github.com/percona/pmm/managed/utils/pprof" "github.com/percona/pmm/utils/pdeathsig" @@ -61,12 +59,12 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker - vmParams *models.VictoriaMetricsParams + vmParams victoriaMetricsParams } // NewLogs creates a new Logs service. // n is a number of last lines of log to read. -func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams *models.VictoriaMetricsParams) *Logs { +func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams victoriaMetricsParams) *Logs { return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, @@ -285,11 +283,7 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten } func (l *Logs) victoriaMetricsTargets(ctx context.Context) ([]byte, error) { - vmURL, err := url.Parse(l.vmParams.URL) - if err != nil { - return nil, err - } - targetsURL, err := vmURL.Parse("api/v1/targets") + targetsURL, err := l.vmParams.URLFor("api/v1/targets") if err != nil { return nil, err } diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index f4fadab40c..38af00df0d 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -429,7 +429,7 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "DataRetentionDays": int(settings.DataRetention.Hours() / 24), "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, - "VMURL": s.vmParams.URL, + "VMURL": s.vmParams.URL(), "ExternalVM": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, "InterfaceToBind": envvars.GetInterfaceToBind(), diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index 86578c5d1e..d80de1afac 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -36,7 +36,8 @@ func TestConfig(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, @@ -67,7 +68,8 @@ func TestDBaaSController(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) var tp *template.Template diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 4f3c06c238..f7015b157c 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -68,7 +68,7 @@ type Service struct { // NewVictoriaMetrics creates new VictoriaMetrics service. func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { - u, err := url.Parse(params.URL) + u, err := url.Parse(params.URL()) if err != nil { return nil, errors.WithStack(err) } @@ -333,7 +333,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params.URL)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params)) if svc.params.ExternalVM() { cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) } @@ -344,8 +344,8 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, baseURL string) *config.ScrapeConfig { - target, err := relativePath(baseURL, "metrics") +func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, vmParams *models.VictoriaMetricsParams) *config.ScrapeConfig { + target, err := vmParams.URLFor("metrics") if err != nil { l.Errorf("couldn't parse relative path to victoria metrics: %q", err) return nil diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 780fad011d..cc6a67a70c 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,7 +43,8 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: models.BasePrometheusConfigPath, URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + check.NoError(err) svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) From f9b458a3a3a0192623d9a52afa6fbbe357f6be84 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 13:11:29 +0300 Subject: [PATCH 14/21] PMM-9374 Fix the test. --- managed/models/victoriametrics_params.go | 2 +- managed/models/victoriametrics_params_test.go | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index b44d4ffa17..30a38f8b27 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -101,7 +101,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { } func (vmp *VictoriaMetricsParams) ExternalVM() bool { - return vmp.url.Hostname() == "127.0.0.1" + return vmp.url.Hostname() != "127.0.0.1" } func (vmp *VictoriaMetricsParams) URL() string { diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index 4c9b80a3f3..fa4aa654fa 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -16,6 +16,7 @@ package models import ( + "github.com/stretchr/testify/assert" "testing" "github.com/stretchr/testify/require" @@ -31,4 +32,35 @@ func TestVictoriaMetricsParams(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) + t.Run("check external VM", func(t *testing.T) { + + tests := []struct { + url string + want bool + }{ + { + "http://127.0.0.1:9090/prometheus", + false, + }, + { + "http://127.0.0.1:9090/prometheus/", + false, + }, + { + "http://victoriametrics:8428/", + true, + }, + { + "https://example.com:9090/", + true, + }, + } + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + vmp, err := NewVictoriaMetricsParams(BasePrometheusConfigPath, tt.url) + require.NoError(t, err) + assert.Equalf(t, tt.want, vmp.ExternalVM(), "ExternalVM()") + }) + } + }) } From 0d2b497e4c43bc370960aa8a55fb3fae94b47a39 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 13:15:59 +0300 Subject: [PATCH 15/21] PMM-9374 Fix the linter. --- managed/services/victoriametrics/victoriametrics.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index f7015b157c..7a70108d67 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -367,14 +367,6 @@ func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, vmP } } -func relativePath(baseURL string, path string) (*url.URL, error) { - u, err := url.Parse(baseURL) - if err != nil { - return nil, err - } - return u.Parse(path) -} - // scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { return &config.ScrapeConfig{ From 986666388576d370089d478a02e5cc8859b2f663 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 14:21:38 +0300 Subject: [PATCH 16/21] PMM-9374 Fix the linter. --- managed/models/victoriametrics_params_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index fa4aa654fa..b0f55796f6 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -16,9 +16,9 @@ package models import ( - "github.com/stretchr/testify/assert" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,7 +33,6 @@ func TestVictoriaMetricsParams(t *testing.T) { require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) t.Run("check external VM", func(t *testing.T) { - tests := []struct { url string want bool From a456c0cb3e911c27e9a783794657d34935c0729c Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 28 Apr 2023 20:49:19 +0300 Subject: [PATCH 17/21] Update pmm_config.go --- managed/services/supervisord/pmm_config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index bc6fa3503a..f7e52bbe4b 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -165,7 +165,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true From e2323e26594cfa24e411d88b85fc5ae029b8805d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 28 Apr 2023 21:06:48 +0300 Subject: [PATCH 18/21] Update pmm-db_disabled.ini --- managed/testdata/supervisord.d/pmm-db_disabled.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/managed/testdata/supervisord.d/pmm-db_disabled.ini b/managed/testdata/supervisord.d/pmm-db_disabled.ini index 210d822422..b45251ab1f 100644 --- a/managed/testdata/supervisord.d/pmm-db_disabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_disabled.ini @@ -61,7 +61,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true From 2f8fc85becf02b9df627f2061f4bb4337b7d4811 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 31 Aug 2023 00:29:03 +0300 Subject: [PATCH 19/21] PMM-9374 fix supervisord for external VM. --- managed/services/supervisord/supervisord.go | 2 +- managed/utils/envvars/parser.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 23efd6fac4..40ac1f617b 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -579,7 +579,7 @@ func (s *Service) UpdateConfiguration(settings *models.Settings, ssoDetails *mod } for _, tmpl := range templates.Templates() { - if tmpl.Name() == "" { + if tmpl.Name() == "" || (tmpl.Name() == "victoriametrics" && s.vmParams.ExternalVM()) { continue } diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 2f70388254..4a5ac40146 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -157,6 +157,12 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs case "PMM_PUBLIC_ADDRESS": envSettings.PMMPublicAddress = v + case "PMM_VM_URL": + _, err := url.Parse(v) + if err != nil { + err = fmt.Errorf("invalid value %q for environment variable %q", v, k) + } + case "NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY": continue From a79a4fa68c8e68a7245ea4004a0e6ef15283ddad Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 5 Sep 2023 11:59:38 +0300 Subject: [PATCH 20/21] PMM-9374 Fix linters. --- managed/models/victoriametrics_params.go | 2 +- managed/utils/envvars/parser.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 30a38f8b27..f033b5a4f4 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -45,7 +45,7 @@ type VictoriaMetricsParams struct { // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { if !strings.HasSuffix(vmURL, "/") { - vmURL = vmURL + "/" + vmURL += "/" } URL, err := url.Parse(vmURL) diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 4a5ac40146..7847419984 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -158,7 +158,7 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs envSettings.PMMPublicAddress = v case "PMM_VM_URL": - _, err := url.Parse(v) + _, err = url.Parse(v) if err != nil { err = fmt.Errorf("invalid value %q for environment variable %q", v, k) } From 5a6a5a2c4201943ae7879256ae0956d114312b84 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 8 Sep 2023 12:16:23 +0300 Subject: [PATCH 21/21] PMM-9374 Don't run victoria metrics if it's external. --- managed/services/supervisord/supervisord.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 1e66d8bded..7172bbe8fa 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -394,20 +394,10 @@ func (s *Service) UpdateLog(offset uint32) ([]string, uint32, error) { // reload asks supervisord to reload configuration. func (s *Service) reload(name string) error { - // See https://github.com/Supervisor/supervisor/issues/1264 for explanation - // why we do reread + stop/remove/add. - if _, err := s.supervisorctl("reread"); err != nil { s.l.Warn(err) } - if _, err := s.supervisorctl("stop", name); err != nil { - s.l.Warn(err) - } - if _, err := s.supervisorctl("remove", name); err != nil { - s.l.Warn(err) - } - - _, err := s.supervisorctl("add", name) + _, err := s.supervisorctl("update", name) return err } @@ -592,7 +582,7 @@ func (s *Service) UpdateConfiguration(settings *models.Settings, ssoDetails *mod } for _, tmpl := range templates.Templates() { - if tmpl.Name() == "" || (tmpl.Name() == "victoriametrics" && s.vmParams.ExternalVM()) { + if tmpl.Name() == "" { continue }