From d5fe25c60fb78caca9158ad3c386eeb3ebf8bb8e Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 2 Mar 2023 01:00:22 +0300 Subject: [PATCH 01/77] PMM-9374 support of external victoria metrics. --- .../rpm/server/SPECS/victoriametrics.spec | 8 +- docker-compose.yml | 36 ++++++++- managed/cmd/pmm-managed/main.go | 27 ++++--- ...iametrics.go => victoriametrics_params.go} | 18 ++++- ...test.go => victoriametrics_params_test.go} | 4 +- managed/services/agents/state.go | 18 +++-- managed/services/agents/vmagent.go | 28 +++++-- managed/services/config/pmm-managed.yaml | 1 - managed/services/supervisord/supervisord.go | 34 +++++++- .../victoriametrics/victoriametrics.go | 77 +++++++++++++------ .../victoriametrics/victoriametrics_test.go | 6 +- managed/utils/envvars/parser.go | 6 ++ update/ansible/playbook/tasks/files/pmm.ini | 1 - 13 files changed, 201 insertions(+), 63 deletions(-) rename managed/models/{victoriametrics.go => victoriametrics_params.go} (81%) rename managed/models/{victoriametrics_test.go => victoriametrics_params_test.go} (90%) diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index 84f2b0a2c2..f8ebcb5366 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -35,11 +35,13 @@ export USER=builder make victoria-metrics-pure make vmalert-pure +make vmagent-pure %install install -D -p -m 0755 ./bin/victoria-metrics-pure %{buildroot}%{_sbindir}/victoriametrics install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert +install -D -p -m 0755 ./bin/vmagent-pure %{buildroot}%{_sbindir}/vmagent %files @@ -47,10 +49,14 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert %doc README.md %{_sbindir}/victoriametrics %{_sbindir}/vmalert +%{_sbindir}/vmagent %changelog -* Thu Oct 20 2022 Michal Kralik - 1.82.1 +* Thu Mar 2 2023 Nurlan Moldomurov - 1.82.1-2 +- add vmagent to PMM Server + +* Thu Oct 20 2022 Michal Kralik - 1.82.1-1 - upgrade victoriametrics to 1.82.1 release * Thu May 11 2022 Michael Okoko - 1.77.1 diff --git a/docker-compose.yml b/docker-compose.yml index ba9d0a0e99..a07fe19f11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,20 +73,46 @@ services: - go-modules:/root/go/pkg/mod - root-cache:/root/.cache - # PMM with external ClickHouse DB + # PMM with external DBs ch: profiles: - - pmm-ch + - pmm-external-dbs image: ${CH_IMAGE:-clickhouse/clickhouse-server:22.6.9.11-alpine} platform: linux/amd64 hostname: ${CH_HOSTNAME:-ch} ports: - ${CH_PORT:-9000}:9000 + networks: + - ${NETWORK:-default} + victoriametrics: + profiles: + - pmm-external-dbs + hostname: ${VM_HOSTNAME:-victoriametrics} + image: victoriametrics/victoria-metrics:v1.88.1 + ports: + - 8428:8428 + - 8089:8089 + - 8089:8089/udp + - 2003:2003 + - 2003:2003/udp + - 4242:4242 + volumes: + - vmdata:/storage + command: + - "--storageDataPath=/storage" + - "--graphiteListenAddr=:2003" + - "--opentsdbListenAddr=:4242" + - "--httpListenAddr=:8428" + - "--influxListenAddr=:8089" + - "--vmalert.proxyURL=http://vmalert:8880" + networks: + - ${NETWORK:-default} pmm-managed-server-ch: profiles: - - pmm-ch + - pmm-external-dbs depends_on: - ch + - victoriametrics image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-managed-server hostname: pmm-managed-server @@ -109,7 +135,8 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 -# - PMM_DEBUG=1 + - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} + - PMM_DEBUG=1 - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest extra_hosts: @@ -151,6 +178,7 @@ services: volumes: go-modules: + vmdata: {} root-cache: networks: diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index fae384b25e..a8f1727767 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -646,9 +646,9 @@ func main() { kingpin.Version(version.FullInfo()) kingpin.HelpFlag.Short('h') - victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL"). - Default("http://127.0.0.1:9090/prometheus/").String() - victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL"). + victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL").Envar("PMM_VM_URL"). + Default(models.VMBaseURL).String() + victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL").Envar("PMM_VM_ALERT_URL"). Default("http://127.0.0.1:8880/").String() victoriaMetricsConfigF := kingpin.Flag("victoriametrics-config", "VictoriaMetrics scrape configuration file path"). Default("/etc/victoriametrics-promscrape.yml").String() @@ -699,8 +699,8 @@ func main() { l.Panicf("Failed to load config: %+v", err) } ds := cfg.Config.Services.Telemetry.DataSources - pmmdb := ds.PmmDBSelect + pmmdb := ds.PmmDBSelect pmmdb.Credentials.Username = *postgresDBUsernameF pmmdb.Credentials.Password = *postgresDBPasswordF pmmdb.DSN.Scheme = "postgres" // TODO: should be configurable @@ -711,9 +711,17 @@ func main() { pmmdb.DSN.Params = q.Encode() clickhousedb := ds.QanDBSelect - clickhousedb.DSN = "tcp://" + *clickhouseAddrF + "/" + *clickHouseDatabaseF + ds.VM.Address = *victoriaMetricsURLF + + vmParams, err := models.NewVictoriaMetricsParams( + models.BasePrometheusConfigPath, + *victoriaMetricsURLF) + if err != nil { + l.Panicf("cannot load victoriametrics params problem: %+v", err) + } + sqlDB, err := models.OpenDB(*postgresAddrF, *postgresDBNameF, *postgresDBUsernameF, *postgresDBPasswordF) if err != nil { l.Panicf("Failed to connect to database: %+v", err) @@ -735,12 +743,7 @@ func main() { cleaner := clean.New(db) externalRules := vmalert.NewExternalRules() - - vmParams, err := models.NewVictoriaMetricsParams(victoriametrics.BasePrometheusConfigPath) - if err != nil { - l.Panicf("cannot load victoriametrics params problem: %+v", err) - } - vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, *victoriaMetricsURLF, vmParams) + vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, vmParams) if err != nil { l.Panicf("VictoriaMetrics service problem: %+v", err) } @@ -792,7 +795,7 @@ func main() { prom.MustRegister(grafanaClient) jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService) - agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb) + agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams) agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) actionsService := agents.NewActionsService(qanClient, agentsRegistry) diff --git a/managed/models/victoriametrics.go b/managed/models/victoriametrics_params.go similarity index 81% rename from managed/models/victoriametrics.go rename to managed/models/victoriametrics_params.go index d8b07caf27..8cda5ee576 100644 --- a/managed/models/victoriametrics.go +++ b/managed/models/victoriametrics_params.go @@ -23,18 +23,28 @@ import ( "gopkg.in/yaml.v3" ) +const ( + // BasePrometheusConfigPath - basic path with prometheus config, + // that user can mount to container. + BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml" + VMBaseURL = "http://127.0.0.1:9090/prometheus/" +) + // VictoriaMetricsParams - defines flags and settings for victoriametrics. type VictoriaMetricsParams struct { // VMAlertFlags additional flags for VMAlert. VMAlertFlags []string // BaseConfigPath defines path for basic prometheus config. BaseConfigPath string + // URL defines URL of Victoria Metrics + URL string } // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. -func NewVictoriaMetricsParams(basePath string) (*VictoriaMetricsParams, error) { +func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, + URL: vmURL, } if err := vmp.UpdateParams(); err != nil { return vmp, err @@ -59,7 +69,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { if !os.IsNotExist(err) { return errors.Wrap(err, "cannot read baseConfigPath for VMAlertParams") } - // fast return if users configuration doesn't exists with path + // fast return if users configuration doesn't exist with path // /srv/prometheus/prometheus.base.yml, // its maybe mounted into container by user. return nil @@ -79,3 +89,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { return nil } + +func (vmp *VictoriaMetricsParams) ExternalVM() bool { + return vmp.URL != VMBaseURL +} diff --git a/managed/models/victoriametrics_test.go b/managed/models/victoriametrics_params_test.go similarity index 90% rename from managed/models/victoriametrics_test.go rename to managed/models/victoriametrics_params_test.go index 02ab59b036..c52c349800 100644 --- a/managed/models/victoriametrics_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -23,11 +23,11 @@ import ( func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "", "", "") require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "", "", "") require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index 9ffddde95a..b784b2d540 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -41,17 +41,19 @@ const ( // StateUpdater handles updating status of agents. type StateUpdater struct { - db *reform.DB - r *Registry - vmdb prometheusService + db *reform.DB + r *Registry + vmdb prometheusService + vmParams *models.VictoriaMetricsParams } // NewStateUpdater creates new agent state updater. -func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService) *StateUpdater { +func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams *models.VictoriaMetricsParams) *StateUpdater { return &StateUpdater{ - db: db, - r: r, - vmdb: vmdb, + db: db, + r: r, + vmdb: vmdb, + vmParams: vmParams, } } @@ -181,7 +183,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI if err != nil { return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.id) } - agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg)) + agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg), u.vmParams) case models.NodeExporterType: node, err := models.FindNodeByID(u.db.Querier, pointer.GetString(row.NodeID)) diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index cb04ce25e7..7688ecd472 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -16,16 +16,25 @@ package agents import ( + "fmt" + "github.com/percona/pmm/managed/utils/envvars" + "os" "sort" + "strings" "github.com/percona/pmm/api/agentpb" "github.com/percona/pmm/api/inventorypb" + "github.com/percona/pmm/managed/models" ) // vmAgentConfig returns desired configuration of vmagent process. -func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { +func vmAgentConfig(scrapeCfg string, params *models.VictoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { + serverURL := "{{.server_url}}/victoriametrics/" + if params.ExternalVM() { + serverURL = params.URL + } args := []string{ - "-remoteWrite.url={{.server_url}}/victoriametrics/api/v1/write", + fmt.Sprintf("-remoteWrite.url=%sapi/v1/write", serverURL), "-remoteWrite.tlsInsecureSkipVerify={{.server_insecure}}", "-remoteWrite.tmpDataPath={{.tmp_dir}}/vmagent-temp-dir", "-promscrape.config={{.TextFiles.vmagentscrapecfg}}", @@ -35,13 +44,22 @@ func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { "-httpListenAddr=127.0.0.1:{{.listen_port}}", // needed for login/password at client side. "-envflag.enable=true", + "-envflag.prefix=VMAGENT_", } sort.Strings(args) - envs := []string{ - "remoteWrite_basicAuth_username={{.server_username}}", - "remoteWrite_basicAuth_password={{.server_password}}", + var envs []string + if !params.ExternalVM() { + envs = []string{ + "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}", + "VMAGENT_remoteWrite_basicAuth_password={{.server_password}}", + } + } + for _, env := range os.Environ() { + if strings.HasPrefix(env, envvars.ENVvmAgentPrefix) { + envs = append(envs, env) + } } sort.Strings(envs) diff --git a/managed/services/config/pmm-managed.yaml b/managed/services/config/pmm-managed.yaml index 8068096020..0ad29456b9 100644 --- a/managed/services/config/pmm-managed.yaml +++ b/managed/services/config/pmm-managed.yaml @@ -5,7 +5,6 @@ services: VM: enabled: true timeout: 2s - address: http://localhost:9090/prometheus QANDB_SELECT: enabled: true timeout: 2s diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 87a486becb..afae6f53b8 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -428,6 +428,8 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "DataRetentionDays": int(settings.DataRetention.Hours() / 24), "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, + "VMURL": s.vmParams.URL, + "EnableVMAgent": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, "ClickhouseAddr": clickhouseAddr, "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, @@ -613,7 +615,36 @@ stdout_logfile_backups = 3 redirect_stderr = true {{end}} +{{define "vmagent"}} +{{- if .EnableVMAgent }} +[program:vmagent] +priority = 7 +command = + /usr/sbin/vmagent + -remoteWrite.url={{ .VMURL }}api/v1/write + -remoteWrite.maxDiskUsagePerURL=1073741824 + -remoteWrite.tmpDataPath=/srv/vmagent/data + -promscrape.config=/etc/victoriametrics-promscrape.yml + -httpListenAddr=127.0.0.1:9091 + -loggerLevel=INFO + -envflag.enable + -envflag.prefix=VMAGENT_ +user = pmm +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = INT +stopwaitsecs = 300 +stdout_logfile = /srv/logs/vmagent.log +stdout_logfile_maxbytes = 10MB +stdout_logfile_backups = 3 +redirect_stderr = true +{{end}} +{{end}} + {{define "victoriametrics"}} +{{- if not .EnableVMAgent }} [program:victoriametrics] priority = 7 command = @@ -647,6 +678,7 @@ stdout_logfile_maxbytes = 10MB stdout_logfile_backups = 3 redirect_stderr = true {{end}} +{{end}} {{define "vmalert"}} [program:vmalert] @@ -685,7 +717,7 @@ redirect_stderr = true priority = 9 command = /usr/sbin/vmproxy - --target-url=http://127.0.0.1:9090/ + --target-url={{ .VMURL }} --listen-port=8430 --listen-address=127.0.0.1 --header-name=X-Proxy-Filter diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 1456d2cbad..43b536287f 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -46,10 +46,7 @@ const ( updateBatchDelay = time.Second configurationUpdateTimeout = 3 * time.Second - // BasePrometheusConfigPath - basic path with prometheus config, - // that user can mount to container. - BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml" - + vmagentDir = "/srv/vmagent" victoriametricsDir = "/srv/victoriametrics" victoriametricsDataDir = "/srv/victoriametrics/data" dirPerm = os.FileMode(0o775) @@ -64,15 +61,19 @@ type Service struct { baseURL *url.URL client *http.Client - baseConfigPath string // for testing + params *models.VictoriaMetricsParams l *logrus.Entry reloadCh chan struct{} } // NewVictoriaMetrics creates new VictoriaMetrics service. -func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, baseURL string, params *models.VictoriaMetricsParams) (*Service, error) { - u, err := url.Parse(baseURL) +func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { + internalVMURL := params.URL + if params.ExternalVM() { + internalVMURL = "http://127.0.0.1:9091/" + } + u, err := url.Parse(internalVMURL) if err != nil { return nil, errors.WithStack(err) } @@ -82,7 +83,7 @@ func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, baseURL string, db: db, baseURL: u, client: &http.Client{}, // TODO instrument with utils/irt; see vmalert package https://jira.percona.com/browse/PMM-7229 - baseConfigPath: params.BaseConfigPath, + params: params, l: logrus.WithField("component", "victoriametrics"), reloadCh: make(chan struct{}, 1), }, nil @@ -95,12 +96,17 @@ func (svc *Service) Run(ctx context.Context) { svc.l.Info("Starting...") defer svc.l.Info("Done.") - - if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } - if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) + if svc.params.ExternalVM() { + if err := dir.CreateDataDir(vmagentDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } + } else { + if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } + if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } } // reloadCh, configuration update loop, and RequestConfigurationUpdate method ensure that configuration @@ -181,18 +187,18 @@ func (svc *Service) reload(ctx context.Context) error { return errors.WithStack(err) } - if resp.StatusCode != http.StatusNoContent { - return errors.Errorf("expected 204, got %d", resp.StatusCode) + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return errors.Errorf("expected 200 or 204, got %d", resp.StatusCode) } return nil } // loadBaseConfig returns parsed base configuration file, or empty configuration on error. func (svc *Service) loadBaseConfig() *config.Config { - buf, err := os.ReadFile(svc.baseConfigPath) + buf, err := os.ReadFile(svc.params.BaseConfigPath) if err != nil { if !os.IsNotExist(err) { - svc.l.Errorf("Failed to load base VictoriaMetrics config %s: %s", svc.baseConfigPath, err) + svc.l.Errorf("Failed to load base VictoriaMetrics config %s: %s", svc.params.BaseConfigPath, err) } return &config.Config{} @@ -200,7 +206,7 @@ func (svc *Service) loadBaseConfig() *config.Config { var cfg config.Config if err := yaml.Unmarshal(buf, &cfg); err != nil { - svc.l.Errorf("Failed to parse base VictoriaMetrics config %s: %s.", svc.baseConfigPath, err) + svc.l.Errorf("Failed to parse base VictoriaMetrics config %s: %s.", svc.params.BaseConfigPath, err) return &config.Config{} } @@ -330,7 +336,14 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR)) + u, err := url.Parse(svc.params.URL) + if err != nil { + return err + } + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u.Host)) + if svc.params.ExternalVM() { + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) + } cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVMAlert(s.HR)) AddInternalServicesToScrape(cfg, s, settings.DBaaS.Enabled) return AddScrapeConfigs(svc.l, cfg, tx.Querier, &s, nil, false) @@ -338,7 +351,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration) *config.ScrapeConfig { +func scrapeConfigForVictoriaMetrics(interval time.Duration, target string) *config.ScrapeConfig { return &config.ScrapeConfig{ JobName: "victoriametrics", ScrapeInterval: config.Duration(interval), @@ -347,8 +360,26 @@ func scrapeConfigForVictoriaMetrics(interval time.Duration) *config.ScrapeConfig ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ StaticConfigs: []*config.Group{ { - Targets: []string{"127.0.0.1:9090"}, - Labels: map[string]string{"instance": "pmm-server"}, + Targets: []string{target}, + Labels: map[string]string{"instance": models.PMMServerAgentID}, + }, + }, + }, + } +} + +// scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. +func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { + return &config.ScrapeConfig{ + JobName: "vmagent", + ScrapeInterval: config.Duration(interval), + ScrapeTimeout: ScrapeTimeout(interval), + MetricsPath: "/metrics", + ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ + StaticConfigs: []*config.Group{ + { + Targets: []string{target}, + Labels: map[string]string{"instance": models.PMMServerAgentID}, }, }, }, diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 811f1b6d83..de824cb29c 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,8 +43,8 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml"} - svc, err := NewVictoriaMetrics(configPath, db, "http://127.0.0.1:9090/prometheus/", vmParams) + vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml", URL: "http://127.0.0.1:9090/prometheus/"} + svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) original, err := os.ReadFile(configPath) @@ -801,7 +801,7 @@ func TestBaseConfig(t *testing.T) { db, svc, original := setup(t) defer teardown(t, db, svc, original) - svc.baseConfigPath = "../../testdata/victoriametrics/promscrape.base.yml" + svc.params.BaseConfigPath = "../../testdata/victoriametrics/promscrape.base.yml" expected := strings.TrimSpace(` # Managed by pmm-managed. DO NOT EDIT. diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index ea4112798a..2cbb0a53b6 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -41,6 +41,7 @@ const ( envEnableAccessControl = "ENABLE_RBAC" envPlatformAPITimeout = "PERCONA_PLATFORM_API_TIMEOUT" defaultPlatformAPITimeout = 30 * time.Second + ENVvmAgentPrefix = "VMAGENT_" ) // InvalidDurationError invalid duration error. @@ -202,6 +203,11 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs continue } + // skip VM Agents environment variables + if strings.HasPrefix(k, ENVvmAgentPrefix) { + continue + } + // skip supervisord environment variables if strings.HasPrefix(k, "SUPERVISOR_") { continue diff --git a/update/ansible/playbook/tasks/files/pmm.ini b/update/ansible/playbook/tasks/files/pmm.ini index e82339638f..e9c467e498 100644 --- a/update/ansible/playbook/tasks/files/pmm.ini +++ b/update/ansible/playbook/tasks/files/pmm.ini @@ -96,7 +96,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --postgres-name=pmm-managed --postgres-username=pmm-managed --postgres-password=pmm-managed From 8400da1eccfa38db1d9e9747c9cca3d3f0e7cb17 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 7 Mar 2023 01:22:02 +0600 Subject: [PATCH 02/77] PMM-9374 update version number of victoria metrics. --- build/packages/rpm/server/SPECS/victoriametrics.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index f8ebcb5366..050f87e970 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -13,7 +13,7 @@ Name: percona-victoriametrics Version: 1.82.1 -Release: 1%{?dist} +Release: 2%{?dist} Summary: VictoriaMetrics monitoring solution and time series database License: Apache-2.0 URL: https://%{provider} From c1ceff0e62b3ff150f769196c34e9b386fafd9ff Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 13 Mar 2023 18:54:03 +0600 Subject: [PATCH 03/77] PMM-9374 vmalert. --- docker-compose.yml | 1 - managed/services/supervisord/logs.go | 8 ++++++-- managed/services/supervisord/supervisord.go | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a07fe19f11..9229b902f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,7 +104,6 @@ services: - "--opentsdbListenAddr=:4242" - "--httpListenAddr=:8428" - "--influxListenAddr=:8089" - - "--vmalert.proxyURL=http://vmalert:8880" networks: - ${NETWORK:-default} pmm-managed-server-ch: diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index c6d94a0258..9efb941108 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -25,6 +25,7 @@ import ( "io" "mime" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -37,6 +38,7 @@ import ( "golang.org/x/sys/unix" "gopkg.in/yaml.v3" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/logger" pprofUtils "github.com/percona/pmm/managed/utils/pprof" "github.com/percona/pmm/utils/pdeathsig" @@ -59,14 +61,16 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker + vmParams *models.VictoriaMetricsParams } // NewLogs creates a new Logs service. // n is a number of last lines of log to read. -func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs { +func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams *models.VictoriaMetricsParams) *Logs { return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, + vmParams: vmParams, } } @@ -213,7 +217,7 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten Data: b, Err: err, }) - + url.Parse(l.vmParams.URL) // add VictoriaMetrics targets b, err = readURL(ctx, "http://127.0.0.1:9090/prometheus/api/v1/targets") files = append(files, fileContent{ diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index afae6f53b8..63944ed36e 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -688,11 +688,11 @@ command = --notifier.url="{{ .AlertmanagerURL }}" --notifier.basicAuth.password='{{ .AlertManagerPassword }}' --notifier.basicAuth.username="{{ .AlertManagerUser }}" - --external.url=http://localhost:9090/prometheus - --datasource.url=http://127.0.0.1:9090/prometheus - --remoteRead.url=http://127.0.0.1:9090/prometheus + --external.url={{ .VMURL }} + --datasource.url={{ .VMURL }} + --remoteRead.url={{ .VMURL }} --remoteRead.ignoreRestoreErrors=false - --remoteWrite.url=http://127.0.0.1:9090/prometheus + --remoteWrite.url={{ .VMURL }} --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml --httpListenAddr=127.0.0.1:8880 From ed017ba8502b3d68de9b10c2eb939c6ffd01ca44 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 15 Mar 2023 17:57:12 +0600 Subject: [PATCH 04/77] PMM-9374 use pmm-clients vmagent instead of a separate one. --- .../rpm/server/SPECS/victoriametrics.spec | 10 +--- managed/cmd/pmm-managed/main.go | 4 +- managed/services/agents/registry.go | 20 ++++--- managed/services/supervisord/logs.go | 16 +++++- managed/services/supervisord/supervisord.go | 28 ---------- .../victoriametrics/victoriametrics.go | 52 +++++++++++-------- 6 files changed, 59 insertions(+), 71 deletions(-) diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index 050f87e970..84f2b0a2c2 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -13,7 +13,7 @@ Name: percona-victoriametrics Version: 1.82.1 -Release: 2%{?dist} +Release: 1%{?dist} Summary: VictoriaMetrics monitoring solution and time series database License: Apache-2.0 URL: https://%{provider} @@ -35,13 +35,11 @@ export USER=builder make victoria-metrics-pure make vmalert-pure -make vmagent-pure %install install -D -p -m 0755 ./bin/victoria-metrics-pure %{buildroot}%{_sbindir}/victoriametrics install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert -install -D -p -m 0755 ./bin/vmagent-pure %{buildroot}%{_sbindir}/vmagent %files @@ -49,14 +47,10 @@ install -D -p -m 0755 ./bin/vmagent-pure %{buildroot}%{_sbindir}/vmagent %doc README.md %{_sbindir}/victoriametrics %{_sbindir}/vmalert -%{_sbindir}/vmagent %changelog -* Thu Mar 2 2023 Nurlan Moldomurov - 1.82.1-2 -- add vmagent to PMM Server - -* Thu Oct 20 2022 Michal Kralik - 1.82.1-1 +* Thu Oct 20 2022 Michal Kralik - 1.82.1 - upgrade victoriametrics to 1.82.1 release * Thu May 11 2022 Michael Okoko - 1.77.1 diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index a8f1727767..34c8df55ad 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -757,7 +757,7 @@ func main() { qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) - agentsRegistry := agents.NewRegistry(db) + agentsRegistry := agents.NewRegistry(db, vmParams.ExternalVM()) backupRemovalService := backup.NewRemovalService(db, minioClient) pitrTimerangeService := backup.NewPITRTimerangeService(minioClient) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) @@ -772,7 +772,7 @@ func main() { pmmUpdateCheck := supervisord.NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker")) - logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck) + logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck, vmParams) supervisord := supervisord.New(*supervisordConfigDirF, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) platformAddress, err := envvars.GetPlatformAddress() diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 28ea5965f6..00caf4f348 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -85,10 +85,12 @@ type Registry struct { mRoundTrip prom.Summary mClockDrift prom.Summary mAgents prom.GaugeFunc + + isExternalVM bool } // NewRegistry creates a new registry with given database connection. -func NewRegistry(db *reform.DB) *Registry { +func NewRegistry(db *reform.DB, isExternalVM bool) *Registry { agents := make(map[string]*pmmAgentInfo) r := &Registry{ db: db, @@ -123,6 +125,8 @@ func NewRegistry(db *reform.DB) *Registry { Help: "Clock drift.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }), + + isExternalVM: isExternalVM, } r.mAgents = prom.NewGaugeFunc(prom.GaugeOpts{ @@ -160,7 +164,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, } var node *models.Node err = r.db.InTransaction(func(tx *reform.TX) error { - node, err = authenticate(agentMD, tx.Querier) + node, err = r.authenticate(agentMD, tx.Querier) if err != nil { return err } @@ -204,7 +208,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, return agent, nil } -func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) { +func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) { if md.ID == "" { return nil, status.Error(codes.PermissionDenied, "Empty Agent ID.") } @@ -233,7 +237,7 @@ func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models. return nil, status.Errorf(codes.InvalidArgument, "Can't parse 'version' for pmm-agent with ID %q.", md.ID) } - if err := addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil { + if err := r.addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil { return nil, err } @@ -296,18 +300,18 @@ func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error { // addOrRemoveVMAgent - creates vmAgent agentType if pmm-agent's version supports it and agent not exists yet, // otherwise ensures that vmAgent not exist for pmm-agent and pmm-agent's agents don't have push_metrics mode, // removes it if needed. -func addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error { +func (r *Registry) addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error { if pmmAgentVersion.Less(models.PMMAgentWithPushMetricsSupport) { // ensure that vmagent not exists and agents dont have push_metrics. return removeVMAgentFromPMMAgent(q, pmmAgentID) } - return addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID) + return r.addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID) } -func addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error { +func (r *Registry) addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error { // TODO remove it after fix // https://jira.percona.com/browse/PMM-4420 - if runsOnNodeID == "pmm-server" { + if runsOnNodeID == "pmm-server" && !r.isExternalVM { return nil } vmAgentType := models.VMAgentType diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 9efb941108..207d5121ee 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -217,9 +217,9 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten Data: b, Err: err, }) - url.Parse(l.vmParams.URL) + // add VictoriaMetrics targets - b, err = readURL(ctx, "http://127.0.0.1:9090/prometheus/api/v1/targets") + b, err = l.victoriaMetricsTargets(ctx) files = append(files, fileContent{ Name: "victoriametrics_targets.json", Data: b, @@ -284,6 +284,18 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten return files } +func (l *Logs) victoriaMetricsTargets(ctx context.Context) ([]byte, error) { + vmURL, err := url.Parse(l.vmParams.URL) + if err != nil { + return nil, err + } + targetsURL, err := vmURL.Parse("api/v1/targets") + if err != nil { + return nil, err + } + return readURL(ctx, targetsURL.String()) +} + // readLog reads last lines (up to given number of lines and bytes) from given file, // and returns them together with modification time. func readLog(name string, maxLines int, maxBytes int64) ([]byte, time.Time, error) { diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 63944ed36e..a7e5adfcc2 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -615,34 +615,6 @@ stdout_logfile_backups = 3 redirect_stderr = true {{end}} -{{define "vmagent"}} -{{- if .EnableVMAgent }} -[program:vmagent] -priority = 7 -command = - /usr/sbin/vmagent - -remoteWrite.url={{ .VMURL }}api/v1/write - -remoteWrite.maxDiskUsagePerURL=1073741824 - -remoteWrite.tmpDataPath=/srv/vmagent/data - -promscrape.config=/etc/victoriametrics-promscrape.yml - -httpListenAddr=127.0.0.1:9091 - -loggerLevel=INFO - -envflag.enable - -envflag.prefix=VMAGENT_ -user = pmm -autorestart = true -autostart = true -startretries = 10 -startsecs = 1 -stopsignal = INT -stopwaitsecs = 300 -stdout_logfile = /srv/logs/vmagent.log -stdout_logfile_maxbytes = 10MB -stdout_logfile_backups = 3 -redirect_stderr = true -{{end}} -{{end}} - {{define "victoriametrics"}} {{- if not .EnableVMAgent }} [program:victoriametrics] diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 43b536287f..f4347b6a4d 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -46,7 +46,6 @@ const ( updateBatchDelay = time.Second configurationUpdateTimeout = 3 * time.Second - vmagentDir = "/srv/vmagent" victoriametricsDir = "/srv/victoriametrics" victoriametricsDataDir = "/srv/victoriametrics/data" dirPerm = os.FileMode(0o775) @@ -69,11 +68,7 @@ type Service struct { // NewVictoriaMetrics creates new VictoriaMetrics service. func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { - internalVMURL := params.URL - if params.ExternalVM() { - internalVMURL = "http://127.0.0.1:9091/" - } - u, err := url.Parse(internalVMURL) + u, err := url.Parse(params.URL) if err != nil { return nil, errors.WithStack(err) } @@ -96,17 +91,12 @@ func (svc *Service) Run(ctx context.Context) { svc.l.Info("Starting...") defer svc.l.Info("Done.") - if svc.params.ExternalVM() { - if err := dir.CreateDataDir(vmagentDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } - } else { - if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } - if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { - svc.l.Error(err) - } + + if err := dir.CreateDataDir(victoriametricsDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) + } + if err := dir.CreateDataDir(victoriametricsDataDir, "pmm", "pmm", dirPerm); err != nil { + svc.l.Error(err) } // reloadCh, configuration update loop, and RequestConfigurationUpdate method ensure that configuration @@ -151,6 +141,9 @@ func (svc *Service) RequestConfigurationUpdate() { // updateConfiguration updates VictoriaMetrics configuration. func (svc *Service) updateConfiguration(ctx context.Context) error { + if svc.params.ExternalVM() { + return nil + } start := time.Now() defer func() { if dur := time.Since(start); dur > time.Second { @@ -158,8 +151,7 @@ func (svc *Service) updateConfiguration(ctx context.Context) error { } }() - base := svc.loadBaseConfig() - cfg, err := svc.marshalConfig(base) + cfg, err := svc.buildVMConfig() if err != nil { return err } @@ -167,6 +159,11 @@ func (svc *Service) updateConfiguration(ctx context.Context) error { return svc.configAndReload(ctx, cfg) } +func (svc *Service) buildVMConfig() ([]byte, error) { + base := svc.loadBaseConfig() + return svc.marshalConfig(base) +} + // reload asks VictoriaMetrics to reload configuration. func (svc *Service) reload(ctx context.Context) error { u := *svc.baseURL @@ -340,7 +337,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if err != nil { return err } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u.Host)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u)) if svc.params.ExternalVM() { cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) } @@ -351,16 +348,18 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration, target string) *config.ScrapeConfig { +func scrapeConfigForVictoriaMetrics(interval time.Duration, target *url.URL) *config.ScrapeConfig { + target, _ = target.Parse("metrics") + return &config.ScrapeConfig{ JobName: "victoriametrics", ScrapeInterval: config.Duration(interval), ScrapeTimeout: ScrapeTimeout(interval), - MetricsPath: "/prometheus/metrics", + MetricsPath: target.Path, ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ StaticConfigs: []*config.Group{ { - Targets: []string{target}, + Targets: []string{target.Host}, Labels: map[string]string{"instance": models.PMMServerAgentID}, }, }, @@ -406,6 +405,9 @@ func scrapeConfigForVMAlert(interval time.Duration) *config.ScrapeConfig { // BuildScrapeConfigForVMAgent builds scrape configuration for given pmm-agent. func (svc *Service) BuildScrapeConfigForVMAgent(pmmAgentID string) ([]byte, error) { + if pmmAgentID == models.PMMServerAgentID { + return svc.buildVMConfig() + } var cfg config.Config e := svc.db.InTransaction(func(tx *reform.TX) error { settings, err := models.GetSettings(tx) @@ -424,6 +426,10 @@ func (svc *Service) BuildScrapeConfigForVMAgent(pmmAgentID string) ([]byte, erro // IsReady verifies that VictoriaMetrics works. func (svc *Service) IsReady(ctx context.Context) error { + if svc.params.ExternalVM() { + svc.l.Debugf("External VM is used, VM healthcheck is skipped") + return nil + } u := *svc.baseURL u.Path = path.Join(u.Path, "health") req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) From 36182b45acc626784565a91fa35361be346e9603 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 29 Mar 2023 15:16:27 +0300 Subject: [PATCH 05/77] PMM-9374 Fix linters. --- managed/services/agents/vmagent.go | 1 - managed/services/supervisord/logs.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index 0934a29634..1740cac088 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -17,7 +17,6 @@ package agents import ( "fmt" - "github.com/percona/pmm/managed/utils/envvars" "os" "sort" "strings" diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 7e58274e14..9748cdf695 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -61,7 +61,7 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker - vmParams *models.VictoriaMetricsParams + vmParams *models.VictoriaMetricsParams } // NewLogs creates a new Logs service. @@ -70,7 +70,7 @@ func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams *mo return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, - vmParams: vmParams, + vmParams: vmParams, } } From b46395af41ff7f2350aa6b6987b2dfbdc99a9eb0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 29 Mar 2023 16:02:17 +0300 Subject: [PATCH 06/77] PMM-9374 Fix tests. --- managed/models/victoriametrics_params_test.go | 4 ++-- managed/services/agents/vmagent_test.go | 10 ++++++++-- managed/services/supervisord/logs_test.go | 9 +++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index c52c349800..6362299128 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -23,11 +23,11 @@ import ( func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "", "", "") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "") require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "", "", "") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "") require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 2adc5fded8..3a1431ee1b 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -16,6 +16,8 @@ package agents import ( + "github.com/percona/pmm/managed/models" + "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/assert" @@ -23,13 +25,17 @@ import ( func TestMaxScrapeSize(t *testing.T) { t.Run("by default 64MiB", func(t *testing.T) { - actual := vmAgentConfig("") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) + actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+maxScrapeSizeDefault) }) t.Run("overridden with ENV", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) newValue := "16MiB" t.Setenv(maxScrapeSizeEnv, newValue) - actual := vmAgentConfig("") + actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+newValue) }) } diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index 265e17c112..02f9f52ff7 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "github.com/percona/pmm/managed/models" "os" "path/filepath" "sort" @@ -122,7 +123,9 @@ func TestAddAdminSummary(t *testing.T) { func TestFiles(t *testing.T) { checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - l := NewLogs("2.4.5", checker) + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) + l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) files := l.files(ctx, nil) @@ -160,7 +163,9 @@ func TestZip(t *testing.T) { t.Skip("FIXME") checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - l := NewLogs("2.4.5", checker) + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + require.NoError(t, err) + l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) var buf bytes.Buffer From 7fe77aa55fc64683c99f0bb5b7ab43a031474f6d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 29 Mar 2023 22:59:42 +0300 Subject: [PATCH 07/77] PMM-9374 Improve work with VM Proxy. --- docker-compose.yml | 7 ------- managed/services/agents/vmagent_test.go | 9 +++++---- managed/services/supervisord/logs_test.go | 4 ++-- managed/services/supervisord/supervisord.go | 2 +- managed/services/supervisord/supervisord_test.go | 4 ++-- managed/testdata/supervisord.d/vmalert.ini | 8 ++++---- managed/testdata/supervisord.d/vmproxy.ini | 2 +- update/ansible/playbook/tasks/files/datasources.yml | 2 +- vmproxy/Makefile | 2 +- vmproxy/proxy/proxy.go | 7 +++++++ 10 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fda1cb6962..5eb978112e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,13 +164,6 @@ services: - ${PMM_PORT_HTTPS:-443}:443 # For headless delve - ${PMM_PORT_DELVE:-2345}:2345 - # PG - - ${PMM_PORT_PG:-15432}:5432 - # VM - - ${PMM_PORT_VM:-9090}:9090 - # CH - - ${PMM_PORT_CH_TCP:-11000}:9000 - - ${PMM_PORT_CH_HTTP:-11123}:8123 volumes: - ./:/root/go/src/github.com/percona/pmm # - "../grafana/public:/usr/share/grafana/public" diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 3a1431ee1b..6522629ab4 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -16,22 +16,23 @@ package agents import ( - "github.com/percona/pmm/managed/models" - "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/percona/pmm/managed/models" ) func TestMaxScrapeSize(t *testing.T) { t.Run("by default 64MiB", func(t *testing.T) { - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+maxScrapeSizeDefault) }) t.Run("overridden with ENV", func(t *testing.T) { - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) newValue := "16MiB" t.Setenv(maxScrapeSizeEnv, newValue) diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index 02f9f52ff7..f9b2d19f7a 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "fmt" - "github.com/percona/pmm/managed/models" "os" "path/filepath" "sort" @@ -32,6 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/logger" ) @@ -163,7 +163,7 @@ func TestZip(t *testing.T) { t.Skip("FIXME") checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 31b56a951b..1a1c854b52 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -654,7 +654,7 @@ stdout_logfile = /srv/logs/victoriametrics.log stdout_logfile_maxbytes = 10MB stdout_logfile_backups = 3 redirect_stderr = true -{{end}} +{{end -}} {{end}} {{define "vmalert"}} diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index aa2b2e90d6..86578c5d1e 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -36,7 +36,7 @@ func TestConfig(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{} + vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, @@ -67,7 +67,7 @@ func TestDBaaSController(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{} + vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) var tp *template.Template diff --git a/managed/testdata/supervisord.d/vmalert.ini b/managed/testdata/supervisord.d/vmalert.ini index e784596bcb..40f61b1d83 100644 --- a/managed/testdata/supervisord.d/vmalert.ini +++ b/managed/testdata/supervisord.d/vmalert.ini @@ -7,11 +7,11 @@ command = --notifier.url="http://127.0.0.1:9093/alertmanager,https://external-alertmanager:6443/alerts" --notifier.basicAuth.password=',"passw!,ord"' --notifier.basicAuth.username=",external-user" - --external.url=http://localhost:9090/prometheus - --datasource.url=http://127.0.0.1:9090/prometheus - --remoteRead.url=http://127.0.0.1:9090/prometheus + --external.url=http://127.0.0.1:9090/prometheus/ + --datasource.url=http://127.0.0.1:9090/prometheus/ + --remoteRead.url=http://127.0.0.1:9090/prometheus/ --remoteRead.ignoreRestoreErrors=false - --remoteWrite.url=http://127.0.0.1:9090/prometheus + --remoteWrite.url=http://127.0.0.1:9090/prometheus/ --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml --httpListenAddr=127.0.0.1:8880 diff --git a/managed/testdata/supervisord.d/vmproxy.ini b/managed/testdata/supervisord.d/vmproxy.ini index 72acc05eaf..5654b85bc4 100644 --- a/managed/testdata/supervisord.d/vmproxy.ini +++ b/managed/testdata/supervisord.d/vmproxy.ini @@ -4,7 +4,7 @@ priority = 9 command = /usr/sbin/vmproxy - --target-url=http://127.0.0.1:9090/ + --target-url=http://127.0.0.1:9090/prometheus/ --listen-port=8430 --listen-address=127.0.0.1 --header-name=X-Proxy-Filter diff --git a/update/ansible/playbook/tasks/files/datasources.yml b/update/ansible/playbook/tasks/files/datasources.yml index 733ea6679d..2145e92684 100644 --- a/update/ansible/playbook/tasks/files/datasources.yml +++ b/update/ansible/playbook/tasks/files/datasources.yml @@ -5,7 +5,7 @@ datasources: orgId: 1 type: prometheus access: proxy - url: http://127.0.0.1:8430/prometheus/ + url: http://127.0.0.1:8430/ isDefault: true jsonData: httpMethod: POST diff --git a/vmproxy/Makefile b/vmproxy/Makefile index 458aea2543..de22f04d71 100644 --- a/vmproxy/Makefile +++ b/vmproxy/Makefile @@ -16,7 +16,7 @@ ifeq ($(GOBIN),) endif LD_FLAGS = -ldflags " \ - -X 'github.com/percona/pmm/version.ProjectName=pmm-admin' \ + -X 'github.com/percona/pmm/version.ProjectName=vmproxy' \ -X 'github.com/percona/pmm/version.Version=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.PMMVersion=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.Timestamp=$(PMM_RELEASE_TIMESTAMP)' \ diff --git a/vmproxy/proxy/proxy.go b/vmproxy/proxy/proxy.go index 907043345b..8d7aec1c8a 100644 --- a/vmproxy/proxy/proxy.go +++ b/vmproxy/proxy/proxy.go @@ -24,6 +24,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "strings" "time" "github.com/sirupsen/logrus" @@ -83,6 +84,12 @@ func director(target *url.URL, headerName string) func(*http.Request) { req.URL.Scheme = target.Scheme req.URL.Host = target.Host + rp, err := target.Parse(strings.TrimPrefix(req.URL.Path, "/")) + if err != nil { + logrus.Error(err) + } + req.URL.Path = rp.Path + // Replace extra filters if present if filters := req.Header.Get(headerName); filters != "" { q := req.URL.Query() From 2ec6403b1b4da855f658091bbc11e5dd283e35c7 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 11:54:33 +0300 Subject: [PATCH 08/77] PMM-9374 Fix tests. --- managed/models/victoriametrics_params_test.go | 4 ++-- managed/services/supervisord/devcontainer_test.go | 2 +- managed/services/supervisord/logs_test.go | 2 +- managed/services/victoriametrics/victoriametrics_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index 6362299128..4c9b80a3f3 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -23,11 +23,11 @@ import ( func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", "") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", VMBaseURL) require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", "") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", VMBaseURL) require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index 0a7604714c..c902faf6c5 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -112,7 +112,7 @@ func TestDevContainer(t *testing.T) { t.Run("UpdateConfiguration", func(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - vmParams := &models.VictoriaMetricsParams{} + vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} s := New("/etc/supervisord.d", checker, vmParams, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index f9b2d19f7a..2f51bc4097 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -123,7 +123,7 @@ func TestAddAdminSummary(t *testing.T) { func TestFiles(t *testing.T) { checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index de824cb29c..780fad011d 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,7 +43,7 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml", URL: "http://127.0.0.1:9090/prometheus/"} + vmParams := &models.VictoriaMetricsParams{BaseConfigPath: models.BasePrometheusConfigPath, URL: models.VMBaseURL} svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) From 91c83c4afa208bfae3108a3d613d6d0c900dc2d2 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 15:14:50 +0300 Subject: [PATCH 09/77] PMM-9374 Fix tests. --- managed/models/victoriametrics_params.go | 3 +- managed/services/supervisord/supervisord.go | 15 +++++----- .../victoriametrics/victoriametrics.go | 30 +++++++++++++------ .../victoriametrics/victoriametrics_test.go | 30 ++++++++++++++++--- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 8cda5ee576..1693fa6a6b 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -17,6 +17,7 @@ package models import ( "os" + "strings" config "github.com/percona/promconfig" "github.com/pkg/errors" @@ -91,5 +92,5 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { } func (vmp *VictoriaMetricsParams) ExternalVM() bool { - return vmp.URL != VMBaseURL + return !strings.HasPrefix(vmp.URL, "http://127.0.0.1") } diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 1a1c854b52..f4fadab40c 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -430,8 +430,9 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, "VMURL": s.vmParams.URL, - "EnableVMAgent": s.vmParams.ExternalVM(), + "ExternalVM": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, + "InterfaceToBind": envvars.GetInterfaceToBind(), "ClickhouseAddr": clickhouseAddr, "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, "ClickhouseDatabase": clickhouseDatabase, @@ -583,8 +584,6 @@ func (s *Service) RestartSupervisedService(serviceName string) error { return err } -var interfaceToBind = envvars.GetInterfaceToBind() - //nolint:lll var templates = template.Must(template.New("").Option("missingkey=error").Parse(` {{define "dbaas-controller"}} @@ -621,7 +620,7 @@ redirect_stderr = true {{end}} {{define "victoriametrics"}} -{{- if not .EnableVMAgent }} +{{- if not .ExternalVM }} [program:victoriametrics] priority = 7 command = @@ -629,7 +628,7 @@ command = --promscrape.config=/etc/victoriametrics-promscrape.yml --retentionPeriod={{ .DataRetentionDays }}d --storageDataPath=/srv/victoriametrics/data - --httpListenAddr=` + interfaceToBind + `:9090 + --httpListenAddr={{ .InterfaceToBind }}:9090 --search.disableCache={{ .VMDBCacheDisable }} --search.maxQueryLen=1MB --search.latencyOffset=5s @@ -672,7 +671,7 @@ command = --remoteWrite.url={{ .VMURL }} --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml - --httpListenAddr=` + interfaceToBind + `:8880 + --httpListenAddr={{ .InterfaceToBind }}:8880 {{- range $index, $param := .VMAlertFlags }} {{ $param }} {{- end }} @@ -696,7 +695,7 @@ command = /usr/sbin/vmproxy --target-url={{ .VMURL }} --listen-port=8430 - --listen-address=` + interfaceToBind + ` + --listen-address={{ .InterfaceToBind }} --header-name=X-Proxy-Filter user = pmm autorestart = true @@ -720,7 +719,7 @@ command = --storage.path=/srv/alertmanager/data --data.retention={{ .DataRetentionHours }}h --web.external-url=http://localhost:9093/alertmanager/ - --web.listen-address=` + interfaceToBind + `:9093 + --web.listen-address={{ .InterfaceToBind }}:9093 --cluster.listen-address="" user = pmm autorestart = true diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 6e32927f5f..c627843425 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -25,6 +25,7 @@ import ( "os/exec" "path" "regexp" + "strings" "time" "github.com/AlekSi/pointer" @@ -184,8 +185,8 @@ func (svc *Service) reload(ctx context.Context) error { return errors.WithStack(err) } - if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { - return errors.Errorf("expected 200 or 204, got %d", resp.StatusCode) + if resp.StatusCode != http.StatusNoContent { + return errors.Errorf("expected 204, got %d", resp.StatusCode) } return nil } @@ -333,11 +334,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - u, err := url.Parse(svc.params.URL) - if err != nil { - return err - } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR, u)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params.URL)) if svc.params.ExternalVM() { cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) } @@ -348,8 +345,12 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration, target *url.URL) *config.ScrapeConfig { - target, _ = target.Parse("metrics") +func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, baseURL string) *config.ScrapeConfig { + target, err := relativePath(baseURL, "metrics") + if err != nil { + l.Errorf("couldn't parse relative path to victoria metrics: %q", err) + return nil + } return &config.ScrapeConfig{ JobName: "victoriametrics", @@ -367,6 +368,17 @@ func scrapeConfigForVictoriaMetrics(interval time.Duration, target *url.URL) *co } } +func relativePath(baseURL string, path string) (*url.URL, error) { + if !strings.HasSuffix(baseURL, "/") { + baseURL = baseURL + "/" + } + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + return u.Parse(path) +} + // scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { return &config.ScrapeConfig{ diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 780fad011d..b262d6fcf6 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -18,10 +18,7 @@ package victoriametrics import ( "context" "database/sql" - "os" - "strings" - "testing" - + "fmt" "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,6 +26,9 @@ import ( "google.golang.org/grpc/status" "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" + "os" + "strings" + "testing" "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/testdb" @@ -893,3 +893,25 @@ scrape_configs: assert.NoError(t, err) assert.Equal(t, expected, string(newcfg), "actual:\n%s", newcfg) } + +func Test_relativePath(t *testing.T) { + tests := []struct { + baseURL string + path string + want string + }{ + { + "http://127.0.0.1:9090/prometheus", + "metrics", + "http://127.0.0.1:9090/prometheus/metrics", + }, + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s", tt.want), func(t *testing.T) { + got, err := relativePath(tt.baseURL, tt.path) + assert.NoError(t, err) + assert.Equalf(t, tt.want, got.String(), "relativePath(%v, %v)", tt.baseURL, tt.path) + }) + } +} From 64b37396fb8a8e2f7cff3cc64ff157439423fd33 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 15:15:21 +0300 Subject: [PATCH 10/77] PMM-9374 Fix tests. --- managed/services/victoriametrics/victoriametrics_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index b262d6fcf6..8d03365820 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -905,7 +905,11 @@ func Test_relativePath(t *testing.T) { "metrics", "http://127.0.0.1:9090/prometheus/metrics", }, - // TODO: Add test cases. + { + "http://127.0.0.1:9090/prometheus/", + "metrics", + "http://127.0.0.1:9090/prometheus/metrics", + }, } for _, tt := range tests { t.Run(fmt.Sprintf("%s", tt.want), func(t *testing.T) { From 71ce8123c3e4f256cccee8fba27b722c488f0359 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 15:38:12 +0300 Subject: [PATCH 11/77] PMM-9374 Fix linters. --- managed/services/victoriametrics/victoriametrics_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 8d03365820..6d83a5d4a2 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -19,6 +19,10 @@ import ( "context" "database/sql" "fmt" + "os" + "strings" + "testing" + "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,9 +30,6 @@ import ( "google.golang.org/grpc/status" "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" - "os" - "strings" - "testing" "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/testdb" From abe1dd798e5fce1f1513b4ba438ad2aacd4d30b4 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 30 Mar 2023 16:42:00 +0300 Subject: [PATCH 12/77] PMM-9374 Add new tests. --- managed/models/victoriametrics_params.go | 3 +++ managed/services/agents/vmagent_test.go | 19 +++++++++++++ .../victoriametrics/victoriametrics.go | 4 --- .../victoriametrics/victoriametrics_test.go | 27 ------------------- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 1693fa6a6b..841ed16b16 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -43,6 +43,9 @@ type VictoriaMetricsParams struct { // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { + if !strings.HasSuffix(vmURL, "/") { + vmURL = vmURL + "/" + } vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, URL: vmURL, diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 6522629ab4..43402e56ad 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -39,4 +39,23 @@ func TestMaxScrapeSize(t *testing.T) { actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+newValue) }) + t.Run("VMAGENT_ ENV variables", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + t.Setenv("VMAGENT_promscrape_maxScrapeSize", "16MiB") + t.Setenv("VM_remoteWrite_basicAuth_password", "password") + actual := vmAgentConfig("", params) + assert.Contains(t, actual.Env, "VMAGENT_promscrape_maxScrapeSize=16MiB") + assert.Contains(t, actual.Env, "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}") + assert.NotContains(t, actual.Env, "VM_remoteWrite_basicAuth_password=password") + }) + t.Run("External Victoria Metrics ENV variables", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "http://victoriametrics:8428") + require.NoError(t, err) + t.Setenv("VMAGENT_promscrape_maxScrapeSize", "16MiB") + actual := vmAgentConfig("", params) + assert.Contains(t, actual.Args, "-remoteWrite.url=http://victoriametrics:8428/api/v1/write") + assert.Contains(t, actual.Env, "VMAGENT_promscrape_maxScrapeSize=16MiB") + assert.NotContains(t, actual.Env, "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}") + }) } diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index c627843425..4f3c06c238 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -25,7 +25,6 @@ import ( "os/exec" "path" "regexp" - "strings" "time" "github.com/AlekSi/pointer" @@ -369,9 +368,6 @@ func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, bas } func relativePath(baseURL string, path string) (*url.URL, error) { - if !strings.HasSuffix(baseURL, "/") { - baseURL = baseURL + "/" - } u, err := url.Parse(baseURL) if err != nil { return nil, err diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 6d83a5d4a2..780fad011d 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -18,7 +18,6 @@ package victoriametrics import ( "context" "database/sql" - "fmt" "os" "strings" "testing" @@ -894,29 +893,3 @@ scrape_configs: assert.NoError(t, err) assert.Equal(t, expected, string(newcfg), "actual:\n%s", newcfg) } - -func Test_relativePath(t *testing.T) { - tests := []struct { - baseURL string - path string - want string - }{ - { - "http://127.0.0.1:9090/prometheus", - "metrics", - "http://127.0.0.1:9090/prometheus/metrics", - }, - { - "http://127.0.0.1:9090/prometheus/", - "metrics", - "http://127.0.0.1:9090/prometheus/metrics", - }, - } - for _, tt := range tests { - t.Run(fmt.Sprintf("%s", tt.want), func(t *testing.T) { - got, err := relativePath(tt.baseURL, tt.path) - assert.NoError(t, err) - assert.Equalf(t, tt.want, got.String(), "relativePath(%v, %v)", tt.baseURL, tt.path) - }) - } -} From c5d6afef4044a83b0420bd91266902e32993f8ee Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 01:00:18 +0300 Subject: [PATCH 13/77] PMM-9374 use interfaces instead of real object. --- managed/cmd/pmm-managed/main.go | 2 +- managed/models/victoriametrics_params.go | 25 ++++++++++++++++--- managed/services/agents/deps.go | 9 +++++++ managed/services/agents/registry.go | 4 +-- managed/services/agents/state.go | 4 +-- managed/services/agents/vmagent.go | 5 ++-- managed/services/supervisord/deps.go | 25 +++++++++++++++++++ .../services/supervisord/devcontainer_test.go | 3 ++- managed/services/supervisord/logs.go | 12 +++------ managed/services/supervisord/supervisord.go | 2 +- .../services/supervisord/supervisord_test.go | 6 +++-- .../victoriametrics/victoriametrics.go | 8 +++--- .../victoriametrics/victoriametrics_test.go | 3 ++- 13 files changed, 78 insertions(+), 30 deletions(-) create mode 100644 managed/services/supervisord/deps.go diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index b85c45d343..ca75a08627 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -781,7 +781,7 @@ func main() { qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) - agentsRegistry := agents.NewRegistry(db, vmParams.ExternalVM()) + agentsRegistry := agents.NewRegistry(db, vmParams) backupRemovalService := backup.NewRemovalService(db, minioClient) pitrTimerangeService := backup.NewPITRTimerangeService(minioClient) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 841ed16b16..b44d4ffa17 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -16,6 +16,7 @@ package models import ( + "net/url" "os" "strings" @@ -37,8 +38,8 @@ type VictoriaMetricsParams struct { VMAlertFlags []string // BaseConfigPath defines path for basic prometheus config. BaseConfigPath string - // URL defines URL of Victoria Metrics - URL string + // url defines url of Victoria Metrics + url *url.URL } // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. @@ -46,9 +47,14 @@ func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsPa if !strings.HasSuffix(vmURL, "/") { vmURL = vmURL + "/" } + + URL, err := url.Parse(vmURL) + if err != nil { + return nil, err + } vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, - URL: vmURL, + url: URL, } if err := vmp.UpdateParams(); err != nil { return vmp, err @@ -95,5 +101,16 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { } func (vmp *VictoriaMetricsParams) ExternalVM() bool { - return !strings.HasPrefix(vmp.URL, "http://127.0.0.1") + return vmp.url.Hostname() == "127.0.0.1" +} + +func (vmp *VictoriaMetricsParams) URL() string { + return vmp.url.String() +} + +func (vmp *VictoriaMetricsParams) URLFor(path string) (*url.URL, error) { + if path == "" { + return vmp.url, nil + } + return vmp.url.Parse(path) } diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index e05582ae11..b540e6f915 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -17,6 +17,7 @@ package agents import ( "context" + "net/url" "github.com/sirupsen/logrus" @@ -53,3 +54,11 @@ type jobsService interface { handleJobResult(ctx context.Context, l *logrus.Entry, result *agentpb.JobResult) handleJobProgress(ctx context.Context, progress *agentpb.JobProgress) } + +// victoriaMetricsParams is a subset of methods of models.VMParams used by this package. +// We use it instead of real type to avoid dependency cycle. +type victoriaMetricsParams interface { + ExternalVM() bool + URLFor(path string) (*url.URL, error) + URL() string +} diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 00caf4f348..84691cb739 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -90,7 +90,7 @@ type Registry struct { } // NewRegistry creates a new registry with given database connection. -func NewRegistry(db *reform.DB, isExternalVM bool) *Registry { +func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry { agents := make(map[string]*pmmAgentInfo) r := &Registry{ db: db, @@ -126,7 +126,7 @@ func NewRegistry(db *reform.DB, isExternalVM bool) *Registry { Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }), - isExternalVM: isExternalVM, + isExternalVM: externalVMChecker.ExternalVM(), } r.mAgents = prom.NewGaugeFunc(prom.GaugeOpts{ diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index b784b2d540..5d86f39c6e 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -44,11 +44,11 @@ type StateUpdater struct { db *reform.DB r *Registry vmdb prometheusService - vmParams *models.VictoriaMetricsParams + vmParams victoriaMetricsParams } // NewStateUpdater creates new agent state updater. -func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams *models.VictoriaMetricsParams) *StateUpdater { +func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams victoriaMetricsParams) *StateUpdater { return &StateUpdater{ db: db, r: r, diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index 1740cac088..b5586df710 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -23,7 +23,6 @@ import ( "github.com/percona/pmm/api/agentpb" "github.com/percona/pmm/api/inventorypb" - "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/envvars" ) @@ -33,10 +32,10 @@ var ( ) // vmAgentConfig returns desired configuration of vmagent process. -func vmAgentConfig(scrapeCfg string, params *models.VictoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { +func vmAgentConfig(scrapeCfg string, params victoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { serverURL := "{{.server_url}}/victoriametrics/" if params.ExternalVM() { - serverURL = params.URL + serverURL = params.URL() } maxScrapeSize := maxScrapeSizeDefault if space := os.Getenv(maxScrapeSizeEnv); space != "" { diff --git a/managed/services/supervisord/deps.go b/managed/services/supervisord/deps.go new file mode 100644 index 0000000000..8dea19e974 --- /dev/null +++ b/managed/services/supervisord/deps.go @@ -0,0 +1,25 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package supervisord + +import "net/url" + +// victoriaMetricsParams is a subset of methods of models.VMParams used by this package. +// We use it instead of real type to avoid dependency cycle. +type victoriaMetricsParams interface { + ExternalVM() bool + URLFor(path string) (*url.URL, error) +} diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index c902faf6c5..ae0ca51959 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -112,7 +112,8 @@ func TestDevContainer(t *testing.T) { t.Run("UpdateConfiguration", func(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New("/etc/supervisord.d", checker, vmParams, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 9748cdf695..1b78998bd0 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -25,7 +25,6 @@ import ( "io" "mime" "net/http" - "net/url" "os" "os/exec" "path/filepath" @@ -38,7 +37,6 @@ import ( "golang.org/x/sys/unix" "gopkg.in/yaml.v3" - "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/utils/logger" pprofUtils "github.com/percona/pmm/managed/utils/pprof" "github.com/percona/pmm/utils/pdeathsig" @@ -61,12 +59,12 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker - vmParams *models.VictoriaMetricsParams + vmParams victoriaMetricsParams } // NewLogs creates a new Logs service. // n is a number of last lines of log to read. -func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams *models.VictoriaMetricsParams) *Logs { +func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams victoriaMetricsParams) *Logs { return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, @@ -285,11 +283,7 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten } func (l *Logs) victoriaMetricsTargets(ctx context.Context) ([]byte, error) { - vmURL, err := url.Parse(l.vmParams.URL) - if err != nil { - return nil, err - } - targetsURL, err := vmURL.Parse("api/v1/targets") + targetsURL, err := l.vmParams.URLFor("api/v1/targets") if err != nil { return nil, err } diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index f4fadab40c..38af00df0d 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -429,7 +429,7 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "DataRetentionDays": int(settings.DataRetention.Hours() / 24), "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, - "VMURL": s.vmParams.URL, + "VMURL": s.vmParams.URL(), "ExternalVM": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, "InterfaceToBind": envvars.GetInterfaceToBind(), diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index 86578c5d1e..d80de1afac 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -36,7 +36,8 @@ func TestConfig(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, @@ -67,7 +68,8 @@ func TestDBaaSController(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New(configDir, pmmUpdateCheck, vmParams, gRPCMessageMaxSize) var tp *template.Template diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 4f3c06c238..f7015b157c 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -68,7 +68,7 @@ type Service struct { // NewVictoriaMetrics creates new VictoriaMetrics service. func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { - u, err := url.Parse(params.URL) + u, err := url.Parse(params.URL()) if err != nil { return nil, errors.WithStack(err) } @@ -333,7 +333,7 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params.URL)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params)) if svc.params.ExternalVM() { cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) } @@ -344,8 +344,8 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, baseURL string) *config.ScrapeConfig { - target, err := relativePath(baseURL, "metrics") +func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, vmParams *models.VictoriaMetricsParams) *config.ScrapeConfig { + target, err := vmParams.URLFor("metrics") if err != nil { l.Errorf("couldn't parse relative path to victoria metrics: %q", err) return nil diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 780fad011d..cc6a67a70c 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,7 +43,8 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: models.BasePrometheusConfigPath, URL: models.VMBaseURL} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + check.NoError(err) svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) From f9b458a3a3a0192623d9a52afa6fbbe357f6be84 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 13:11:29 +0300 Subject: [PATCH 14/77] PMM-9374 Fix the test. --- managed/models/victoriametrics_params.go | 2 +- managed/models/victoriametrics_params_test.go | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index b44d4ffa17..30a38f8b27 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -101,7 +101,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { } func (vmp *VictoriaMetricsParams) ExternalVM() bool { - return vmp.url.Hostname() == "127.0.0.1" + return vmp.url.Hostname() != "127.0.0.1" } func (vmp *VictoriaMetricsParams) URL() string { diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index 4c9b80a3f3..fa4aa654fa 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -16,6 +16,7 @@ package models import ( + "github.com/stretchr/testify/assert" "testing" "github.com/stretchr/testify/require" @@ -31,4 +32,35 @@ func TestVictoriaMetricsParams(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) + t.Run("check external VM", func(t *testing.T) { + + tests := []struct { + url string + want bool + }{ + { + "http://127.0.0.1:9090/prometheus", + false, + }, + { + "http://127.0.0.1:9090/prometheus/", + false, + }, + { + "http://victoriametrics:8428/", + true, + }, + { + "https://example.com:9090/", + true, + }, + } + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + vmp, err := NewVictoriaMetricsParams(BasePrometheusConfigPath, tt.url) + require.NoError(t, err) + assert.Equalf(t, tt.want, vmp.ExternalVM(), "ExternalVM()") + }) + } + }) } From 0d2b497e4c43bc370960aa8a55fb3fae94b47a39 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 13:15:59 +0300 Subject: [PATCH 15/77] PMM-9374 Fix the linter. --- managed/services/victoriametrics/victoriametrics.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index f7015b157c..7a70108d67 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -367,14 +367,6 @@ func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, vmP } } -func relativePath(baseURL string, path string) (*url.URL, error) { - u, err := url.Parse(baseURL) - if err != nil { - return nil, err - } - return u.Parse(path) -} - // scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { return &config.ScrapeConfig{ From 986666388576d370089d478a02e5cc8859b2f663 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 5 Apr 2023 14:21:38 +0300 Subject: [PATCH 16/77] PMM-9374 Fix the linter. --- managed/models/victoriametrics_params_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/managed/models/victoriametrics_params_test.go b/managed/models/victoriametrics_params_test.go index fa4aa654fa..b0f55796f6 100644 --- a/managed/models/victoriametrics_params_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -16,9 +16,9 @@ package models import ( - "github.com/stretchr/testify/assert" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,7 +33,6 @@ func TestVictoriaMetricsParams(t *testing.T) { require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) t.Run("check external VM", func(t *testing.T) { - tests := []struct { url string want bool From 661e49adad3367c254076a6c5d2e950811a4c18a Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Apr 2023 01:53:18 +0300 Subject: [PATCH 17/77] PMM-11952 docker configuration for external PostgreSQL usage. --- docker-compose.yml | 21 +++++++++++++++++--- managed/testdata/pg/Makefile | 26 ++++++++++++++++++++----- managed/testdata/pg/conf/pg_hba.conf | 4 ++-- managed/testdata/pg/queries/grafana.sql | 2 +- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 10244fc887..a934479962 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,6 +139,7 @@ services: depends_on: - ch - victoriametrics + - pg image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-managed-server hostname: pmm-managed-server @@ -161,8 +162,17 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 + - POSTGRES_ADDR=pg:5432 + - POSTGRES_USERNAME=pmm-managed + - POSTGRES_DBPASSWORD=pmm-password + - PERCONA_TEST_POSTGRES_SSL_MODE=require + - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt + - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key + - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt + - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana?ssl_mode=require + - GF_DATABASE_SSL_MODE=require +# - PMM_DEBUG=1 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - - PMM_DEBUG=1 - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest extra_hosts: @@ -194,17 +204,19 @@ services: # caching - go-modules:/root/go/pkg/mod - root-cache:/root/.cache + - ./managed/testdata/pg/certs/:/certs/ # PMM with external Postgres DB pg: profiles: - - pmm-pg + - pmm-ch build: context: ./managed/testdata/pg args: POSTGRES_IMAGE: ${POSTGRES_IMAGE:-postgres:14} dockerfile: Dockerfile - container_name: pmm-pg +# image: postgres:14 + container_name: pg environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pmm-password} ports: @@ -223,6 +235,9 @@ services: -c hba_file=/conf/pg_hba.conf networks: - ${NETWORK:-default} + volumes: + - ./managed/testdata/pg/conf/:/conf/ + - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ volumes: go-modules: diff --git a/managed/testdata/pg/Makefile b/managed/testdata/pg/Makefile index f112b5db0c..ebfac42936 100644 --- a/managed/testdata/pg/Makefile +++ b/managed/testdata/pg/Makefile @@ -11,12 +11,12 @@ all: root-ssl server-ssl pmm-managed-ssl grafana-ssl ## Generates all required d root-ssl: ## Generates root-ssl cert. openssl req -new -sha256 -nodes -newkey rsa:2048 \ -config ./certs/root.cnf \ - -keyout /tmp/root.key \ - -out /tmp/root.csr + -keyout ./certs/root.key \ + -out ./certs/root.csr openssl x509 -req -days 3653 -sha256 \ - -in /tmp/root.csr \ + -in ./certs/root.csr \ -extfile /etc/ssl/openssl.cnf -extensions v3_ca \ - -signkey /tmp/root.key \ + -signkey ./certs/root.key \ -out ./certs/root.crt server-ssl: ## Generates server-ssl cert. @@ -26,9 +26,11 @@ server-ssl: ## Generates server-ssl cer -out /tmp/server.csr openssl x509 -req -days 3653 -sha256 \ -extfile ./certs/server.cnf -extensions req_ext \ - -CA ./certs/root.crt -CAkey /tmp/root.key -CAcreateserial \ + -CA ./certs/root.crt -CAkey ./certs/root.key -CAcreateserial \ -in /tmp/server.csr \ -out ./certs/server.crt + chmod 600 certs/server.crt + chmod 600 certs/server.key pmm-managed-ssl: ## Generates pmm-managed-ssl cert. openssl req -new -sha256 -nodes -newkey rsa:2048 \ @@ -39,6 +41,20 @@ pmm-managed-ssl: ## Generates pmm-managed-ssl -CA ./certs/root.crt -CAkey /tmp/root.key -CAcreateserial \ -in /tmp/pmm-managed.csr \ -out ./certs/pmm-managed.crt + chmod 600 certs/pmm-managed.crt + chmod 600 certs/pmm-managed.key + +grafana-ssl: ## Generates grafana-ssl cert. + openssl req -new -sha256 -nodes -newkey rsa:2048 \ + -config ./certs/grafana.cnf \ + -keyout ./certs/grafana.key \ + -out /tmp/grafana.csr + openssl x509 -req -days 3653 -sha256 \ + -CA ./certs/root.crt -CAkey /tmp/root.key -CAcreateserial \ + -in /tmp/grafana.csr \ + -out ./certs/grafana.crt + chmod 600 certs/grafana.crt + chmod 600 certs/grafana.key grafana-ssl: ## Generates grafana-ssl cert. openssl req -new -sha256 -nodes -newkey rsa:2048 \ diff --git a/managed/testdata/pg/conf/pg_hba.conf b/managed/testdata/pg/conf/pg_hba.conf index 81144295c3..828afc295b 100644 --- a/managed/testdata/pg/conf/pg_hba.conf +++ b/managed/testdata/pg/conf/pg_hba.conf @@ -1,5 +1,5 @@ local all all trust hostnossl all pmm-managed all reject hostssl all pmm-managed all cert -hostnossl all grafana all reject -hostssl all grafana all cert \ No newline at end of file +hostnossl all grafana all trust +hostssl all grafana all trust \ No newline at end of file diff --git a/managed/testdata/pg/queries/grafana.sql b/managed/testdata/pg/queries/grafana.sql index 5bdbb0b4c8..44b1d4bf72 100644 --- a/managed/testdata/pg/queries/grafana.sql +++ b/managed/testdata/pg/queries/grafana.sql @@ -1,3 +1,3 @@ CREATE DATABASE "grafana"; CREATE USER "grafana" WITH PASSWORD 'grafana'; -GRANT ALL PRIVILEGES ON DATABASE "grafana" TO "grafana"; \ No newline at end of file +GRANT ALL PRIVILEGES ON DATABASE "grafana" TO "grafana" From c6390f0fc2ad1fab053723ae8012c8245ef6f646 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Apr 2023 01:53:54 +0300 Subject: [PATCH 18/77] PMM-11952 docker configuration for external PostgreSQL usage. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a934479962..acbe5b4cbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,9 +162,9 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 - - POSTGRES_ADDR=pg:5432 - - POSTGRES_USERNAME=pmm-managed - - POSTGRES_DBPASSWORD=pmm-password + - PERCONA_TEST_POSTGRES_ADDR=pg:5432 + - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed + - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-password - PERCONA_TEST_POSTGRES_SSL_MODE=require - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key From 60382e0a5407ddaa7e7c98d0182810a77eff7967 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Apr 2023 02:10:38 +0300 Subject: [PATCH 19/77] PMM-11924 PMM HA Phase 3. --- docker-compose.yml | 139 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 107 insertions(+), 32 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index acbe5b4cbf..0e2fc04d09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,7 +103,7 @@ services: # PMM with external DBs ch: profiles: - - pmm-external-dbs + - pmm-ha image: ${CH_IMAGE:-clickhouse/clickhouse-server:22.6.9.11-alpine} platform: linux/amd64 hostname: ${CH_HOSTNAME:-ch} @@ -111,9 +111,10 @@ services: - ${CH_PORT:-9000}:9000 networks: - ${NETWORK:-default} + victoriametrics: profiles: - - pmm-external-dbs + - pmm-ha hostname: ${VM_HOSTNAME:-victoriametrics} image: victoriametrics/victoria-metrics:v1.88.1 ports: @@ -133,9 +134,43 @@ services: - "--influxListenAddr=:8089" networks: - ${NETWORK:-default} - pmm-managed-server-ch: + + # PMM with external Postgres DB + pg: profiles: - - pmm-external-dbs + - pmm-ha + build: + context: ./managed/testdata/pg + args: + POSTGRES_IMAGE: ${POSTGRES_IMAGE:-postgres:14} + dockerfile: Dockerfile + # image: postgres:14 + container_name: pg + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pmm-password} + ports: + - ${POSTGRES_PORT:-5432}:5432 + command: | + postgres + -c shared_preload_libraries=pg_stat_statements + -c pg_stat_statements.max=10000 + -c pg_stat_statements.track=all + -c pg_stat_statements.save=off + -c fsync=off + -c ssl=on + -c ssl_ca_file=/certs/root.crt + -c ssl_cert_file=/certs/server.crt + -c ssl_key_file=/certs/server.key + -c hba_file=/conf/pg_hba.conf + networks: + - ${NETWORK:-default} + volumes: + - ./managed/testdata/pg/conf/:/conf/ + - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ + + pmm-managed-server-active: + profiles: + - pmm-ha depends_on: - ch - victoriametrics @@ -206,38 +241,78 @@ services: - root-cache:/root/.cache - ./managed/testdata/pg/certs/:/certs/ - # PMM with external Postgres DB - pg: + pmm-managed-server-passive: profiles: - - pmm-ch - build: - context: ./managed/testdata/pg - args: - POSTGRES_IMAGE: ${POSTGRES_IMAGE:-postgres:14} - dockerfile: Dockerfile -# image: postgres:14 - container_name: pg - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pmm-password} - ports: - - ${POSTGRES_PORT:-5432}:5432 - command: | - postgres - -c shared_preload_libraries=pg_stat_statements - -c pg_stat_statements.max=10000 - -c pg_stat_statements.track=all - -c pg_stat_statements.save=off - -c fsync=off - -c ssl=on - -c ssl_ca_file=/certs/root.crt - -c ssl_cert_file=/certs/server.crt - -c ssl_key_file=/certs/server.key - -c hba_file=/conf/pg_hba.conf + - pmm-ha + depends_on: + - ch + - pg + - victoriametrics + image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} + container_name: pmm-managed-server + hostname: pmm-managed-server networks: - ${NETWORK:-default} + environment: + - PMM_RELEASE_PATH=/root/go/bin + - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} + - ENABLE_DBAAS=${ENABLE_DBAAS:-0} + - AWS_ACCESS_KEY=${AWS_ACCESS_KEY} + - AWS_SECRET_KEY=${AWS_SECRET_KEY} +# - PERCONA_TEST_PLATFORM_ADDRESS=https://check.localhost +# - PERCONA_TEST_PLATFORM_INSECURE=1 +# - PERCONA_TEST_PLATFORM_PUBLIC_KEY= +# - PERCONA_TEST_TELEMETRY_INTERVAL=10s +# - PERCONA_TEST_TELEMETRY_RETRY_BACKOFF=10s +# - PERCONA_TEST_TELEMETRY_DISABLE_START_DELAY=1 + - PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000 + - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm + - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 + - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 + - PERCONA_TEST_POSTGRES_ADDR=pg:5432 + - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed + - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-password + - PERCONA_TEST_POSTGRES_SSL_MODE=require + - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt + - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key + - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt + - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana?ssl_mode=require + - GF_DATABASE_SSL_MODE=require +# - PMM_DEBUG=1 + - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} + - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest + - PERCONA_TEST_HA_PASSIVE=true + + extra_hosts: + - host.docker.internal:host-gateway + # - portal.localhost:${PORTAL_HOST:-host-gateway} + # - check.localhost:${PORTAL_CHECK_HOST:-host-gateway} + # - pmm.localhost:${PORTAL_PMM_HOST:-host-gateway} + # - check-dev.percona.com:${PORTAL_PMM_HOST:-host-gateway} + + # for delve + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + + # see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do + ulimits: + memlock: 67108864 + + ports: + - ${PMM_PORT_HTTP:-80}:80 + - ${PMM_PORT_HTTPS:-443}:443 + # For headless delve + - ${PMM_PORT_DELVE:-2345}:2345 volumes: - - ./managed/testdata/pg/conf/:/conf/ - - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ + - ./:/root/go/src/github.com/percona/pmm +# - "../grafana/public:/usr/share/grafana/public" + - ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # change Makefile in devcontainer + # caching + - go-modules:/root/go/pkg/mod + - root-cache:/root/.cache + - ./managed/testdata/pg/certs/:/certs/ volumes: go-modules: From 5875881af2b6becea6a1844be85e941ae7a7a64e Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 11 Apr 2023 00:12:45 +0300 Subject: [PATCH 20/77] PMM-11924 PMM HA Phase 3 PoC. --- docker-compose.yml | 56 +++++++++------- managed/cmd/pmm-managed-init/main.go | 1 + managed/cmd/pmm-managed/main.go | 66 ++++++++++++++----- managed/services/agents/deps.go | 5 ++ managed/services/agents/handler.go | 9 ++- .../highavailability/highavailability.go | 35 ++++++++++ managed/services/supervisord/pmm_config.go | 2 + managed/testdata/pg/conf/pg_hba.conf | 5 +- 8 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 managed/services/highavailability/highavailability.go diff --git a/docker-compose.yml b/docker-compose.yml index 0e2fc04d09..44318baa0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -144,7 +144,7 @@ services: args: POSTGRES_IMAGE: ${POSTGRES_IMAGE:-postgres:14} dockerfile: Dockerfile - # image: postgres:14 +# image: postgres:14 container_name: pg environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pmm-password} @@ -157,11 +157,11 @@ services: -c pg_stat_statements.track=all -c pg_stat_statements.save=off -c fsync=off - -c ssl=on - -c ssl_ca_file=/certs/root.crt - -c ssl_cert_file=/certs/server.crt - -c ssl_key_file=/certs/server.key - -c hba_file=/conf/pg_hba.conf +# -c hba_file=/conf/pg_hba.conf +# -c ssl=on +# -c ssl_ca_file=/certs/root.crt +# -c ssl_cert_file=/certs/server.crt +# -c ssl_key_file=/certs/server.key networks: - ${NETWORK:-default} volumes: @@ -199,16 +199,19 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 - PERCONA_TEST_POSTGRES_ADDR=pg:5432 - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed - - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-password - - PERCONA_TEST_POSTGRES_SSL_MODE=require - - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt - - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key - - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt - - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana?ssl_mode=require - - GF_DATABASE_SSL_MODE=require + - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-managed + - PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 +# - PERCONA_TEST_POSTGRES_SSL_MODE=require +# - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt +# - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key +# - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt + - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana +# - GF_DATABASE_SSL_MODE=require # - PMM_DEBUG=1 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=192.168.48.5:9094 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9094,pmm-managed-server-passive:9094 extra_hosts: - host.docker.internal:host-gateway @@ -249,8 +252,8 @@ services: - pg - victoriametrics image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} - container_name: pmm-managed-server - hostname: pmm-managed-server + container_name: pmm-managed-server-passive + hostname: pmm-managed-server-passive networks: - ${NETWORK:-default} environment: @@ -271,17 +274,20 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 - PERCONA_TEST_POSTGRES_ADDR=pg:5432 - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed - - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-password - - PERCONA_TEST_POSTGRES_SSL_MODE=require - - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt - - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key - - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt - - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana?ssl_mode=require - - GF_DATABASE_SSL_MODE=require + - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-managed + - PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 +# - PERCONA_TEST_POSTGRES_SSL_MODE=require +# - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt +# - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key +# - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt + - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana +# - GF_DATABASE_SSL_MODE=require # - PMM_DEBUG=1 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - PERCONA_TEST_HA_PASSIVE=true + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=192.168.48.6:9094 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9094,pmm-managed-server-passive:9094 extra_hosts: - host.docker.internal:host-gateway @@ -301,10 +307,10 @@ services: memlock: 67108864 ports: - - ${PMM_PORT_HTTP:-80}:80 - - ${PMM_PORT_HTTPS:-443}:443 + - ${PMM_PORT_HTTP:-8080}:80 + - ${PMM_PORT_HTTPS:-4433}:443 # For headless delve - - ${PMM_PORT_DELVE:-2345}:2345 + - ${PMM_PORT_DELVE:-12345}:2345 volumes: - ./:/root/go/src/github.com/percona/pmm # - "../grafana/public:/usr/share/grafana/public" diff --git a/managed/cmd/pmm-managed-init/main.go b/managed/cmd/pmm-managed-init/main.go index f04f7efe0a..763776c805 100644 --- a/managed/cmd/pmm-managed-init/main.go +++ b/managed/cmd/pmm-managed-init/main.go @@ -54,6 +54,7 @@ func main() { pmmConfigParams := make(map[string]any) pmmConfigParams["DisableInternalDB"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES")) + pmmConfigParams["PassivePMM"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_HA_PASSIVE")) if err := supervisord.SavePMMConfig(pmmConfigParams); err != nil { logrus.Errorf("PMM Server configuration error: %s.", err) os.Exit(1) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index bbf9b692dd..6f0f4d0971 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -21,6 +21,7 @@ import ( "database/sql" _ "expvar" // register /debug/vars "fmt" + "github.com/percona/pmm/managed/services/highavailability" "html/template" "log" "net" @@ -182,6 +183,7 @@ func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) { type gRPCServerDeps struct { db *reform.DB + ha *highavailability.Service vmdb *victoriametrics.Service platformClient *platformClient.Client server *server.Server @@ -299,7 +301,10 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { backuppb.RegisterRestoreHistoryServer(gRPCServer, managementbackup.NewRestoreHistoryService(deps.db)) k8sServer := managementdbaas.NewKubernetesServer(deps.db, deps.dbaasClient, deps.versionServiceClient, deps.grafanaClient) - deps.dbaasInitializer.RegisterKubernetesServer(k8sServer) + + if deps.ha.ActiveMode() { + deps.dbaasInitializer.RegisterKubernetesServer(k8sServer) + } dbaasv1beta1.RegisterKubernetesServer(gRPCServer, k8sServer) dbaasv1beta1.RegisterDBClustersServer(gRPCServer, managementdbaas.NewDBClusterService(deps.db, deps.grafanaClient, deps.versionServiceClient)) dbaasv1beta1.RegisterPXCClustersServer(gRPCServer, managementdbaas.NewPXCClusterService(deps.db, deps.grafanaClient, deps.componentsService, deps.versionServiceClient.GetVersionServiceURL())) @@ -524,6 +529,7 @@ func runDebugServer(ctx context.Context) { type setupDeps struct { sqlDB *sql.DB + ha *highavailability.Service supervisord *supervisord.Service vmdb *victoriametrics.Service vmalert *vmalert.Service @@ -572,21 +578,27 @@ func setup(ctx context.Context, deps *setupDeps) bool { deps.l.Warnf("VictoriaMetrics problem: %+v.", err) return false } - deps.vmdb.RequestConfigurationUpdate() + if deps.ha.ActiveMode() { + deps.vmdb.RequestConfigurationUpdate() + } deps.l.Infof("Checking VMAlert...") if err = deps.vmalert.IsReady(ctx); err != nil { deps.l.Warnf("VMAlert problem: %+v.", err) return false } - deps.vmalert.RequestConfigurationUpdate() + if deps.ha.ActiveMode() { + deps.vmalert.RequestConfigurationUpdate() + } deps.l.Infof("Checking Alertmanager...") if err = deps.alertmanager.IsReady(ctx); err != nil { deps.l.Warnf("Alertmanager problem: %+v.", err) return false } - deps.alertmanager.RequestConfigurationUpdate() + if deps.ha.ActiveMode() { + deps.alertmanager.RequestConfigurationUpdate() + } deps.l.Info("Setup completed.") return true @@ -709,6 +721,10 @@ func main() { Envar("PERCONA_TEST_POSTGRES_SSL_CERT_PATH"). String() + passiveModeF := kingpin.Flag("ha-passive-mode", "Run in Passive mode"). + Envar("PERCONA_TEST_HA_PASSIVE"). + Bool() + supervisordConfigDirF := kingpin.Flag("supervisord-config-dir", "Supervisord configuration directory").Required().String() logLevelF := kingpin.Flag("log-level", "Set logging level").Envar("PMM_LOG_LEVEL").Default("info").Enum("trace", "debug", "info", "warn", "error", "fatal") @@ -738,6 +754,8 @@ func main() { ctx = logger.Set(ctx, "main") defer l.Info("Done.") + ha := highavailability.New(passiveModeF) + cfg := config.NewService() if err := cfg.Load(); err != nil { l.Panicf("Failed to load config: %+v", err) @@ -794,17 +812,21 @@ func main() { } defer sqlDB.Close() //nolint:errcheck - migrateDB(ctx, sqlDB, setupParams) + if ha.ActiveMode() { + migrateDB(ctx, sqlDB, setupParams) + } prom.MustRegister(sqlmetrics.NewCollector("postgres", *postgresDBNameF, sqlDB)) reformL := sqlmetrics.NewReform("postgres", *postgresDBNameF, logrus.WithField("component", "reform").Tracef) prom.MustRegister(reformL) db := reform.NewDB(sqlDB, postgresql.Dialect, reformL) - // Generate unique PMM Server ID if it's not already set. - err = models.SetPMMServerID(db) - if err != nil { - l.Panicf("failed to set PMM Server ID") + if ha.ActiveMode() { + // Generate unique PMM Server ID if it's not already set. + err = models.SetPMMServerID(db) + if err != nil { + l.Panicf("failed to set PMM Server ID") + } } cleaner := clean.New(db) @@ -884,7 +906,7 @@ func main() { jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService) agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams) - agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) + agentsHandler := agents.NewHandler(db, ha, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) actionsService := agents.NewActionsService(qanClient, agentsRegistry) @@ -1007,11 +1029,13 @@ func main() { }() } - // Set all agents status to unknown at startup. The ones that are alive - // will get their status updated after they connect to the pmm-managed. - err = agentsHandler.SetAllAgentsStatusUnknown(ctx) - if err != nil { - l.Errorf("Failed to set status of all agents to invalid at startup: %s", err) + if ha.ActiveMode() { + // Set all agents status to unknown at startup. The ones that are alive + // will get their status updated after they connect to the pmm-managed. + err = agentsHandler.SetAllAgentsStatusUnknown(ctx) + if err != nil { + l.Errorf("Failed to set status of all agents to invalid at startup: %s", err) + } } settings, err := models.GetSettings(sqlDB) @@ -1050,7 +1074,9 @@ func main() { wg.Add(1) go func() { defer wg.Done() - checksService.Run(ctx) + if ha.ActiveMode() { + checksService.Run(ctx) + } }() wg.Add(1) @@ -1062,13 +1088,17 @@ func main() { wg.Add(1) go func() { defer wg.Done() - telemetry.Run(ctx) + if ha.ActiveMode() { + telemetry.Run(ctx) + } }() wg.Add(1) go func() { defer wg.Done() - schedulerService.Run(ctx) + if ha.ActiveMode() { + schedulerService.Run(ctx) + } }() wg.Add(1) diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index b540e6f915..ad86b939a7 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -62,3 +62,8 @@ type victoriaMetricsParams interface { URLFor(path string) (*url.URL, error) URL() string } + +// highAvailablityService is an interface to get information related to High Availability. +type highAvailablityService interface { + PassiveMode() bool +} diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go index 9a841e7979..ee148c4717 100644 --- a/managed/services/agents/handler.go +++ b/managed/services/agents/handler.go @@ -37,6 +37,7 @@ import ( // Handler handles agent requests. type Handler struct { db *reform.DB + ha highAvailablityService r *Registry vmdb prometheusService qanClient qanClient @@ -45,11 +46,10 @@ type Handler struct { } // NewHandler creates new agents handler. -func NewHandler(db *reform.DB, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater, - jobsService jobsService, -) *Handler { +func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater, jobsService jobsService, ) *Handler { h := &Handler{ db: db, + ha: ha, r: registry, vmdb: vmdb, qanClient: qanClient, @@ -61,6 +61,9 @@ func NewHandler(db *reform.DB, qanClient qanClient, vmdb prometheusService, regi // Run takes over pmm-agent gRPC stream and runs it until completion. func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { + if h.ha.PassiveMode() { + return status.Errorf(codes.FailedPrecondition, "Current PMM Server is running in passive mode.") + } disconnectReason := "unknown" ctx := stream.Context() diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go new file mode 100644 index 0000000000..e326351d00 --- /dev/null +++ b/managed/services/highavailability/highavailability.go @@ -0,0 +1,35 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package highavailability contains everything related to high availability. +package highavailability + +import "github.com/AlekSi/pointer" + +type Service struct { + passiveMode bool +} + +func New(passiveMode *bool) *Service { + return &Service{passiveMode: pointer.GetBool(passiveMode)} +} + +func (s *Service) ActiveMode() bool { + return !s.passiveMode +} + +func (s *Service) PassiveMode() bool { + return s.passiveMode +} diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index bc6fa3503a..78cf838602 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -178,6 +178,7 @@ stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true +{{ if not .PassivePMM }} [program:pmm-agent] priority = 15 command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml @@ -191,6 +192,7 @@ stdout_logfile = /srv/logs/pmm-agent.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true +{{ end }} [program:pmm-update-perform] command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml diff --git a/managed/testdata/pg/conf/pg_hba.conf b/managed/testdata/pg/conf/pg_hba.conf index 828afc295b..c9d25ee111 100644 --- a/managed/testdata/pg/conf/pg_hba.conf +++ b/managed/testdata/pg/conf/pg_hba.conf @@ -1,5 +1,2 @@ local all all trust -hostnossl all pmm-managed all reject -hostssl all pmm-managed all cert -hostnossl all grafana all trust -hostssl all grafana all trust \ No newline at end of file +hostnossl all all trust \ No newline at end of file From 1f609cb1e23bfe93e4c7207ee528a21510bf3f0a Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 11 Apr 2023 14:18:23 +0300 Subject: [PATCH 21/77] PMM-11924 PMM HA Phase 3 PoC. --- managed/cmd/pmm-managed/main.go | 2 ++ managed/services/supervisord/pmm_config.go | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 6f0f4d0971..e05ed3b025 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -1002,6 +1002,7 @@ func main() { // try synchronously once, then retry in the background deps := &setupDeps{ sqlDB: sqlDB, + ha: ha, supervisord: supervisord, vmdb: vmdb, vmalert: vmalert, @@ -1113,6 +1114,7 @@ func main() { runGRPCServer(ctx, &gRPCServerDeps{ db: db, + ha: ha, vmdb: vmdb, platformClient: platformClient, server: server, diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index 78cf838602..3a555443f4 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -165,7 +165,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true From 6f8c8065f6d632d697831e2f085b10bb8a44febb Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Apr 2023 02:10:38 +0300 Subject: [PATCH 22/77] PMM-11924 PMM HA Phase 3 PoC. --- managed/cmd/pmm-managed/main.go | 2 +- managed/services/supervisord/supervisord_test.go | 12 +++++++++++- managed/testdata/supervisord.d/grafana.ini | 16 ++++++++-------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index e05ed3b025..dd8c6fb5c3 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -21,7 +21,6 @@ import ( "database/sql" _ "expvar" // register /debug/vars "fmt" - "github.com/percona/pmm/managed/services/highavailability" "html/template" "log" "net" @@ -41,6 +40,7 @@ import ( grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/validator" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/percona/pmm/managed/services/highavailability" "github.com/pkg/errors" metrics "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index 37bc3bfc05..09dab857d4 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -38,7 +38,17 @@ func TestConfig(t *testing.T) { configDir := filepath.Join("..", "..", "testdata", "supervisord.d") vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) - s := New(configDir, pmmUpdateCheck, vmParams, models.PGParams{}, gRPCMessageMaxSize) + pgParams := models.PGParams{ + Addr: "127.0.0.1:5432", + DBName: "postgres", + DBUsername: "db_username", + DBPassword: "db_password", + SSLMode: "verify", + SSLCAPath: "path-to-CA-cert", + SSLKeyPath: "path-to-key", + SSLCertPath: "path-to-cert", + } + s := New(configDir, pmmUpdateCheck, vmParams, pgParams, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, AlertManagerURL: "https://external-user:passw!,ord@external-alertmanager:6443/alerts", diff --git a/managed/testdata/supervisord.d/grafana.ini b/managed/testdata/supervisord.d/grafana.ini index 573136a1de..82893b7f3d 100644 --- a/managed/testdata/supervisord.d/grafana.ini +++ b/managed/testdata/supervisord.d/grafana.ini @@ -13,14 +13,14 @@ command = cfg:default.log.console.format=console cfg:default.server.root_url="https://%%(domain)s/graph" environment = - PERCONA_TEST_POSTGRES_ADDR="", - PERCONA_TEST_POSTGRES_DBNAME="", - PERCONA_TEST_POSTGRES_USERNAME="", - PERCONA_TEST_POSTGRES_DBPASSWORD="", - PERCONA_TEST_POSTGRES_SSL_MODE="", - PERCONA_TEST_POSTGRES_SSL_CA_PATH="", - PERCONA_TEST_POSTGRES_SSL_KEY_PATH="", - PERCONA_TEST_POSTGRES_SSL_CERT_PATH="", + PERCONA_TEST_POSTGRES_ADDR="127.0.0.1:5432", + PERCONA_TEST_POSTGRES_DBNAME="postgres", + PERCONA_TEST_POSTGRES_USERNAME="db_username", + PERCONA_TEST_POSTGRES_DBPASSWORD="db_password", + PERCONA_TEST_POSTGRES_SSL_MODE="verify", + PERCONA_TEST_POSTGRES_SSL_CA_PATH="path-to-CA-cert", + PERCONA_TEST_POSTGRES_SSL_KEY_PATH="path-to-key", + PERCONA_TEST_POSTGRES_SSL_CERT_PATH="path-to-cert", PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE_ADDR="127.0.0.1:8123", user = grafana directory = /usr/share/grafana From e7eb9aca78fcd428da1fbb927d843171efc27c94 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 18 Apr 2023 14:23:10 +0300 Subject: [PATCH 23/77] PMM-11924 Fix tests. --- .../services/supervisord/supervisord_test.go | 9 +- .../testdata/supervisord.d/pmm-passive.ini | 90 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 managed/testdata/supervisord.d/pmm-passive.ini diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index 09dab857d4..19e9127d4f 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -177,14 +177,19 @@ func TestSavePMMConfig(t *testing.T) { }{ { description: "disable internal postgresql db", - params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false}, + params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "PassivePMM": false}, file: "pmm-db_disabled", }, { description: "enable internal postgresql db", - params: map[string]any{"DisableInternalDB": false, "DisableSupervisor": false}, + params: map[string]any{"DisableInternalDB": false, "DisableSupervisor": false, "PassivePMM": false}, file: "pmm-db_enabled", }, + { + description: "passive pmm", + params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "PassivePMM": true}, + file: "pmm-passive", + }, } for _, test := range tests { t.Run(test.description, func(t *testing.T) { diff --git a/managed/testdata/supervisord.d/pmm-passive.ini b/managed/testdata/supervisord.d/pmm-passive.ini new file mode 100644 index 0000000000..3565e746c3 --- /dev/null +++ b/managed/testdata/supervisord.d/pmm-passive.ini @@ -0,0 +1,90 @@ + +[unix_http_server] +chmod = 0700 +username = dummy +password = dummy + +[supervisorctl] +username = dummy +password = dummy + +[program:pmm-update-perform-init] +command = /usr/sbin/pmm-update -run-playbook -playbook=/usr/share/pmm-update/ansible/playbook/tasks/init.yml +directory = / +autorestart = unexpected +priority=-1 +exitcodes = 0 +autostart = true +startretries = 3 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +stdout_logfile = /srv/logs/pmm-update-perform-init.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 3 +redirect_stderr = true + +[program:clickhouse] +priority = 2 +command = /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +; config.xml contains settings to log to stdout (console), +; so we delegate logfile managemenet to supervisord +stdout_logfile = /srv/logs/clickhouse-server.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:nginx] +priority = 4 +command = nginx +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 10 +; nginx.conf contains settings to log to /dev/sdtout and /dev/stderr, +; which allows supervisord to manage the logs further. +stdout_logfile = /srv/logs/nginx.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:pmm-managed] +priority = 14 +command = + /usr/sbin/pmm-managed + --victoriametrics-config=/etc/victoriametrics-promscrape.yml + --victoriametrics-url=http://127.0.0.1:9090/prometheus + --supervisord-config-dir=/etc/supervisord.d +autorestart = true +autostart = true +startretries = 1000 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +stdout_logfile = /srv/logs/pmm-managed.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:pmm-update-perform] +command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml +directory = / +autorestart = unexpected +exitcodes = 0 +autostart = false +startretries = 10 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +stdout_logfile = /srv/logs/pmm-update-perform.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 3 +redirect_stderr = true From 45288f6466c0750a29574dc8a842a14b89441ed1 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 19 Apr 2023 11:20:03 +0300 Subject: [PATCH 24/77] PMM-11924 Fix tests and linters. --- managed/cmd/pmm-managed/main.go | 2 +- managed/services/agents/handler.go | 2 +- managed/services/supervisord/pmm_config.go | 4 ++-- managed/testdata/supervisord.d/pmm-db_disabled.ini | 1 - managed/testdata/supervisord.d/pmm-passive.ini | 1 - 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 639bc9c6d2..c394d6d4af 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -40,7 +40,6 @@ import ( grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/validator" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - "github.com/percona/pmm/managed/services/highavailability" "github.com/pkg/errors" metrics "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" @@ -84,6 +83,7 @@ import ( "github.com/percona/pmm/managed/services/config" "github.com/percona/pmm/managed/services/dbaas" "github.com/percona/pmm/managed/services/grafana" + "github.com/percona/pmm/managed/services/highavailability" "github.com/percona/pmm/managed/services/inventory" inventorygrpc "github.com/percona/pmm/managed/services/inventory/grpc" "github.com/percona/pmm/managed/services/management" diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go index ee148c4717..54d101a336 100644 --- a/managed/services/agents/handler.go +++ b/managed/services/agents/handler.go @@ -46,7 +46,7 @@ type Handler struct { } // NewHandler creates new agents handler. -func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater, jobsService jobsService, ) *Handler { +func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater, jobsService jobsService) *Handler { h := &Handler{ db: db, ha: ha, diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index 3a555443f4..e92e07ab05 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -177,7 +177,7 @@ stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true -{{ if not .PassivePMM }} +{{- if not .PassivePMM }} [program:pmm-agent] priority = 15 command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml @@ -191,7 +191,7 @@ stdout_logfile = /srv/logs/pmm-agent.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true -{{ end }} +{{ end -}} [program:pmm-update-perform] command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml diff --git a/managed/testdata/supervisord.d/pmm-db_disabled.ini b/managed/testdata/supervisord.d/pmm-db_disabled.ini index 210d822422..b45251ab1f 100644 --- a/managed/testdata/supervisord.d/pmm-db_disabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_disabled.ini @@ -61,7 +61,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true diff --git a/managed/testdata/supervisord.d/pmm-passive.ini b/managed/testdata/supervisord.d/pmm-passive.ini index 3565e746c3..c89b8bc9e3 100644 --- a/managed/testdata/supervisord.d/pmm-passive.ini +++ b/managed/testdata/supervisord.d/pmm-passive.ini @@ -61,7 +61,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true From 78c59e8b5e652cfebf13cf2373b869d598b44590 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 19 Apr 2023 12:50:51 +0300 Subject: [PATCH 25/77] PMM-11924 Fix tests. --- managed/services/supervisord/pmm_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index e92e07ab05..112e26108d 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -176,8 +176,8 @@ stdout_logfile = /srv/logs/pmm-managed.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true - {{- if not .PassivePMM }} + [program:pmm-agent] priority = 15 command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml @@ -191,7 +191,7 @@ stdout_logfile = /srv/logs/pmm-agent.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true -{{ end -}} +{{- end }} [program:pmm-update-perform] command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml From 7008fde8ed7124ef6f8a5dd275058e810da5c958 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 19 Apr 2023 13:07:47 +0300 Subject: [PATCH 26/77] PMM-11924 Fix linter. --- managed/services/agents/handler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go index 54d101a336..026bb6db9c 100644 --- a/managed/services/agents/handler.go +++ b/managed/services/agents/handler.go @@ -46,7 +46,9 @@ type Handler struct { } // NewHandler creates new agents handler. -func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater, jobsService jobsService) *Handler { +func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, vmdb prometheusService, registry *Registry, + state *StateUpdater, jobsService jobsService, +) *Handler { h := &Handler{ db: db, ha: ha, From a456c0cb3e911c27e9a783794657d34935c0729c Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 28 Apr 2023 20:49:19 +0300 Subject: [PATCH 27/77] Update pmm_config.go --- managed/services/supervisord/pmm_config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index bc6fa3503a..f7e52bbe4b 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -165,7 +165,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true From e2323e26594cfa24e411d88b85fc5ae029b8805d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 28 Apr 2023 21:06:48 +0300 Subject: [PATCH 28/77] Update pmm-db_disabled.ini --- managed/testdata/supervisord.d/pmm-db_disabled.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/managed/testdata/supervisord.d/pmm-db_disabled.ini b/managed/testdata/supervisord.d/pmm-db_disabled.ini index 210d822422..b45251ab1f 100644 --- a/managed/testdata/supervisord.d/pmm-db_disabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_disabled.ini @@ -61,7 +61,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true From 7363a013ba27df993be87f87a0b4bc4b7bf62c66 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 3 May 2023 17:47:01 +0300 Subject: [PATCH 29/77] PMM-12078 HA Phase 3 Active-Active PoC. --- .devcontainer/setup.py | 2 +- admin/cli/cli.go | 1 + admin/commands/reload.go | 56 +++++ docker-compose.yml | 19 +- go.mod | 8 +- go.sum | 16 +- managed/cmd/pmm-managed/main.go | 118 +++++---- managed/services/agents/deps.go | 3 +- managed/services/agents/handler.go | 3 - .../highavailability/highavailability.go | 233 +++++++++++++++++- .../highavailability/leaderservice.go | 59 +++++ .../services/highavailability/memberlist.go | 22 ++ managed/services/highavailability/raft.go | 77 ++++++ managed/services/highavailability/services.go | 91 +++++++ 14 files changed, 642 insertions(+), 66 deletions(-) create mode 100644 admin/commands/reload.go create mode 100644 managed/services/highavailability/leaderservice.go create mode 100644 managed/services/highavailability/memberlist.go create mode 100644 managed/services/highavailability/raft.go create mode 100644 managed/services/highavailability/services.go diff --git a/.devcontainer/setup.py b/.devcontainer/setup.py index 829c77fb85..820461f2a5 100755 --- a/.devcontainer/setup.py +++ b/.devcontainer/setup.py @@ -102,7 +102,7 @@ def setup(): def main(): install_packages() install_go() - make_init() + # make_init() # do basic setup setup() diff --git a/admin/cli/cli.go b/admin/cli/cli.go index 83012c274f..195272e9ff 100644 --- a/admin/cli/cli.go +++ b/admin/cli/cli.go @@ -49,6 +49,7 @@ type PMMAdminCommands struct { flags.GlobalFlags Status commands.StatusCommand `cmd:"" help:"Show information about local pmm-agent"` + Reload commands.ReloadCommand `cmd:"" help:"Show information about local pmm-agent"` Summary commands.SummaryCommand `cmd:"" help:"Fetch system data for diagnostics"` List commands.ListCommand `cmd:"" help:"Show Services and Agents running on this Node"` Config commands.ConfigCommand `cmd:"" help:"Configure local pmm-agent"` diff --git a/admin/commands/reload.go b/admin/commands/reload.go new file mode 100644 index 0000000000..fd990c81ed --- /dev/null +++ b/admin/commands/reload.go @@ -0,0 +1,56 @@ +// Copyright 2019 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "time" + + "github.com/percona/pmm/api/agentlocalpb/json/client" + "github.com/percona/pmm/api/agentlocalpb/json/client/agent_local" +) + +var reloadResultT = ParseTemplate(` +Reloaded. +`) + +type reloadResult struct{} + +func (res *reloadResult) Result() {} + +func (res *reloadResult) String() string { + return RenderTemplate(reloadResultT, res) +} + +// ReloadCommand is used by Kong for CLI flags and commands. +type ReloadCommand struct { + Timeout time.Duration `name:"wait" help:"Time to wait for a successful response from pmm-agent"` +} + +// BeforeApply is run before the command is applied. +func (cmd *ReloadCommand) BeforeApply() error { + return nil +} + +// RunCmd runs the ReloadCommand. +func (cmd *ReloadCommand) RunCmd() (Result, error) { + _, err := client.Default.AgentLocal.Reload(&agent_local.ReloadParams{ + Context: Ctx, + }) + if err != nil { + return nil, err + } + + return &reloadResult{}, nil +} diff --git a/docker-compose.yml b/docker-compose.yml index ac1de74cfb..98042a4e63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -210,8 +210,14 @@ services: # - PMM_DEBUG=1 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=192.168.48.5:9094 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9094,pmm-managed-server-passive:9094 + - PERCONA_TEST_HA_ENABLE=1 + - PERCONA_TEST_HA_BOOTSTRAP=1 + - PERCONA_TEST_HA_NODE_ID=pmm-managed-server + - PERCONA_TEST_HA_ADVERTISE_ADDRESS=pmm-managed-server + - PERCONA_TEST_HA_PEERS=pmm-managed-server,pmm-managed-server-passive + - PERCONA_TEST_HA_GOSSIP_PORT=9096 + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=pmm-managed-server:9095 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9095,pmm-managed-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway @@ -286,8 +292,13 @@ services: - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - PERCONA_TEST_HA_PASSIVE=true - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=192.168.48.6:9094 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9094,pmm-managed-server-passive:9094 + - PERCONA_TEST_HA_ENABLE=1 + - PERCONA_TEST_HA_NODE_ID=pmm-managed-server-passive + - PERCONA_TEST_HA_ADVERTISE_ADDRESS=pmm-managed-server-passive + - PERCONA_TEST_HA_PEERS=pmm-managed-server,pmm-managed-server-passive + - PERCONA_TEST_HA_GOSSIP_PORT=9096 + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=pmm-managed-server-passive:9095 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9095,pmm-managed-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway diff --git a/go.mod b/go.mod index d5a0ca448c..9fc907968f 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 github.com/hashicorp/go-version v1.6.0 + github.com/hashicorp/raft v1.5.0 github.com/jmoiron/sqlx v1.3.5 github.com/jotaen/kong-completion v0.0.5 github.com/lib/pq v1.10.6 @@ -106,6 +107,7 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -116,8 +118,10 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/minio/minio-go v6.0.14+incompatible // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -173,7 +177,7 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/andybalholm/brotli v1.0.5 // indirect - github.com/armon/go-metrics v0.4.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -204,7 +208,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect - github.com/hashicorp/memberlist v0.5.0 // indirect + github.com/hashicorp/memberlist v0.5.0 github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5724421694..49f99b3278 100644 --- a/go.sum +++ b/go.sum @@ -176,8 +176,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30/go.mod h1:Q7 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q= -github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -509,6 +509,8 @@ github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -818,6 +820,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -850,6 +854,8 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/raft v1.5.0 h1:uNs9EfJ4FwiArZRxxfd/dQ5d33nV31/CdCHArH89hT8= +github.com/hashicorp/raft v1.5.0/go.mod h1:pKHB2mf/Y25u3AHNSXVRv+yT+WAnmeTX0BwVppVQV+M= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -1022,6 +1028,9 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -1429,6 +1438,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= @@ -1845,6 +1855,7 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1853,6 +1864,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220317061510-51cd9980dadf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index c394d6d4af..f68951260b 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -306,9 +306,10 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { k8sServer := managementdbaas.NewKubernetesServer(deps.db, deps.dbaasClient, deps.versionServiceClient, deps.grafanaClient) - if deps.ha.ActiveMode() { + deps.ha.AddLeaderService(highavailability.NewContextService("dbaas-register", func(ctx context.Context) error { deps.dbaasInitializer.RegisterKubernetesServer(k8sServer) - } + return nil + })) dbaasv1beta1.RegisterKubernetesServer(gRPCServer, k8sServer) dbaasv1beta1.RegisterDBClustersServer(gRPCServer, managementdbaas.NewDBClusterService(deps.db, deps.grafanaClient, deps.versionServiceClient)) dbaasv1beta1.RegisterPXCClustersServer(gRPCServer, managementdbaas.NewPXCClusterService(deps.db, deps.grafanaClient, deps.componentsService, deps.versionServiceClient.GetVersionServiceURL())) @@ -587,27 +588,21 @@ func setup(ctx context.Context, deps *setupDeps) bool { deps.l.Warnf("VictoriaMetrics problem: %+v.", err) return false } - if deps.ha.ActiveMode() { - deps.vmdb.RequestConfigurationUpdate() - } + deps.vmdb.RequestConfigurationUpdate() deps.l.Infof("Checking VMAlert...") if err = deps.vmalert.IsReady(ctx); err != nil { deps.l.Warnf("VMAlert problem: %+v.", err) return false } - if deps.ha.ActiveMode() { - deps.vmalert.RequestConfigurationUpdate() - } + deps.vmalert.RequestConfigurationUpdate() deps.l.Infof("Checking Alertmanager...") if err = deps.alertmanager.IsReady(ctx); err != nil { deps.l.Warnf("Alertmanager problem: %+v.", err) return false } - if deps.ha.ActiveMode() { - deps.alertmanager.RequestConfigurationUpdate() - } + deps.alertmanager.RequestConfigurationUpdate() deps.l.Info("Setup completed.") return true @@ -730,9 +725,33 @@ func main() { Envar("PERCONA_TEST_POSTGRES_SSL_CERT_PATH"). String() - passiveModeF := kingpin.Flag("ha-passive-mode", "Run in Passive mode"). - Envar("PERCONA_TEST_HA_PASSIVE"). + haEnabled := kingpin.Flag("ha-enable", "Enable HA"). + Envar("PERCONA_TEST_HA_ENABLE"). + Bool() + haBootstrap := kingpin.Flag("ha-bootstrap", "Bootstrap HA cluster"). + Envar("PERCONA_TEST_HA_BOOTSTRAP"). Bool() + haNodeID := kingpin.Flag("ha-node-id", "HA Node ID"). + Envar("PERCONA_TEST_HA_NODE_ID"). + String() + haAdvertiseAddress := kingpin.Flag("ha-advertise-address", "HA Advertise address"). + Envar("PERCONA_TEST_HA_ADVERTISE_ADDRESS"). + String() + haGossipNodes := kingpin.Flag("ha-peers", "HA Peers"). + Envar("PERCONA_TEST_HA_PEERS"). + String() + haRaftPort := kingpin.Flag("ha-raft-port", "HA raft port"). + Envar("PERCONA_TEST_HA_RAFT_PORT"). + Default("9760"). + Int() + haGossipPort := kingpin.Flag("ha-gossip-port", "HA gossip port"). + Envar("PERCONA_TEST_HA_GOSSIP_PORT"). + Default("9761"). + Int() + //haGrafanaPort := kingpin.Flag("ha-grafana-port", "HA grafana port"). + // Envar("PERCONA_TEST_HA_GRAFANA_PORT"). + // Default("9760"). + // Int() supervisordConfigDirF := kingpin.Flag("supervisord-config-dir", "Supervisord configuration directory").Required().String() @@ -763,7 +782,22 @@ func main() { ctx = logger.Set(ctx, "main") defer l.Info("Done.") - ha := highavailability.New(passiveModeF) + haParams := &highavailability.Params{ + Enabled: *haEnabled, + Bootstrap: *haBootstrap, + NodeID: *haNodeID, + AdvertiseAddress: *haAdvertiseAddress, + Nodes: strings.Split(*haGossipNodes, ","), + RaftPort: *haRaftPort, + GossipPort: *haGossipPort, + } + ha := highavailability.New(haParams) + go func() { + err := ha.Run(ctx) + if err != nil { + l.Panicf("cannot start high availability service: %+v", err) + } + }() cfg := config.NewService() if err := cfg.Load(); err != nil { @@ -821,7 +855,7 @@ func main() { } defer sqlDB.Close() //nolint:errcheck - if ha.ActiveMode() { + if ha.Bootstrap() { migrateDB(ctx, sqlDB, setupParams) } @@ -830,7 +864,7 @@ func main() { prom.MustRegister(reformL) db := reform.NewDB(sqlDB, postgresql.Dialect, reformL) - if ha.ActiveMode() { + if ha.Bootstrap() { // Generate unique PMM Server ID if it's not already set. err = models.SetPMMServerID(db) if err != nil { @@ -939,7 +973,9 @@ func main() { l.Fatalf("Could not create templates service: %s", err) } // We should collect templates before rules service created, because it will regenerate rule files on startup. - templatesService.CollectTemplates(ctx) + if ha.Bootstrap() { + templatesService.CollectTemplates(ctx) + } rulesService := ia.NewRulesService(db, templatesService, vmalert, alertManager) alertsService := ia.NewAlertsService(db, alertManager, templatesService) @@ -1039,7 +1075,7 @@ func main() { }() } - if ha.ActiveMode() { + if !*haEnabled { // Set all agents status to unknown at startup. The ones that are alive // will get their status updated after they connect to the pmm-managed. err = agentsHandler.SetAllAgentsStatusUnknown(ctx) @@ -1081,13 +1117,10 @@ func main() { alertManager.Run(ctx) }() - wg.Add(1) - go func() { - defer wg.Done() - if ha.ActiveMode() { - checksService.Run(ctx) - } - }() + ha.AddLeaderService(highavailability.NewContextService("checks", func(ctx context.Context) error { + checksService.Run(ctx) + return nil + })) wg.Add(1) go func() { @@ -1096,26 +1129,21 @@ func main() { }() wg.Add(1) - go func() { + ha.AddLeaderService(highavailability.NewContextService("telemetry", func(ctx context.Context) error { defer wg.Done() - if ha.ActiveMode() { - telemetry.Run(ctx) - } - }() + telemetry.Run(ctx) + return nil + })) - wg.Add(1) - go func() { - defer wg.Done() - if ha.ActiveMode() { - schedulerService.Run(ctx) - } - }() + ha.AddLeaderService(highavailability.NewContextService("scheduler", func(ctx context.Context) error { + schedulerService.Run(ctx) + return nil + })) - wg.Add(1) - go func() { - defer wg.Done() + ha.AddLeaderService(highavailability.NewContextService("versionCache", func(ctx context.Context) error { versionCache.Run(ctx) - }() + return nil + })) wg.Add(1) go func() { @@ -1176,11 +1204,10 @@ func main() { runDebugServer(ctx) }() - wg.Add(1) - go func() { - defer wg.Done() + ha.AddLeaderService(highavailability.NewContextService("cleaner", func(ctx context.Context) error { cleaner.Run(ctx, cleanInterval, cleanOlderThan) - }() + return nil + })) if settings.DBaaS.Enabled { err = supervisord.RestartSupervisedService("dbaas-controller") if err != nil { @@ -1200,6 +1227,7 @@ func main() { } } + ha.Wait() wg.Wait() } diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index ad86b939a7..36421b6ed5 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -65,5 +65,6 @@ type victoriaMetricsParams interface { // highAvailablityService is an interface to get information related to High Availability. type highAvailablityService interface { - PassiveMode() bool + //TODO: extend by send message + BroadcastMessage(message []byte) } diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go index 026bb6db9c..0bf7f8ebc1 100644 --- a/managed/services/agents/handler.go +++ b/managed/services/agents/handler.go @@ -63,9 +63,6 @@ func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, v // Run takes over pmm-agent gRPC stream and runs it until completion. func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { - if h.ha.PassiveMode() { - return status.Errorf(codes.FailedPrecondition, "Current PMM Server is running in passive mode.") - } disconnectReason := "unknown" ctx := stream.Context() diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index e326351d00..d12984f5b2 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -16,20 +16,237 @@ // Package highavailability contains everything related to high availability. package highavailability -import "github.com/AlekSi/pointer" +import ( + "context" + "fmt" + "io" + "net" + "time" + + "github.com/hashicorp/memberlist" + "github.com/hashicorp/raft" + "github.com/sirupsen/logrus" +) + +type Params struct { + Enabled bool + Bootstrap bool + NodeID string + AdvertiseAddress string + Nodes []string + RaftPort int + GossipPort int +} type Service struct { - passiveMode bool + params *Params + bootstrapCluster bool + + services *services + + messages chan []byte + nodeCh chan *memberlist.Node + leaderCh chan raft.Observation + + l *logrus.Entry + + raftNode *raft.Raft + memberlist *memberlist.Memberlist +} + +func (s *Service) Apply(logEntry *raft.Log) interface{} { + s.l.Printf("raft: got a message: %s", string(logEntry.Data)) + return nil +} + +func (s *Service) Snapshot() (raft.FSMSnapshot, error) { + //TODO implement me + return nil, nil +} + +func (s *Service) Restore(snapshot io.ReadCloser) error { + //TODO implement me + return nil +} + +func New(params *Params) *Service { + return &Service{ + params: params, + bootstrapCluster: params.Bootstrap, + services: newServices(), + nodeCh: make(chan *memberlist.Node, 5), + leaderCh: make(chan raft.Observation), + messages: make(chan []byte), + l: logrus.WithField("component", "ha"), + } +} + +func (s *Service) Run(ctx context.Context) error { + go func() { + for { + select { + case <-s.services.ServiceAdded(): + if s.IsLeader() { + s.services.StartAllServices(ctx) + } + case <-ctx.Done(): + return + } + } + }() + + if !s.params.Enabled { + return nil + } + // Create the memberlist configuration + memberlistConfig := memberlist.DefaultWANConfig() + memberlistConfig.Name = s.params.NodeID + memberlistConfig.BindAddr = "0.0.0.0" + memberlistConfig.BindPort = s.params.GossipPort + memberlistConfig.AdvertiseAddr = s.params.AdvertiseAddress + memberlistConfig.AdvertisePort = s.params.GossipPort + memberlistConfig.Events = &eventDelegate{s.nodeCh} + + // Create the memberlist + var err error + s.memberlist, err = memberlist.Create(memberlistConfig) + if err != nil { + return fmt.Errorf("failed to create memberlist: %q", err) + } + defer s.memberlist.Leave(5 * time.Second) + + // Create the Raft configuration + raftConfig := raft.DefaultConfig() + raftConfig.LocalID = raft.ServerID(s.params.NodeID) + raftConfig.LogLevel = "INFO" + + // Create a new Raft transport + raa, err := net.ResolveTCPAddr("", fmt.Sprintf("%s:%d", s.params.AdvertiseAddress, s.params.RaftPort)) + if err != nil { + return err + } + raftTrans, err := raft.NewTCPTransport(fmt.Sprintf("0.0.0.0:%d", s.params.RaftPort), raa, 3, 10*time.Second, nil) + if err != nil { + return err + } + + // Create a new Raft node + s.raftNode, err = raft.NewRaft(raftConfig, s, raft.NewInmemStore(), raft.NewInmemStore(), raft.NewInmemSnapshotStore(), raftTrans) + if err != nil { + return err + } + + if s.bootstrapCluster { + // Start the Raft node + cfg := raft.Configuration{ + Servers: []raft.Server{ + { + Suffrage: raft.Voter, + ID: raft.ServerID(s.params.NodeID), + Address: raft.ServerAddress(raa.String()), + }, + }, + } + if err := s.raftNode.BootstrapCluster(cfg).Error(); err != nil { + return fmt.Errorf("failed to bootstrap Raft cluster: %q", err) + } + } else { + _, err := s.memberlist.Join(s.params.Nodes) + if err != nil { + return fmt.Errorf("failed to join memberlist cluster: %q", err) + } + } + + go s.runLeaderObserver(ctx) + + go s.runRaftNodesSynchronizer(ctx) + + go func() { + for { + select { + case message := <-s.messages: + //for _, member := range memberlist.Members() { + // if member.Name == s.params.NodeID { + // continue + // } + // memberlist.SendReliable(member, message) + //} + if err := s.raftNode.Apply(message, 5*time.Second).Error(); err != nil { + s.l.Errorf("Failed to apply command: %q", err) + } + s.l.Debugf("Broadcasting message: %s", message) + case <-ctx.Done(): + return + } + } + }() + + <-ctx.Done() + + return nil +} + +func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { + for { + select { + case node := <-s.nodeCh: + if !s.IsLeader() { + continue + } + switch node.State { + case memberlist.StateAlive: + s.raftNode.AddVoter(raft.ServerID(node.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", node.Addr.String(), s.params.RaftPort)), s.raftNode.AppliedIndex(), 10*time.Second).Error() + case memberlist.StateLeft, memberlist.StateDead: + s.raftNode.RemoveServer(raft.ServerID(node.Name), 0, 10*time.Second).Error() + } + case <-ctx.Done(): + return + } + } +} + +func (s *Service) runLeaderObserver(ctx context.Context) { + for { + select { + case isLeader := <-s.raftNode.LeaderCh(): + if isLeader { + s.services.StartAllServices(ctx) + // This node is the leader + s.l.Printf("I am a leader!") + peers := s.memberlist.Members() + for _, peer := range peers { + if peer.Name == s.params.NodeID { + continue + } + if err := s.raftNode.AddVoter(raft.ServerID(peer.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", peer.Addr.String(), s.params.RaftPort)), s.raftNode.AppliedIndex(), 10*time.Second).Error(); err != nil { + s.l.Warnf("Failed to add Raft member: %v", err) + } + } + } else { + s.services.StopRunningServices() + } + case <-ctx.Done(): + return + } + } +} + +func (s *Service) AddLeaderService(leaderService LeaderService) { + s.services.Add(leaderService) +} + +func (s *Service) BroadcastMessage(message []byte) { + s.messages <- message } -func New(passiveMode *bool) *Service { - return &Service{passiveMode: pointer.GetBool(passiveMode)} +func (s *Service) IsLeader() bool { + return (s.raftNode != nil && s.raftNode.State() == raft.Leader) || !s.params.Enabled } -func (s *Service) ActiveMode() bool { - return !s.passiveMode +func (s *Service) Bootstrap() bool { + return s.params.Bootstrap } -func (s *Service) PassiveMode() bool { - return s.passiveMode +func (s *Service) Wait() { + s.services.Wait() } diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go new file mode 100644 index 0000000000..47db61bf4f --- /dev/null +++ b/managed/services/highavailability/leaderservice.go @@ -0,0 +1,59 @@ +package highavailability + +import ( + "context" + "sync" +) + +type LeaderService interface { + Start(ctx context.Context) error + Stop() + ID() string +} + +type ContextService struct { + id string + + startFunc func(context.Context) error + ctx context.Context + cancel context.CancelFunc +} + +func NewContextService(id string, startFunc func(context.Context) error) *ContextService { + return &ContextService{ + startFunc: startFunc, + } +} +func (s *ContextService) ID() string { + return s.id +} +func (s *ContextService) Start(ctx context.Context) error { + s.ctx, s.cancel = context.WithCancel(ctx) + return s.startFunc(ctx) +} +func (s *ContextService) Stop() { + s.cancel() +} + +type RunOnceService struct { + id string + + startFunc func(context.Context) error + o sync.Once +} + +func NewRunOnceService(id string, startFunc func(context.Context) error) *RunOnceService { + return &RunOnceService{ + startFunc: startFunc, + } +} +func (s *RunOnceService) ID() string { + return s.id +} +func (s *RunOnceService) Start(ctx context.Context) error { + s.o.Do(func() { + s.startFunc(ctx) + }) + return nil +} +func (s *RunOnceService) Stop() {} diff --git a/managed/services/highavailability/memberlist.go b/managed/services/highavailability/memberlist.go new file mode 100644 index 0000000000..b04c4eb320 --- /dev/null +++ b/managed/services/highavailability/memberlist.go @@ -0,0 +1,22 @@ +package highavailability + +import ( + "github.com/hashicorp/memberlist" + "github.com/sirupsen/logrus" +) + +type eventDelegate struct { + nodeCh chan *memberlist.Node +} + +func (e *eventDelegate) NotifyJoin(node *memberlist.Node) { + e.nodeCh <- node +} + +func (e *eventDelegate) NotifyLeave(node *memberlist.Node) { + e.nodeCh <- node +} + +func (e *eventDelegate) NotifyUpdate(node *memberlist.Node) { + logrus.Printf("NotifyUpdate: %v", node) +} diff --git a/managed/services/highavailability/raft.go b/managed/services/highavailability/raft.go new file mode 100644 index 0000000000..939d0361cd --- /dev/null +++ b/managed/services/highavailability/raft.go @@ -0,0 +1,77 @@ +package highavailability + +import ( + "context" + "github.com/hashicorp/memberlist" + "github.com/hashicorp/raft" + "github.com/sirupsen/logrus" + "io" + "sync" +) + +// raftFSM is a simple example implementation of a Raft FSM. +type raftFSM struct { + m *memberlist.Memberlist + mu sync.Mutex + nodeID raft.ServerID + leaderID raft.ServerID + + l *logrus.Entry +} + +func newFSM(m *memberlist.Memberlist, nodeID raft.ServerID) *raftFSM { + return &raftFSM{ + m: m, + nodeID: nodeID, + l: logrus.WithField("component", "raftFSM"), + } +} +func (f *raftFSM) Run(ctx context.Context, leaderCh chan raft.Observation) { + for { + select { + case o := <-leaderCh: + l := o.Data.(raft.LeaderObservation) + f.LeaderChange(l.LeaderID) + case <-ctx.Done(): + return + } + } +} + +func (f *raftFSM) Apply(logEntry *raft.Log) interface{} { + f.mu.Lock() + defer f.mu.Unlock() + f.l.Printf("raft: got a message: %s", string(logEntry.Data)) + // If this node is the leader, broadcast the message to all the other nodes + if f.nodeID == f.leaderID { + f.l.Printf("Broadcasting message: %s", string(logEntry.Data)) + for _, peer := range f.m.Members() { + if peer.Name == string(f.nodeID) { + continue + } + f.l.Printf("Sending message to: %s", peer.Name) + err := f.m.SendReliable(peer, logEntry.Data) + if err != nil { + f.l.Printf("Failed to send message to node %s: %v", peer.Name, err) + } + } + } + + return nil + +} + +func (f *raftFSM) Snapshot() (raft.FSMSnapshot, error) { + return nil, nil +} + +func (f *raftFSM) Restore(rc io.ReadCloser) error { + return nil +} + +func (f *raftFSM) LeaderChange(leader raft.ServerID) { + f.mu.Lock() + defer f.mu.Unlock() + f.l.Printf("Leader changed from %s to %s", f.leaderID, leader) + f.leaderID = leader +} diff --git a/managed/services/highavailability/services.go b/managed/services/highavailability/services.go new file mode 100644 index 0000000000..8b9e267b5b --- /dev/null +++ b/managed/services/highavailability/services.go @@ -0,0 +1,91 @@ +package highavailability + +import ( + "context" + "fmt" + "sync" +) + +type services struct { + wg *sync.WaitGroup + rw sync.Mutex + all map[string]LeaderService + running map[string]LeaderService + + serviceAdded chan struct{} +} + +func newServices() *services { + return &services{ + wg: new(sync.WaitGroup), + all: make(map[string]LeaderService), + running: make(map[string]LeaderService), + serviceAdded: make(chan struct{}), + } +} + +func (s *services) Add(service LeaderService) error { + s.rw.Lock() + defer s.rw.Unlock() + + id := service.ID() + if _, ok := s.all[id]; ok { + return fmt.Errorf("service with id %s is already exist", id) + } + s.all[id] = service + select { + case s.serviceAdded <- struct{}{}: + default: + } + return nil +} + +func (s *services) StartAllServices(ctx context.Context) { + s.rw.Lock() + defer s.rw.Unlock() + + for id, service := range s.all { + if _, ok := s.running[id]; !ok { + s.wg.Add(1) + go func(ls LeaderService) { + defer s.wg.Done() + ls.Start(ctx) + s.rw.Lock() + defer s.rw.Unlock() + delete(s.running, id) + }(service) + s.running[id] = service + } + } +} + +func (s *services) StartService(ctx context.Context, id string) error { + s.rw.Lock() + defer s.rw.Unlock() + service, ok := s.all[id] + if !ok { + return fmt.Errorf("service with ID %s not found", id) + } + if _, ok := s.running[id]; !ok { + go service.Start(ctx) + s.running[id] = service + } + return nil +} + +func (s *services) StopRunningServices() { + s.rw.Lock() + defer s.rw.Unlock() + + for _, service := range s.running { + go service.Stop() + } +} + +func (s *services) ServiceAdded() chan struct{} { + return s.serviceAdded +} + +func (s *services) Wait() { + s.wg.Wait() +} From fd4bc423b8a60d8e03e0c04f92e88b7b3da0e9bb Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 30 May 2023 11:55:18 +0300 Subject: [PATCH 30/77] PMM-12078 HA Phase 3 Active-Active PoC. --- Makefile.include | 2 +- agent/agents/process/process_logger_test.go | 14 + agent/cmd/pmm-agent-entrypoint/main.go | 4 +- api/hapb/ha.pb.go | 295 +++++++++++++++ api/hapb/ha.pb.validate.go | 357 ++++++++++++++++++ api/hapb/ha.proto | 21 ++ docker-compose.yml | 34 +- go.mod | 3 +- go.sum | 20 +- managed/cmd/pmm-managed-init/main.go | 1 + managed/cmd/pmm-managed/main.go | 30 +- managed/services/agents/actions.go | 42 +-- managed/services/agents/agents.go | 4 +- managed/services/agents/channel/channel.go | 12 +- .../services/agents/channel/channel_test.go | 2 +- managed/services/agents/connection_checker.go | 2 +- managed/services/agents/deps.go | 11 +- managed/services/agents/handler.go | 24 +- managed/services/agents/jobs.go | 12 +- managed/services/agents/registry.go | 42 ++- managed/services/agents/state.go | 24 +- managed/services/agents/versioner.go | 2 +- managed/services/highavailability/channel.go | 19 + .../highavailability/highavailability.go | 133 ++++--- .../highavailability/leaderservice.go | 5 + managed/services/highavailability/raft.go | 7 +- managed/services/supervisord/pmm_config.go | 2 + 27 files changed, 961 insertions(+), 163 deletions(-) create mode 100644 api/hapb/ha.pb.go create mode 100644 api/hapb/ha.pb.validate.go create mode 100644 api/hapb/ha.proto create mode 100644 managed/services/highavailability/channel.go diff --git a/Makefile.include b/Makefile.include index 55d8d48f6d..41027570a0 100644 --- a/Makefile.include +++ b/Makefile.include @@ -47,7 +47,7 @@ gen-api: ## Generate PMM API bin/buf generate -v api - for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/service api/platformpb api/userpb api/onboardingpb; do \ + for API in api/agentlocalpb api/serverpb api/inventorypb api/hapb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/service api/platformpb api/userpb api/onboardingpb; do \ set -x ; \ bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \ bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \ diff --git a/agent/agents/process/process_logger_test.go b/agent/agents/process/process_logger_test.go index ccf7a53cf2..9117496d12 100644 --- a/agent/agents/process/process_logger_test.go +++ b/agent/agents/process/process_logger_test.go @@ -127,6 +127,20 @@ func TestProcessLogger(t *testing.T) { 0, 0, }, + { + "redact keywords with special symbols", + 3, + []string{ + "text\nsecond ", + "line\nthird r.o,w line\n", + "fourth ", + "line\nlast r.o,w\n", + }, + []string{"r.o,w"}, + []string{"third *** line", "fourth line", "last ***"}, + 0, + 0, + }, } for _, tt := range tests { t.Run(tt.testName, func(t *testing.T) { diff --git a/agent/cmd/pmm-agent-entrypoint/main.go b/agent/cmd/pmm-agent-entrypoint/main.go index ee6d2d5329..af90699131 100644 --- a/agent/cmd/pmm-agent-entrypoint/main.go +++ b/agent/cmd/pmm-agent-entrypoint/main.go @@ -71,7 +71,7 @@ var ( var pmmAgentProcessID = 0 func runPmmAgent(ctx context.Context, commandLineArgs []string, restartPolicy restartPolicy, l *logrus.Entry, pmmAgentSidecarSleep int) int { - pmmAgentFullCommand := "pmm-admin " + strings.Join(commandLineArgs, " ") + pmmAgentFullCommand := "pmm-agent " + strings.Join(commandLineArgs, " ") for { select { case <-ctx.Done(): @@ -79,7 +79,7 @@ func runPmmAgent(ctx context.Context, commandLineArgs []string, restartPolicy re default: } var exitCode int - l.Infof("Starting 'pmm-admin %s'...", strings.Join(commandLineArgs, " ")) + l.Infof("Starting 'pmm-agent %s'...", strings.Join(commandLineArgs, " ")) cmd := commandPmmAgent(commandLineArgs) if err := cmd.Start(); err != nil { l.Errorf("Can't run: '%s', Error: %s", commandLineArgs, err) diff --git a/api/hapb/ha.pb.go b/api/hapb/ha.pb.go new file mode 100644 index 0000000000..99466f5ecf --- /dev/null +++ b/api/hapb/ha.pb.go @@ -0,0 +1,295 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc (unknown) +// source: hapb/ha.proto + +package hapb + +import ( + reflect "reflect" + sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + + agentpb "github.com/percona/pmm/api/agentpb" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Metadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Sender string `protobuf:"bytes,1,opt,name=sender,proto3" json:"sender,omitempty"` + Receiver string `protobuf:"bytes,2,opt,name=receiver,proto3" json:"receiver,omitempty"` +} + +func (x *Metadata) Reset() { + *x = Metadata{} + if protoimpl.UnsafeEnabled { + mi := &file_hapb_ha_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metadata) ProtoMessage() {} + +func (x *Metadata) ProtoReflect() protoreflect.Message { + mi := &file_hapb_ha_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metadata.ProtoReflect.Descriptor instead. +func (*Metadata) Descriptor() ([]byte, []int) { + return file_hapb_ha_proto_rawDescGZIP(), []int{0} +} + +func (x *Metadata) GetSender() string { + if x != nil { + return x.Sender + } + return "" +} + +func (x *Metadata) GetReceiver() string { + if x != nil { + return x.Receiver + } + return "" +} + +type ClusterMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Metadata *Metadata `protobuf:"bytes,2,opt,name=metadata,proto3" json:"metadata,omitempty"` + // Types that are assignable to Payload: + // + // *ClusterMessage_ServerMessage + // *ClusterMessage_AgentMessage + Payload isClusterMessage_Payload `protobuf_oneof:"payload"` +} + +func (x *ClusterMessage) Reset() { + *x = ClusterMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_hapb_ha_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClusterMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClusterMessage) ProtoMessage() {} + +func (x *ClusterMessage) ProtoReflect() protoreflect.Message { + mi := &file_hapb_ha_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClusterMessage.ProtoReflect.Descriptor instead. +func (*ClusterMessage) Descriptor() ([]byte, []int) { + return file_hapb_ha_proto_rawDescGZIP(), []int{1} +} + +func (x *ClusterMessage) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *ClusterMessage) GetMetadata() *Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (m *ClusterMessage) GetPayload() isClusterMessage_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (x *ClusterMessage) GetServerMessage() *agentpb.ServerMessage { + if x, ok := x.GetPayload().(*ClusterMessage_ServerMessage); ok { + return x.ServerMessage + } + return nil +} + +func (x *ClusterMessage) GetAgentMessage() *agentpb.AgentMessage { + if x, ok := x.GetPayload().(*ClusterMessage_AgentMessage); ok { + return x.AgentMessage + } + return nil +} + +type isClusterMessage_Payload interface { + isClusterMessage_Payload() +} + +type ClusterMessage_ServerMessage struct { + ServerMessage *agentpb.ServerMessage `protobuf:"bytes,11,opt,name=server_message,json=serverMessage,proto3,oneof"` +} + +type ClusterMessage_AgentMessage struct { + AgentMessage *agentpb.AgentMessage `protobuf:"bytes,12,opt,name=agent_message,json=agentMessage,proto3,oneof"` +} + +func (*ClusterMessage_ServerMessage) isClusterMessage_Payload() {} + +func (*ClusterMessage_AgentMessage) isClusterMessage_Payload() {} + +var File_hapb_ha_proto protoreflect.FileDescriptor + +var file_hapb_ha_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x68, 0x61, 0x70, 0x62, 0x2f, 0x68, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x02, 0x68, 0x61, 0x1a, 0x13, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x70, 0x62, 0x2f, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3e, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, + 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x22, 0xd0, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x08, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, + 0x68, 0x61, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3d, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x12, 0x3a, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x48, 0x00, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x5a, 0x0a, 0x06, 0x63, + 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x42, 0x07, 0x48, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x1f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x65, 0x72, + 0x63, 0x6f, 0x6e, 0x61, 0x2f, 0x70, 0x6d, 0x6d, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x61, 0x70, + 0x62, 0xa2, 0x02, 0x03, 0x48, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x48, 0x61, 0xca, 0x02, 0x02, 0x48, + 0x61, 0xe2, 0x02, 0x0e, 0x48, 0x61, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0xea, 0x02, 0x02, 0x48, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_hapb_ha_proto_rawDescOnce sync.Once + file_hapb_ha_proto_rawDescData = file_hapb_ha_proto_rawDesc +) + +func file_hapb_ha_proto_rawDescGZIP() []byte { + file_hapb_ha_proto_rawDescOnce.Do(func() { + file_hapb_ha_proto_rawDescData = protoimpl.X.CompressGZIP(file_hapb_ha_proto_rawDescData) + }) + return file_hapb_ha_proto_rawDescData +} + +var ( + file_hapb_ha_proto_msgTypes = make([]protoimpl.MessageInfo, 2) + file_hapb_ha_proto_goTypes = []interface{}{ + (*Metadata)(nil), // 0: ha.Metadata + (*ClusterMessage)(nil), // 1: ha.ClusterMessage + (*agentpb.ServerMessage)(nil), // 2: agent.ServerMessage + (*agentpb.AgentMessage)(nil), // 3: agent.AgentMessage + } +) + +var file_hapb_ha_proto_depIdxs = []int32{ + 0, // 0: ha.ClusterMessage.metadata:type_name -> ha.Metadata + 2, // 1: ha.ClusterMessage.server_message:type_name -> agent.ServerMessage + 3, // 2: ha.ClusterMessage.agent_message:type_name -> agent.AgentMessage + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_hapb_ha_proto_init() } +func file_hapb_ha_proto_init() { + if File_hapb_ha_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_hapb_ha_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Metadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hapb_ha_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClusterMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_hapb_ha_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*ClusterMessage_ServerMessage)(nil), + (*ClusterMessage_AgentMessage)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_hapb_ha_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_hapb_ha_proto_goTypes, + DependencyIndexes: file_hapb_ha_proto_depIdxs, + MessageInfos: file_hapb_ha_proto_msgTypes, + }.Build() + File_hapb_ha_proto = out.File + file_hapb_ha_proto_rawDesc = nil + file_hapb_ha_proto_goTypes = nil + file_hapb_ha_proto_depIdxs = nil +} diff --git a/api/hapb/ha.pb.validate.go b/api/hapb/ha.pb.validate.go new file mode 100644 index 0000000000..ff8e848601 --- /dev/null +++ b/api/hapb/ha.pb.validate.go @@ -0,0 +1,357 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: hapb/ha.proto + +package hapb + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on Metadata with the rules defined in the +// proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *Metadata) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Metadata with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in MetadataMultiError, or nil +// if none found. +func (m *Metadata) ValidateAll() error { + return m.validate(true) +} + +func (m *Metadata) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Sender + + // no validation rules for Receiver + + if len(errors) > 0 { + return MetadataMultiError(errors) + } + + return nil +} + +// MetadataMultiError is an error wrapping multiple validation errors returned +// by Metadata.ValidateAll() if the designated constraints aren't met. +type MetadataMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m MetadataMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m MetadataMultiError) AllErrors() []error { return m } + +// MetadataValidationError is the validation error returned by +// Metadata.Validate if the designated constraints aren't met. +type MetadataValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e MetadataValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e MetadataValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e MetadataValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e MetadataValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e MetadataValidationError) ErrorName() string { return "MetadataValidationError" } + +// Error satisfies the builtin error interface +func (e MetadataValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sMetadata.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = MetadataValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = MetadataValidationError{} + +// Validate checks the field values on ClusterMessage with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *ClusterMessage) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on ClusterMessage with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in ClusterMessageMultiError, +// or nil if none found. +func (m *ClusterMessage) ValidateAll() error { + return m.validate(true) +} + +func (m *ClusterMessage) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Id + + if all { + switch v := interface{}(m.GetMetadata()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ClusterMessageValidationError{ + field: "Metadata", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ClusterMessageValidationError{ + field: "Metadata", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetMetadata()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ClusterMessageValidationError{ + field: "Metadata", + reason: "embedded message failed validation", + cause: err, + } + } + } + + switch v := m.Payload.(type) { + case *ClusterMessage_ServerMessage: + if v == nil { + err := ClusterMessageValidationError{ + field: "Payload", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetServerMessage()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ClusterMessageValidationError{ + field: "ServerMessage", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ClusterMessageValidationError{ + field: "ServerMessage", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetServerMessage()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ClusterMessageValidationError{ + field: "ServerMessage", + reason: "embedded message failed validation", + cause: err, + } + } + } + + case *ClusterMessage_AgentMessage: + if v == nil { + err := ClusterMessageValidationError{ + field: "Payload", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetAgentMessage()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ClusterMessageValidationError{ + field: "AgentMessage", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ClusterMessageValidationError{ + field: "AgentMessage", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetAgentMessage()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ClusterMessageValidationError{ + field: "AgentMessage", + reason: "embedded message failed validation", + cause: err, + } + } + } + + default: + _ = v // ensures v is used + } + + if len(errors) > 0 { + return ClusterMessageMultiError(errors) + } + + return nil +} + +// ClusterMessageMultiError is an error wrapping multiple validation errors +// returned by ClusterMessage.ValidateAll() if the designated constraints +// aren't met. +type ClusterMessageMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m ClusterMessageMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m ClusterMessageMultiError) AllErrors() []error { return m } + +// ClusterMessageValidationError is the validation error returned by +// ClusterMessage.Validate if the designated constraints aren't met. +type ClusterMessageValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e ClusterMessageValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e ClusterMessageValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e ClusterMessageValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e ClusterMessageValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e ClusterMessageValidationError) ErrorName() string { return "ClusterMessageValidationError" } + +// Error satisfies the builtin error interface +func (e ClusterMessageValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sClusterMessage.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = ClusterMessageValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = ClusterMessageValidationError{} diff --git a/api/hapb/ha.proto b/api/hapb/ha.proto new file mode 100644 index 0000000000..4c1b5f6b78 --- /dev/null +++ b/api/hapb/ha.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package ha; + +import "agentpb/agent.proto"; + +option go_package = "api/hapb;hapb"; + +message Metadata { + string sender = 1; + string receiver = 2; +} + +message ClusterMessage { + uint32 id = 1; + Metadata metadata = 2; + oneof payload { + agent.ServerMessage server_message = 11; + agent.AgentMessage agent_message = 12; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 98042a4e63..33ec360933 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,7 +110,8 @@ services: ports: - ${CH_PORT:-9000}:9000 networks: - - ${NETWORK:-default} + ha: + ipv4_address: 172.20.0.7 victoriametrics: profiles: @@ -133,7 +134,8 @@ services: - "--httpListenAddr=:8428" - "--influxListenAddr=:8089" networks: - - ${NETWORK:-default} + ha: + ipv4_address: 172.20.0.4 # PMM with external Postgres DB pg: @@ -163,7 +165,8 @@ services: # -c ssl_cert_file=/certs/server.crt # -c ssl_key_file=/certs/server.key networks: - - ${NETWORK:-default} + ha: + ipv4_address: 172.20.0.3 volumes: - ./managed/testdata/pg/conf/:/conf/ - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ @@ -179,7 +182,8 @@ services: container_name: pmm-managed-server hostname: pmm-managed-server networks: - - ${NETWORK:-default} + ha: + ipv4_address: 172.20.0.5 environment: - PMM_RELEASE_PATH=/root/go/bin - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} @@ -197,6 +201,7 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 + - PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 - PERCONA_TEST_POSTGRES_ADDR=pg:5432 - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-managed @@ -208,15 +213,15 @@ services: - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana # - GF_DATABASE_SSL_MODE=require # - PMM_DEBUG=1 + - GO_VERSION=1.20 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - PERCONA_TEST_HA_ENABLE=1 - PERCONA_TEST_HA_BOOTSTRAP=1 - PERCONA_TEST_HA_NODE_ID=pmm-managed-server - - PERCONA_TEST_HA_ADVERTISE_ADDRESS=pmm-managed-server - - PERCONA_TEST_HA_PEERS=pmm-managed-server,pmm-managed-server-passive + - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=pmm-managed-server:9095 + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.5:9095 - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9095,pmm-managed-server-passive:9095 extra_hosts: @@ -260,8 +265,9 @@ services: image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-managed-server-passive hostname: pmm-managed-server-passive - networks: - - ${NETWORK:-default} + networks: + ha: + ipv4_address: 172.20.0.6 environment: - PMM_RELEASE_PATH=/root/go/bin - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} @@ -278,6 +284,7 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 + - PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 - PERCONA_TEST_POSTGRES_ADDR=pg:5432 - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-managed @@ -289,15 +296,16 @@ services: - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana # - GF_DATABASE_SSL_MODE=require # - PMM_DEBUG=1 + - GO_VERSION=1.20 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - PERCONA_TEST_HA_PASSIVE=true - PERCONA_TEST_HA_ENABLE=1 - PERCONA_TEST_HA_NODE_ID=pmm-managed-server-passive - - PERCONA_TEST_HA_ADVERTISE_ADDRESS=pmm-managed-server-passive + - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - PERCONA_TEST_HA_PEERS=pmm-managed-server,pmm-managed-server-passive - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=pmm-managed-server-passive:9095 + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.6:9095 - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9095,pmm-managed-server-passive:9095 extra_hosts: @@ -340,3 +348,7 @@ networks: minikube: external: true name: minikube + ha: + ipam: + config: + - subnet: 172.20.0.0/24 diff --git a/go.mod b/go.mod index 9fc907968f..09ca45224a 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000 github.com/ClickHouse/clickhouse-go/v2 v2.9.0 github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/Jille/raft-grpc-transport v1.4.0 github.com/alecthomas/kong v0.7.1 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 @@ -204,7 +205,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect diff --git a/go.sum b/go.sum index 49f99b3278..55f83a632c 100644 --- a/go.sum +++ b/go.sum @@ -109,9 +109,12 @@ github.com/ClickHouse/clickhouse-go/v2 v2.9.0 h1:vh0D+9p5hKWLYqCfU8Hd7/GMXNfHC84 github.com/ClickHouse/clickhouse-go/v2 v2.9.0/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Jille/raft-grpc-transport v1.4.0 h1:Kwk+IceQD8MpLKOulBu2ignX+aZAEjOhffEhN44sdzQ= +github.com/Jille/raft-grpc-transport v1.4.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -176,6 +179,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30/go.mod h1:Q7 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -509,6 +514,7 @@ github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -820,14 +826,17 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= +github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -846,6 +855,7 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -854,6 +864,7 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.5.0 h1:uNs9EfJ4FwiArZRxxfd/dQ5d33nV31/CdCHArH89hT8= github.com/hashicorp/raft v1.5.0/go.mod h1:pKHB2mf/Y25u3AHNSXVRv+yT+WAnmeTX0BwVppVQV+M= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= @@ -1026,8 +1037,10 @@ github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHef github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -1039,6 +1052,7 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -1572,6 +1586,7 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1697,6 +1712,7 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1777,6 +1793,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1920,6 +1937,7 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/managed/cmd/pmm-managed-init/main.go b/managed/cmd/pmm-managed-init/main.go index 763776c805..3e61334248 100644 --- a/managed/cmd/pmm-managed-init/main.go +++ b/managed/cmd/pmm-managed-init/main.go @@ -54,6 +54,7 @@ func main() { pmmConfigParams := make(map[string]any) pmmConfigParams["DisableInternalDB"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES")) + pmmConfigParams["DisableInternalClickhouse"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE")) pmmConfigParams["PassivePMM"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_HA_PASSIVE")) if err := supervisord.SavePMMConfig(pmmConfigParams); err != nil { logrus.Errorf("PMM Server configuration error: %s.", err) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index f68951260b..f912f996b4 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -253,6 +253,8 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { l.Debug("RPC response latency histogram enabled.") grpcMetrics.EnableHandlingTimeHistogram() } + deps.ha.RegisterGRPC(gRPCServer) + serverpb.RegisterServerServer(gRPCServer, deps.server) agentpb.RegisterAgentServer(gRPCServer, agentgrpc.NewAgentServer(deps.handler)) @@ -748,7 +750,7 @@ func main() { Envar("PERCONA_TEST_HA_GOSSIP_PORT"). Default("9761"). Int() - //haGrafanaPort := kingpin.Flag("ha-grafana-port", "HA grafana port"). + // haGrafanaPort := kingpin.Flag("ha-grafana-port", "HA grafana port"). // Envar("PERCONA_TEST_HA_GRAFANA_PORT"). // Default("9760"). // Int() @@ -792,12 +794,6 @@ func main() { GossipPort: *haGossipPort, } ha := highavailability.New(haParams) - go func() { - err := ha.Run(ctx) - if err != nil { - l.Panicf("cannot start high availability service: %+v", err) - } - }() cfg := config.NewService() if err := cfg.Load(); err != nil { @@ -888,6 +884,7 @@ func main() { qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) + highavailability.NewChannel() agentsRegistry := agents.NewRegistry(db, vmParams) backupRemovalService := backup.NewRemovalService(db, minioClient) pitrTimerangeService := backup.NewPITRTimerangeService(minioClient) @@ -1075,15 +1072,6 @@ func main() { }() } - if !*haEnabled { - // Set all agents status to unknown at startup. The ones that are alive - // will get their status updated after they connect to the pmm-managed. - err = agentsHandler.SetAllAgentsStatusUnknown(ctx) - if err != nil { - l.Errorf("Failed to set status of all agents to invalid at startup: %s", err) - } - } - settings, err := models.GetSettings(sqlDB) if err != nil { l.Fatalf("Failed to get settings: %+v.", err) @@ -1227,7 +1215,15 @@ func main() { } } - ha.Wait() + wg.Add(1) + go func() { + defer wg.Done() + err := ha.Run(ctx) + if err != nil { + l.Panicf("cannot start high availability service: %+v", err) + } + }() + wg.Wait() } diff --git a/managed/services/agents/actions.go b/managed/services/agents/actions.go index d8a206b4e4..817d05cb25 100644 --- a/managed/services/agents/actions.go +++ b/managed/services/agents/actions.go @@ -111,7 +111,7 @@ func (s *ActionsService) StartMySQLExplainAction( Timeout: defaultActionTimeout, } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -138,7 +138,7 @@ func (s *ActionsService) StartMySQLShowCreateTableAction(_ context.Context, id, if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -165,7 +165,7 @@ func (s *ActionsService) StartMySQLShowTableStatusAction(_ context.Context, id, if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -192,7 +192,7 @@ func (s *ActionsService) StartMySQLShowIndexAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -213,7 +213,7 @@ func (s *ActionsService) StartPostgreSQLShowCreateTableAction(_ context.Context, if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -234,7 +234,7 @@ func (s *ActionsService) StartPostgreSQLShowIndexAction(_ context.Context, id, p if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -260,7 +260,7 @@ func (s *ActionsService) StartMongoDBExplainAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -287,7 +287,7 @@ func (s *ActionsService) StartMySQLQueryShowAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -314,7 +314,7 @@ func (s *ActionsService) StartMySQLQuerySelectAction(_ context.Context, id, pmmA if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -334,7 +334,7 @@ func (s *ActionsService) StartPostgreSQLQueryShowAction(_ context.Context, id, p if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -355,7 +355,7 @@ func (s *ActionsService) StartPostgreSQLQuerySelectAction(_ context.Context, id, if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -380,7 +380,7 @@ func (s *ActionsService) StartMongoDBQueryGetParameterAction(_ context.Context, if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -405,7 +405,7 @@ func (s *ActionsService) StartMongoDBQueryBuildInfoAction(_ context.Context, id, if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -430,7 +430,7 @@ func (s *ActionsService) StartMongoDBQueryGetCmdLineOptsAction(_ context.Context if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -455,7 +455,7 @@ func (s *ActionsService) StartMongoDBQueryReplSetGetStatusAction(_ context.Conte if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -480,7 +480,7 @@ func (s *ActionsService) StartMongoDBQueryGetDiagnosticDataAction(_ context.Cont if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -499,7 +499,7 @@ func (s *ActionsService) StartPTSummaryAction(_ context.Context, id, pmmAgentID if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(aRequest) + _, err = agent.Channel().SendAndWaitResponse(aRequest) return err } @@ -522,7 +522,7 @@ func (s *ActionsService) StartPTPgSummaryAction(_ context.Context, id, pmmAgentI if err != nil { return err } - _, err = pmmAgent.channel.SendAndWaitResponse(actionRequest) + _, err = pmmAgent.Channel().SendAndWaitResponse(actionRequest) return err } @@ -548,7 +548,7 @@ func (s *ActionsService) StartPTMongoDBSummaryAction(_ context.Context, id, pmmA if err != nil { return err } - _, err = pmmAgent.channel.SendAndWaitResponse(actionRequest) + _, err = pmmAgent.Channel().SendAndWaitResponse(actionRequest) return err } @@ -574,7 +574,7 @@ func (s *ActionsService) StartPTMySQLSummaryAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = pmmAgent.channel.SendAndWaitResponse(actionRequest) + _, err = pmmAgent.Channel().SendAndWaitResponse(actionRequest) return err } @@ -585,6 +585,6 @@ func (s *ActionsService) StopAction(_ context.Context, actionID string) error { if err != nil { return err } - _, err = agent.channel.SendAndWaitResponse(&agentpb.StopActionRequest{ActionId: actionID}) + _, err = agent.Channel().SendAndWaitResponse(&agentpb.StopActionRequest{ActionId: actionID}) return err } diff --git a/managed/services/agents/agents.go b/managed/services/agents/agents.go index 0d7ffa4853..b77bbe0e99 100644 --- a/managed/services/agents/agents.go +++ b/managed/services/agents/agents.go @@ -46,7 +46,7 @@ func (a *AgentService) Logs(_ context.Context, pmmAgentID, agentID string, limit return nil, 0, err } - resp, err := agent.channel.SendAndWaitResponse(&agentpb.AgentLogsRequest{ + resp, err := agent.Channel().SendAndWaitResponse(&agentpb.AgentLogsRequest{ AgentId: agentID, Limit: limit, }) @@ -79,7 +79,7 @@ func (a *AgentService) PBMSwitchPITR(pmmAgentID, dsn string, files map[string]st Enabled: enabled, } - _, err = agent.channel.SendAndWaitResponse(req) + _, err = agent.Channel().SendAndWaitResponse(req) return err } diff --git a/managed/services/agents/channel/channel.go b/managed/services/agents/channel/channel.go index 0c7ee09243..c6310cf8b3 100644 --- a/managed/services/agents/channel/channel.go +++ b/managed/services/agents/channel/channel.go @@ -17,6 +17,7 @@ package channel import ( + "context" "sync" "sync/atomic" @@ -66,13 +67,18 @@ type Response struct { Error error } +type Stream interface { + Send(*agentpb.ServerMessage) error + Recv() (*agentpb.AgentMessage, error) +} + // Channel encapsulates two-way communication channel between pmm-managed and pmm-agent. // // All exported methods are thread-safe. // //nolint:maligned type Channel struct { - s agentpb.Agent_ConnectServer + s Stream mSent, mRecv uint32 @@ -94,7 +100,7 @@ type Channel struct { // New creates new two-way communication channel with given stream. // // Stream should not be used by the caller after channel is created. -func New(stream agentpb.Agent_ConnectServer) *Channel { +func New(ctx context.Context, stream Stream) *Channel { s := &Channel{ s: stream, @@ -103,7 +109,7 @@ func New(stream agentpb.Agent_ConnectServer) *Channel { closeWait: make(chan struct{}), - l: logger.Get(stream.Context()), + l: logger.Get(ctx), } go s.runReceiver() diff --git a/managed/services/agents/channel/channel_test.go b/managed/services/agents/channel/channel_test.go index 0bb2ce7726..7a8700e57c 100644 --- a/managed/services/agents/channel/channel_test.go +++ b/managed/services/agents/channel/channel_test.go @@ -69,7 +69,7 @@ func setup(t *testing.T, connect func(*Channel) error, expected ...error) (agent agentpb.RegisterAgentServer(server, &testServer{ connectFunc: func(stream agentpb.Agent_ConnectServer) error { - channel = New(stream) + channel = New(nil, stream) return connect(channel) }, }) diff --git a/managed/services/agents/connection_checker.go b/managed/services/agents/connection_checker.go index 201eb31573..0846ccc899 100644 --- a/managed/services/agents/connection_checker.go +++ b/managed/services/agents/connection_checker.go @@ -91,7 +91,7 @@ func (c *ConnectionChecker) CheckConnectionToService(ctx context.Context, q *ref sanitizedDSN = strings.ReplaceAll(request.Dsn, word, "****") } l.Infof("CheckConnectionRequest: type: %s, DSN: %s timeout: %s.", request.Type, sanitizedDSN, request.Timeout) - resp, err := pmmAgent.channel.SendAndWaitResponse(request) + resp, err := pmmAgent.Channel().SendAndWaitResponse(request) if err != nil { return err } diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index 36421b6ed5..7be8fecacb 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -23,6 +23,7 @@ import ( "github.com/percona/pmm/api/agentpb" qanpb "github.com/percona/pmm/api/qanpb" + "github.com/percona/pmm/managed/services/agents/channel" ) // prometheusService is a subset of methods of victoriametrics.Service used by this package. @@ -65,6 +66,14 @@ type victoriaMetricsParams interface { // highAvailablityService is an interface to get information related to High Availability. type highAvailablityService interface { - //TODO: extend by send message + // TODO: extend by send message BroadcastMessage(message []byte) } + +type communicationChannel interface { + Requests() <-chan *channel.AgentRequest + Send(resp *channel.ServerResponse) + SendAndWaitResponse(payload agentpb.ServerRequestPayload) (agentpb.AgentResponsePayload, error) + Metrics() *channel.Metrics + Wait() error +} diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go index 0bf7f8ebc1..228763a98d 100644 --- a/managed/services/agents/handler.go +++ b/managed/services/agents/handler.go @@ -79,7 +79,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { // run pmm-agent state update loop for the current agent. go h.state.runStateChangeHandler(ctx, agent) - h.state.RequestStateUpdate(ctx, agent.id) + h.state.RequestStateUpdate(ctx, agent.ID()) ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -88,31 +88,31 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { case <-ticker.C: err := h.r.ping(ctx, agent) if err != nil { - l.Errorf("agent %s ping: %v", agent.id, err) + l.Errorf("agent %s ping: %v", agent.ID(), err) } // see unregister and Kick methods - case <-agent.kick: + case <-agent.KickChan(): // already unregistered, no need to call unregister method l.Warn("Kicked.") disconnectReason = "kicked" err = status.Errorf(codes.Aborted, "Kicked.") return err - case req := <-agent.channel.Requests(): + case req := <-agent.Channel().Requests(): if req == nil { disconnectReason = "done" - err = agent.channel.Wait() - h.r.unregister(agent.id, disconnectReason) + err = agent.Channel().Wait() + h.r.unregister(agent.ID(), disconnectReason) if err != nil { l.Error(errors.WithStack(err)) } - return h.updateAgentStatusForChildren(ctx, agent.id, inventorypb.AgentStatus_DONE) + return nil } switch p := req.Payload.(type) { case *agentpb.Ping: - agent.channel.Send(&channel.ServerResponse{ + agent.Channel().Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.Pong{ CurrentTime: timestamppb.Now(), @@ -125,7 +125,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { l.Errorf("%+v", err) } - agent.channel.Send(&channel.ServerResponse{ + agent.Channel().Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.StateChangedResponse{}, }) @@ -137,7 +137,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { l.Errorf("%+v", err) } - agent.channel.Send(&channel.ServerResponse{ + agent.Channel().Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.QANCollectResponse{}, }) @@ -145,7 +145,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { case *agentpb.ActionResultRequest: // TODO: PMM-3978: In the future we need to merge action parts before send it to storage. - err := models.ChangeActionResult(h.db.Querier, p.ActionId, agent.id, p.Error, string(p.Output), p.Done) + err := models.ChangeActionResult(h.db.Querier, p.ActionId, agent.ID(), p.Error, string(p.Output), p.Done) if err != nil { l.Warnf("Failed to change action: %+v", err) } @@ -154,7 +154,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { l.Warnf("Action was done with an error: %v.", p.Error) } - agent.channel.Send(&channel.ServerResponse{ + agent.Channel().Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.ActionResultResponse{}, }) diff --git a/managed/services/agents/jobs.go b/managed/services/agents/jobs.go index 82b44fa62d..a2b9a55819 100644 --- a/managed/services/agents/jobs.go +++ b/managed/services/agents/jobs.go @@ -375,7 +375,7 @@ func (s *JobsService) StartMySQLBackupJob(jobID, pmmAgentID string, timeout time return err } - resp, err := agent.channel.SendAndWaitResponse(req) + resp, err := agent.Channel().SendAndWaitResponse(req) if err != nil { return err } @@ -455,7 +455,7 @@ func (s *JobsService) StartMongoDBBackupJob( return err } - resp, err := agent.channel.SendAndWaitResponse(req) + resp, err := agent.Channel().SendAndWaitResponse(req) if err != nil { return err } @@ -503,7 +503,7 @@ func (s *JobsService) StartMySQLRestoreBackupJob( return err } - resp, err := agent.channel.SendAndWaitResponse(req) + resp, err := agent.Channel().SendAndWaitResponse(req) if err != nil { return err } @@ -590,7 +590,7 @@ func (s *JobsService) StartMongoDBRestoreBackupJob( return err } - resp, err := agent.channel.SendAndWaitResponse(req) + resp, err := agent.Channel().SendAndWaitResponse(req) if err != nil { return err } @@ -679,7 +679,7 @@ func (s *JobsService) restartSystemService(agentID string, service agentpb.Start if err != nil { return errors.Wrapf(err, "failed to get information about PMM agent: %s", agentID) } - _, err = agent.channel.SendAndWaitResponse(req) + _, err = agent.Channel().SendAndWaitResponse(req) if err != nil { return errors.Wrapf(err, "failed to restart %s on agent: %s", service, agentID) } @@ -703,7 +703,7 @@ func (s *JobsService) StopJob(jobID string) error { return errors.WithStack(err) } - _, err = agent.channel.SendAndWaitResponse(&agentpb.StopJobRequest{JobId: jobID}) + _, err = agent.Channel().SendAndWaitResponse(&agentpb.StopJobRequest{JobId: jobID}) return err } diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 84691cb739..d8999c7b4a 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -65,10 +65,26 @@ var ( ) type pmmAgentInfo struct { - channel *channel.Channel + channel communicationChannel id string stateChangeChan chan struct{} - kick chan struct{} + kickChan chan struct{} +} + +func (p *pmmAgentInfo) Channel() communicationChannel { + return p.channel +} + +func (p *pmmAgentInfo) ID() string { + return p.id +} + +func (p *pmmAgentInfo) StateChangeChan() chan struct{} { + return p.stateChangeChan +} + +func (p *pmmAgentInfo) KickChan() chan struct{} { + return p.kickChan } // Registry keeps track of all connected pmm-agents. @@ -199,10 +215,10 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, defer r.rw.Unlock() agent := &pmmAgentInfo{ - channel: channel.New(stream), + channel: channel.New(ctx, stream), id: agentMD.ID, stateChangeChan: make(chan struct{}, 1), - kick: make(chan struct{}), + kickChan: make(chan struct{}), } r.agents[agentMD.ID] = agent return agent, nil @@ -278,7 +294,7 @@ func (r *Registry) unregister(pmmAgentID, disconnectReason string) *pmmAgentInfo func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error { l := logger.Get(ctx) start := time.Now() - resp, err := agent.channel.SendAndWaitResponse(&agentpb.Ping{}) + resp, err := agent.Channel().SendAndWaitResponse(&agentpb.Ping{}) if err != nil { return err } @@ -362,7 +378,7 @@ func removeVMAgentFromPMMAgent(q *reform.Querier, pmmAgentID string) error { // Kick unregisters and forcefully disconnects pmm-agent with given ID. func (r *Registry) Kick(ctx context.Context, pmmAgentID string) { - agent := r.unregister(pmmAgentID, "kick") + agent := r.unregister(pmmAgentID, "kickChan") if agent == nil { return } @@ -371,10 +387,10 @@ func (r *Registry) Kick(ctx context.Context, pmmAgentID string) { l.Debugf("pmm-agent with ID %q will be kicked in a moment.", pmmAgentID) // see Run method - close(agent.kick) + close(agent.KickChan()) // Do not close agent.stateChangeChan to avoid breaking RequestStateUpdate; - // closing agent.kick is enough to exit runStateChangeHandler goroutine. + // closing agent.kickChan is enough to exit runStateChangeHandler goroutine. } func (r *Registry) get(pmmAgentID string) (*pmmAgentInfo, error) { @@ -401,12 +417,12 @@ func (r *Registry) Collect(ch chan<- prom.Metric) { r.rw.RLock() for _, agent := range r.agents { - m := agent.channel.Metrics() + m := agent.Channel().Metrics() - ch <- prom.MustNewConstMetric(mSentDesc, prom.CounterValue, m.Sent, agent.id) - ch <- prom.MustNewConstMetric(mRecvDesc, prom.CounterValue, m.Recv, agent.id) - ch <- prom.MustNewConstMetric(mResponsesDesc, prom.GaugeValue, m.Responses, agent.id) - ch <- prom.MustNewConstMetric(mRequestsDesc, prom.GaugeValue, m.Requests, agent.id) + ch <- prom.MustNewConstMetric(mSentDesc, prom.CounterValue, m.Sent, agent.ID()) + ch <- prom.MustNewConstMetric(mRecvDesc, prom.CounterValue, m.Recv, agent.ID()) + ch <- prom.MustNewConstMetric(mResponsesDesc, prom.GaugeValue, m.Responses, agent.ID()) + ch <- prom.MustNewConstMetric(mRequestsDesc, prom.GaugeValue, m.Requests, agent.ID()) } r.rw.RUnlock() diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index 5d86f39c6e..59cd4ad05b 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -69,7 +69,7 @@ func (u *StateUpdater) RequestStateUpdate(ctx context.Context, pmmAgentID string } select { - case agent.stateChangeChan <- struct{}{}: + case agent.StateChangeChan() <- struct{}{}: default: } } @@ -97,7 +97,7 @@ func (u *StateUpdater) UpdateAgentsState(ctx context.Context) error { // runStateChangeHandler runs pmm-agent state update loop for given pmm-agent until ctx is canceled or agent is kicked. func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgentInfo) { - l := logger.Get(ctx).WithField("agent_id", agent.id) + l := logger.Get(ctx).WithField("agent_id", agent.ID()) l.Info("Starting runStateChangeHandler ...") defer l.Info("Done runStateChangeHandler.") @@ -105,7 +105,7 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgen // stateChangeChan, state update loop, and RequestStateUpdate method ensure that state // is reloaded when requested, but several requests are batched together to avoid too often reloads. // That allows the caller to just call RequestStateUpdate when it seems fit. - if cap(agent.stateChangeChan) != 1 { + if cap(agent.StateChangeChan()) != 1 { panic("stateChangeChan should have capacity 1") } @@ -114,10 +114,10 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgen case <-ctx.Done(): return - case <-agent.kick: + case <-agent.KickChan(): return - case <-agent.stateChangeChan: + case <-agent.StateChangeChan(): // batch several update requests together by delaying the first one sleepCtx, sleepCancel := context.WithTimeout(ctx, updateBatchDelay) <-sleepCtx.Done() @@ -131,7 +131,7 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgen err := u.sendSetStateRequest(nCtx, agent) if err != nil { l.Error(err) - u.RequestStateUpdate(ctx, agent.id) + u.RequestStateUpdate(ctx, agent.ID()) } cancel() } @@ -147,7 +147,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI l.Warnf("sendSetStateRequest took %s.", dur) } }() - pmmAgent, err := models.FindAgentByID(u.db.Querier, agent.id) + pmmAgent, err := models.FindAgentByID(u.db.Querier, agent.ID()) if err != nil { return errors.Wrap(err, "failed to get PMM Agent") } @@ -156,7 +156,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI return errors.Wrapf(err, "failed to parse PMM agent version %q", *pmmAgent.Version) } - agents, err := models.FindAgents(u.db.Querier, models.AgentFilters{PMMAgentID: agent.id}) + agents, err := models.FindAgents(u.db.Querier, models.AgentFilters{PMMAgentID: agent.ID()}) if err != nil { return errors.Wrap(err, "failed to collect agents") } @@ -179,9 +179,9 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI case models.PMMAgentType: continue case models.VMAgentType: - scrapeCfg, err := u.vmdb.BuildScrapeConfigForVMAgent(agent.id) + scrapeCfg, err := u.vmdb.BuildScrapeConfigForVMAgent(agent.ID()) if err != nil { - return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.id) + return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.ID()) } agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg), u.vmParams) @@ -268,7 +268,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI } sort.Strings(rdsExporterIDs) - groupID := u.r.roster.add(agent.id, rdsGroup, rdsExporterIDs) + groupID := u.r.roster.add(agent.ID(), rdsGroup, rdsExporterIDs) c, err := rdsExporterConfig(rdsExporters, redactMode, pmmAgentVersion) if err != nil { return err @@ -280,7 +280,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI BuiltinAgents: builtinAgents, } l.Debugf("sendSetStateRequest:\n%s", proto.MarshalTextString(state)) - resp, err := agent.channel.SendAndWaitResponse(state) + resp, err := agent.Channel().SendAndWaitResponse(state) if err != nil { return err } diff --git a/managed/services/agents/versioner.go b/managed/services/agents/versioner.go index f62290f53b..e0f99c9b8f 100644 --- a/managed/services/agents/versioner.go +++ b/managed/services/agents/versioner.go @@ -157,7 +157,7 @@ func (s *VersionerService) GetVersions(pmmAgentID string, softwareList []Softwar } request := &agentpb.GetVersionsRequest{Softwares: softwareRequest} - response, err := agent.channel.SendAndWaitResponse(request) + response, err := agent.Channel().SendAndWaitResponse(request) if err != nil { return nil, errors.WithStack(err) } diff --git a/managed/services/highavailability/channel.go b/managed/services/highavailability/channel.go new file mode 100644 index 0000000000..c643593f04 --- /dev/null +++ b/managed/services/highavailability/channel.go @@ -0,0 +1,19 @@ +package highavailability + +import ( + "github.com/percona/pmm/api/agentpb" +) + +type Channel struct{} + +func NewChannel() *Channel { + return &Channel{} +} + +func (c *Channel) Send(*agentpb.ServerMessage) error { + return nil +} + +func (c *Channel) Recv() (*agentpb.AgentMessage, error) { + return nil, nil +} diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index d12984f5b2..48e26bf29c 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -21,11 +21,17 @@ import ( "fmt" "io" "net" + "sync" "time" + transport "github.com/Jille/raft-grpc-transport" "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + + "github.com/percona/pmm/api/hapb" ) type Params struct { @@ -44,28 +50,41 @@ type Service struct { services *services - messages chan []byte - nodeCh chan *memberlist.Node - leaderCh chan raft.Observation + receivedMessages chan []byte + nodeCh chan memberlist.NodeEvent + leaderCh chan raft.Observation - l *logrus.Entry + l *logrus.Entry + wg *sync.WaitGroup raftNode *raft.Raft memberlist *memberlist.Memberlist + tm *transport.Manager } func (s *Service) Apply(logEntry *raft.Log) interface{} { s.l.Printf("raft: got a message: %s", string(logEntry.Data)) + var m hapb.ClusterMessage + err := proto.Unmarshal(logEntry.Data, &m) + if err != nil { + return err + } + switch m.Payload.(type) { + case *hapb.ClusterMessage_ServerMessage: + case *hapb.ClusterMessage_AgentMessage: + + } + s.receivedMessages <- logEntry.Data return nil } func (s *Service) Snapshot() (raft.FSMSnapshot, error) { - //TODO implement me + // TODO implement me return nil, nil } func (s *Service) Restore(snapshot io.ReadCloser) error { - //TODO implement me + // TODO implement me return nil } @@ -74,15 +93,19 @@ func New(params *Params) *Service { params: params, bootstrapCluster: params.Bootstrap, services: newServices(), - nodeCh: make(chan *memberlist.Node, 5), + nodeCh: make(chan memberlist.NodeEvent, 5), leaderCh: make(chan raft.Observation), - messages: make(chan []byte), + receivedMessages: make(chan []byte), l: logrus.WithField("component", "ha"), + wg: &sync.WaitGroup{}, + tm: transport.New(raft.ServerAddress(fmt.Sprintf("%s:443", params.AdvertiseAddress)), []grpc.DialOption{grpc.WithInsecure()}), } } func (s *Service) Run(ctx context.Context) error { + s.wg.Add(1) go func() { + defer s.wg.Done() for { select { case <-s.services.ServiceAdded(): @@ -98,22 +121,6 @@ func (s *Service) Run(ctx context.Context) error { if !s.params.Enabled { return nil } - // Create the memberlist configuration - memberlistConfig := memberlist.DefaultWANConfig() - memberlistConfig.Name = s.params.NodeID - memberlistConfig.BindAddr = "0.0.0.0" - memberlistConfig.BindPort = s.params.GossipPort - memberlistConfig.AdvertiseAddr = s.params.AdvertiseAddress - memberlistConfig.AdvertisePort = s.params.GossipPort - memberlistConfig.Events = &eventDelegate{s.nodeCh} - - // Create the memberlist - var err error - s.memberlist, err = memberlist.Create(memberlistConfig) - if err != nil { - return fmt.Errorf("failed to create memberlist: %q", err) - } - defer s.memberlist.Leave(5 * time.Second) // Create the Raft configuration raftConfig := raft.DefaultConfig() @@ -135,6 +142,23 @@ func (s *Service) Run(ctx context.Context) error { if err != nil { return err } + defer s.raftNode.Shutdown().Error() + + // Create the memberlist configuration + memberlistConfig := memberlist.DefaultWANConfig() + memberlistConfig.Name = s.params.NodeID + memberlistConfig.BindAddr = "0.0.0.0" + memberlistConfig.BindPort = s.params.GossipPort + memberlistConfig.AdvertiseAddr = raa.IP.String() + memberlistConfig.AdvertisePort = s.params.GossipPort + memberlistConfig.Events = &memberlist.ChannelEventDelegate{Ch: s.nodeCh} + + // Create the memberlist + s.memberlist, err = memberlist.Create(memberlistConfig) + if err != nil { + return fmt.Errorf("failed to create memberlist: %q", err) + } + defer s.memberlist.Leave(5 * time.Second) if s.bootstrapCluster { // Start the Raft node @@ -156,47 +180,38 @@ func (s *Service) Run(ctx context.Context) error { return fmt.Errorf("failed to join memberlist cluster: %q", err) } } + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.runLeaderObserver(ctx) + }() - go s.runLeaderObserver(ctx) - - go s.runRaftNodesSynchronizer(ctx) - + s.wg.Add(1) go func() { - for { - select { - case message := <-s.messages: - //for _, member := range memberlist.Members() { - // if member.Name == s.params.NodeID { - // continue - // } - // memberlist.SendReliable(member, message) - //} - if err := s.raftNode.Apply(message, 5*time.Second).Error(); err != nil { - s.l.Errorf("Failed to apply command: %q", err) - } - s.l.Debugf("Broadcasting message: %s", message) - case <-ctx.Done(): - return - } - } + defer s.wg.Done() + s.runRaftNodesSynchronizer(ctx) }() <-ctx.Done() + s.services.Wait() + s.wg.Wait() + return nil } func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { for { select { - case node := <-s.nodeCh: + case event := <-s.nodeCh: if !s.IsLeader() { continue } - switch node.State { - case memberlist.StateAlive: + node := event.Node + switch event.Event { + case memberlist.NodeJoin: s.raftNode.AddVoter(raft.ServerID(node.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", node.Addr.String(), s.params.RaftPort)), s.raftNode.AppliedIndex(), 10*time.Second).Error() - case memberlist.StateLeft, memberlist.StateDead: + case memberlist.NodeLeave: s.raftNode.RemoveServer(raft.ServerID(node.Name), 0, 10*time.Second).Error() } case <-ctx.Done(): @@ -223,6 +238,7 @@ func (s *Service) runLeaderObserver(ctx context.Context) { } } } else { + s.l.Printf("I am not a leader!") s.services.StopRunningServices() } case <-ctx.Done(): @@ -232,11 +248,18 @@ func (s *Service) runLeaderObserver(ctx context.Context) { } func (s *Service) AddLeaderService(leaderService LeaderService) { - s.services.Add(leaderService) + err := s.services.Add(leaderService) + if err != nil { + s.l.Errorf("couldn't add HA service: +%v", err) + } } func (s *Service) BroadcastMessage(message []byte) { - s.messages <- message + if s.params.Enabled { + s.raftNode.Apply(message, 3*time.Second) + } else { + s.receivedMessages <- message + } } func (s *Service) IsLeader() bool { @@ -244,9 +267,11 @@ func (s *Service) IsLeader() bool { } func (s *Service) Bootstrap() bool { - return s.params.Bootstrap + return s.params.Bootstrap || !s.params.Enabled } -func (s *Service) Wait() { - s.services.Wait() +func (s *Service) RegisterGRPC(server *grpc.Server) { + if s.params.Enabled { + s.tm.Register(server) + } } diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 47db61bf4f..6da2f77a93 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -24,13 +24,16 @@ func NewContextService(id string, startFunc func(context.Context) error) *Contex startFunc: startFunc, } } + func (s *ContextService) ID() string { return s.id } + func (s *ContextService) Start(ctx context.Context) error { s.ctx, s.cancel = context.WithCancel(ctx) return s.startFunc(ctx) } + func (s *ContextService) Stop() { s.cancel() } @@ -47,9 +50,11 @@ func NewRunOnceService(id string, startFunc func(context.Context) error) *RunOnc startFunc: startFunc, } } + func (s *RunOnceService) ID() string { return s.id } + func (s *RunOnceService) Start(ctx context.Context) error { s.o.Do(func() { s.startFunc(ctx) diff --git a/managed/services/highavailability/raft.go b/managed/services/highavailability/raft.go index 939d0361cd..06676f073c 100644 --- a/managed/services/highavailability/raft.go +++ b/managed/services/highavailability/raft.go @@ -2,11 +2,12 @@ package highavailability import ( "context" + "io" + "sync" + "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/sirupsen/logrus" - "io" - "sync" ) // raftFSM is a simple example implementation of a Raft FSM. @@ -26,6 +27,7 @@ func newFSM(m *memberlist.Memberlist, nodeID raft.ServerID) *raftFSM { l: logrus.WithField("component", "raftFSM"), } } + func (f *raftFSM) Run(ctx context.Context, leaderCh chan raft.Observation) { for { select { @@ -58,7 +60,6 @@ func (f *raftFSM) Apply(logEntry *raft.Log) interface{} { } return nil - } func (f *raftFSM) Snapshot() (raft.FSMSnapshot, error) { diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index 112e26108d..25f1ba7df7 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -127,6 +127,7 @@ stdout_logfile_maxbytes = 30MB stdout_logfile_backups = 2 redirect_stderr = true {{- end }} +{{- if not .DisableInternalClickhouse }} [program:clickhouse] priority = 2 @@ -143,6 +144,7 @@ stdout_logfile = /srv/logs/clickhouse-server.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true +{{- end }} [program:nginx] priority = 4 From 74e46708fcea6a4b75d8cb9c787cec6eb097218e Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 8 Jun 2023 01:15:58 +0300 Subject: [PATCH 31/77] PMM-12182 use interface. --- managed/services/agents/registry.go | 31 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index d8999c7b4a..1a2ae1dd5e 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -64,26 +64,33 @@ var ( nil) ) -type pmmAgentInfo struct { +type pmmAgentInfo interface { + Channel() communicationChannel + ID() string + StateChangeChan() chan struct{} + KickChan() chan struct{} +} + +type localPMMAgentInfo struct { channel communicationChannel id string stateChangeChan chan struct{} kickChan chan struct{} } -func (p *pmmAgentInfo) Channel() communicationChannel { +func (p *localPMMAgentInfo) Channel() communicationChannel { return p.channel } -func (p *pmmAgentInfo) ID() string { +func (p *localPMMAgentInfo) ID() string { return p.id } -func (p *pmmAgentInfo) StateChangeChan() chan struct{} { +func (p *localPMMAgentInfo) StateChangeChan() chan struct{} { return p.stateChangeChan } -func (p *pmmAgentInfo) KickChan() chan struct{} { +func (p *localPMMAgentInfo) KickChan() chan struct{} { return p.kickChan } @@ -92,7 +99,7 @@ type Registry struct { db *reform.DB rw sync.RWMutex - agents map[string]*pmmAgentInfo // id -> info + agents map[string]pmmAgentInfo // id -> info roster *roster @@ -107,7 +114,7 @@ type Registry struct { // NewRegistry creates a new registry with given database connection. func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry { - agents := make(map[string]*pmmAgentInfo) + agents := make(map[string]pmmAgentInfo) r := &Registry{ db: db, @@ -169,7 +176,7 @@ func (r *Registry) IsConnected(pmmAgentID string) bool { return err == nil } -func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, error) { +func (r *Registry) register(stream agentpb.Agent_ConnectServer) (pmmAgentInfo, error) { ctx := stream.Context() l := logger.Get(ctx) r.mConnects.Inc() @@ -214,7 +221,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, r.rw.Lock() defer r.rw.Unlock() - agent := &pmmAgentInfo{ + agent := &localPMMAgentInfo{ channel: channel.New(ctx, stream), id: agentMD.ID, stateChangeChan: make(chan struct{}, 1), @@ -271,7 +278,7 @@ func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Quer } // unregister removes pmm-agent with given ID from the registry. -func (r *Registry) unregister(pmmAgentID, disconnectReason string) *pmmAgentInfo { +func (r *Registry) unregister(pmmAgentID, disconnectReason string) pmmAgentInfo { r.mDisconnects.WithLabelValues(disconnectReason).Inc() r.rw.Lock() @@ -291,7 +298,7 @@ func (r *Registry) unregister(pmmAgentID, disconnectReason string) *pmmAgentInfo } // ping sends Ping message to given Agent, waits for Pong and observes round-trip time and clock drift. -func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error { +func (r *Registry) ping(ctx context.Context, agent pmmAgentInfo) error { l := logger.Get(ctx) start := time.Now() resp, err := agent.Channel().SendAndWaitResponse(&agentpb.Ping{}) @@ -393,7 +400,7 @@ func (r *Registry) Kick(ctx context.Context, pmmAgentID string) { // closing agent.kickChan is enough to exit runStateChangeHandler goroutine. } -func (r *Registry) get(pmmAgentID string) (*pmmAgentInfo, error) { +func (r *Registry) get(pmmAgentID string) (pmmAgentInfo, error) { r.rw.RLock() pmmAgent := r.agents[pmmAgentID] r.rw.RUnlock() From 67ed38294eaea5dc5120e260666197022eb17b93 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 8 Jun 2023 01:24:12 +0300 Subject: [PATCH 32/77] PMM-12078 fix go.sum. --- go.sum | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/go.sum b/go.sum index 99118c7a62..1fdb937af3 100644 --- a/go.sum +++ b/go.sum @@ -468,7 +468,6 @@ github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4 github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.5.0 h1:uNs9EfJ4FwiArZRxxfd/dQ5d33nV31/CdCHArH89hT8= github.com/hashicorp/raft v1.5.0/go.mod h1:pKHB2mf/Y25u3AHNSXVRv+yT+WAnmeTX0BwVppVQV+M= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -552,10 +551,7 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -565,10 +561,7 @@ github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqf github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -713,6 +706,7 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/alertmanager v0.25.0 h1:vbXKUR6PYRiZPRIKfmXaG+dmCKG52RtPL4Btl8hQGvg= github.com/prometheus/alertmanager v0.25.0/go.mod h1:MEZ3rFVHqKZsw7IcNS/m4AWZeXThmJhumpiWR4eHU/w= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -725,6 +719,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -738,6 +733,7 @@ github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57J github.com/prometheus/exporter-toolkit v0.8.2 h1:sbJAfBXQFkG6sUkbwBun8MNdzW9+wd5YfPYofbmj0YM= github.com/prometheus/exporter-toolkit v0.8.2/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -866,9 +862,7 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= @@ -900,6 +894,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -916,6 +911,7 @@ golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -949,8 +945,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -985,6 +979,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -998,10 +993,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1034,8 +1026,11 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1122,6 +1117,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= @@ -1191,6 +1187,7 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= From 3396e8a72cfc383a589bd1774ab03d3689276e39 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 8 Jun 2023 10:58:31 +0300 Subject: [PATCH 33/77] PMM-12078 fix build. --- managed/services/agents/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index 59cd4ad05b..70ecb8ccd0 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -96,7 +96,7 @@ func (u *StateUpdater) UpdateAgentsState(ctx context.Context) error { } // runStateChangeHandler runs pmm-agent state update loop for given pmm-agent until ctx is canceled or agent is kicked. -func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgentInfo) { +func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgentInfo) { l := logger.Get(ctx).WithField("agent_id", agent.ID()) l.Info("Starting runStateChangeHandler ...") @@ -139,7 +139,7 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgen } // sendSetStateRequest sends SetStateRequest to given pmm-agent. -func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentInfo) error { //nolint:cyclop +func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentInfo) error { //nolint:cyclop l := logger.Get(ctx) start := time.Now() defer func() { From 749ddfee50649c48c865ce9c48744cedba613c8d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 6 Jul 2023 15:54:10 +0300 Subject: [PATCH 34/77] PMM-12078 Improve the code --- api/hapb/ha.pb.go | 2 +- .../server/leader_health_check_parameters.go | 146 +++ .../server/leader_health_check_responses.go | 368 ++++++ .../json/client/server/server_client.go | 41 + api/serverpb/json/serverpb.json | 62 + api/serverpb/server.pb.go | 1159 +++++++++-------- api/serverpb/server.pb.gw.go | 81 ++ api/serverpb/server.pb.validate.go | 204 +++ api/serverpb/server.proto | 17 + api/serverpb/server_grpc.pb.go | 40 + api/swagger/swagger-dev.json | 62 + api/swagger/swagger.json | 62 + docker-compose.yml | 33 +- managed/cmd/pmm-managed/main.go | 8 +- managed/services/agents/registry.go | 7 + .../highavailability/highavailability.go | 21 +- .../highavailability/leaderservice.go | 40 +- .../services/highavailability/memberlist.go | 22 - managed/services/server/deps.go | 8 +- managed/services/server/server.go | 12 + managed/services/supervisord/supervisord.go | 2 +- managed/testdata/haproxy/haproxy.cfg | 33 + 22 files changed, 1867 insertions(+), 563 deletions(-) create mode 100644 api/serverpb/json/client/server/leader_health_check_parameters.go create mode 100644 api/serverpb/json/client/server/leader_health_check_responses.go delete mode 100644 managed/services/highavailability/memberlist.go create mode 100644 managed/testdata/haproxy/haproxy.cfg diff --git a/api/hapb/ha.pb.go b/api/hapb/ha.pb.go index 99466f5ecf..4750b4792e 100644 --- a/api/hapb/ha.pb.go +++ b/api/hapb/ha.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.31.0 // protoc (unknown) // source: hapb/ha.proto diff --git a/api/serverpb/json/client/server/leader_health_check_parameters.go b/api/serverpb/json/client/server/leader_health_check_parameters.go new file mode 100644 index 0000000000..595d3d650d --- /dev/null +++ b/api/serverpb/json/client/server/leader_health_check_parameters.go @@ -0,0 +1,146 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package server + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// NewLeaderHealthCheckParams creates a new LeaderHealthCheckParams object, +// with the default timeout for this client. +// +// Default values are not hydrated, since defaults are normally applied by the API server side. +// +// To enforce default values in parameter, use SetDefaults or WithDefaults. +func NewLeaderHealthCheckParams() *LeaderHealthCheckParams { + return &LeaderHealthCheckParams{ + timeout: cr.DefaultTimeout, + } +} + +// NewLeaderHealthCheckParamsWithTimeout creates a new LeaderHealthCheckParams object +// with the ability to set a timeout on a request. +func NewLeaderHealthCheckParamsWithTimeout(timeout time.Duration) *LeaderHealthCheckParams { + return &LeaderHealthCheckParams{ + timeout: timeout, + } +} + +// NewLeaderHealthCheckParamsWithContext creates a new LeaderHealthCheckParams object +// with the ability to set a context for a request. +func NewLeaderHealthCheckParamsWithContext(ctx context.Context) *LeaderHealthCheckParams { + return &LeaderHealthCheckParams{ + Context: ctx, + } +} + +// NewLeaderHealthCheckParamsWithHTTPClient creates a new LeaderHealthCheckParams object +// with the ability to set a custom HTTPClient for a request. +func NewLeaderHealthCheckParamsWithHTTPClient(client *http.Client) *LeaderHealthCheckParams { + return &LeaderHealthCheckParams{ + HTTPClient: client, + } +} + +/* +LeaderHealthCheckParams contains all the parameters to send to the API endpoint + + for the leader health check operation. + + Typically these are written to a http.Request. +*/ +type LeaderHealthCheckParams struct { + // Body. + Body interface{} + + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithDefaults hydrates default values in the leader health check params (not the query body). +// +// All values with no default are reset to their zero value. +func (o *LeaderHealthCheckParams) WithDefaults() *LeaderHealthCheckParams { + o.SetDefaults() + return o +} + +// SetDefaults hydrates default values in the leader health check params (not the query body). +// +// All values with no default are reset to their zero value. +func (o *LeaderHealthCheckParams) SetDefaults() { + // no default values defined for this parameter +} + +// WithTimeout adds the timeout to the leader health check params +func (o *LeaderHealthCheckParams) WithTimeout(timeout time.Duration) *LeaderHealthCheckParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the leader health check params +func (o *LeaderHealthCheckParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the leader health check params +func (o *LeaderHealthCheckParams) WithContext(ctx context.Context) *LeaderHealthCheckParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the leader health check params +func (o *LeaderHealthCheckParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the leader health check params +func (o *LeaderHealthCheckParams) WithHTTPClient(client *http.Client) *LeaderHealthCheckParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the leader health check params +func (o *LeaderHealthCheckParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WithBody adds the body to the leader health check params +func (o *LeaderHealthCheckParams) WithBody(body interface{}) *LeaderHealthCheckParams { + o.SetBody(body) + return o +} + +// SetBody adds the body to the leader health check params +func (o *LeaderHealthCheckParams) SetBody(body interface{}) { + o.Body = body +} + +// WriteToRequest writes these params to a swagger request +func (o *LeaderHealthCheckParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + if o.Body != nil { + if err := r.SetBodyParam(o.Body); err != nil { + return err + } + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/api/serverpb/json/client/server/leader_health_check_responses.go b/api/serverpb/json/client/server/leader_health_check_responses.go new file mode 100644 index 0000000000..bdbe4dbab2 --- /dev/null +++ b/api/serverpb/json/client/server/leader_health_check_responses.go @@ -0,0 +1,368 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package server + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "fmt" + "io" + "strconv" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// LeaderHealthCheckReader is a Reader for the LeaderHealthCheck structure. +type LeaderHealthCheckReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *LeaderHealthCheckReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + case 200: + result := NewLeaderHealthCheckOK() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + default: + result := NewLeaderHealthCheckDefault(response.Code()) + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + if response.Code()/100 == 2 { + return result, nil + } + return nil, result + } +} + +// NewLeaderHealthCheckOK creates a LeaderHealthCheckOK with default headers values +func NewLeaderHealthCheckOK() *LeaderHealthCheckOK { + return &LeaderHealthCheckOK{} +} + +/* +LeaderHealthCheckOK describes a response with status code 200, with default header values. + +A successful response. +*/ +type LeaderHealthCheckOK struct { + Payload interface{} +} + +func (o *LeaderHealthCheckOK) Error() string { + return fmt.Sprintf("[POST /v1/leaderHealthCheck][%d] leaderHealthCheckOk %+v", 200, o.Payload) +} + +func (o *LeaderHealthCheckOK) GetPayload() interface{} { + return o.Payload +} + +func (o *LeaderHealthCheckOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} + +// NewLeaderHealthCheckDefault creates a LeaderHealthCheckDefault with default headers values +func NewLeaderHealthCheckDefault(code int) *LeaderHealthCheckDefault { + return &LeaderHealthCheckDefault{ + _statusCode: code, + } +} + +/* +LeaderHealthCheckDefault describes a response with status code -1, with default header values. + +An unexpected error response. +*/ +type LeaderHealthCheckDefault struct { + _statusCode int + + Payload *LeaderHealthCheckDefaultBody +} + +// Code gets the status code for the leader health check default response +func (o *LeaderHealthCheckDefault) Code() int { + return o._statusCode +} + +func (o *LeaderHealthCheckDefault) Error() string { + return fmt.Sprintf("[POST /v1/leaderHealthCheck][%d] LeaderHealthCheck default %+v", o._statusCode, o.Payload) +} + +func (o *LeaderHealthCheckDefault) GetPayload() *LeaderHealthCheckDefaultBody { + return o.Payload +} + +func (o *LeaderHealthCheckDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + o.Payload = new(LeaderHealthCheckDefaultBody) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} + +/* +LeaderHealthCheckDefaultBody leader health check default body +swagger:model LeaderHealthCheckDefaultBody +*/ +type LeaderHealthCheckDefaultBody struct { + // code + Code int32 `json:"code,omitempty"` + + // message + Message string `json:"message,omitempty"` + + // details + Details []*LeaderHealthCheckDefaultBodyDetailsItems0 `json:"details"` +} + +// Validate validates this leader health check default body +func (o *LeaderHealthCheckDefaultBody) Validate(formats strfmt.Registry) error { + var res []error + + if err := o.validateDetails(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (o *LeaderHealthCheckDefaultBody) validateDetails(formats strfmt.Registry) error { + if swag.IsZero(o.Details) { // not required + return nil + } + + for i := 0; i < len(o.Details); i++ { + if swag.IsZero(o.Details[i]) { // not required + continue + } + + if o.Details[i] != nil { + if err := o.Details[i].Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("LeaderHealthCheck default" + "." + "details" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("LeaderHealthCheck default" + "." + "details" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + +// ContextValidate validate this leader health check default body based on the context it is used +func (o *LeaderHealthCheckDefaultBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := o.contextValidateDetails(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (o *LeaderHealthCheckDefaultBody) contextValidateDetails(ctx context.Context, formats strfmt.Registry) error { + for i := 0; i < len(o.Details); i++ { + if o.Details[i] != nil { + if err := o.Details[i].ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("LeaderHealthCheck default" + "." + "details" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("LeaderHealthCheck default" + "." + "details" + "." + strconv.Itoa(i)) + } + return err + } + } + } + + return nil +} + +// MarshalBinary interface implementation +func (o *LeaderHealthCheckDefaultBody) MarshalBinary() ([]byte, error) { + if o == nil { + return nil, nil + } + return swag.WriteJSON(o) +} + +// UnmarshalBinary interface implementation +func (o *LeaderHealthCheckDefaultBody) UnmarshalBinary(b []byte) error { + var res LeaderHealthCheckDefaultBody + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *o = res + return nil +} + +/* +LeaderHealthCheckDefaultBodyDetailsItems0 `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// // or ... +// if (any.isSameTypeAs(Foo.getDefaultInstance())) { +// foo = any.unpack(Foo.getDefaultInstance()); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// JSON +// +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +swagger:model LeaderHealthCheckDefaultBodyDetailsItems0 +*/ +type LeaderHealthCheckDefaultBodyDetailsItems0 struct { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + AtType string `json:"@type,omitempty"` +} + +// Validate validates this leader health check default body details items0 +func (o *LeaderHealthCheckDefaultBodyDetailsItems0) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this leader health check default body details items0 based on context it is used +func (o *LeaderHealthCheckDefaultBodyDetailsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (o *LeaderHealthCheckDefaultBodyDetailsItems0) MarshalBinary() ([]byte, error) { + if o == nil { + return nil, nil + } + return swag.WriteJSON(o) +} + +// UnmarshalBinary interface implementation +func (o *LeaderHealthCheckDefaultBodyDetailsItems0) UnmarshalBinary(b []byte) error { + var res LeaderHealthCheckDefaultBodyDetailsItems0 + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *o = res + return nil +} diff --git a/api/serverpb/json/client/server/server_client.go b/api/serverpb/json/client/server/server_client.go index fffdea9044..02e7fab28f 100644 --- a/api/serverpb/json/client/server/server_client.go +++ b/api/serverpb/json/client/server/server_client.go @@ -38,6 +38,8 @@ type ClientService interface { GetSettings(params *GetSettingsParams, opts ...ClientOption) (*GetSettingsOK, error) + LeaderHealthCheck(params *LeaderHealthCheckParams, opts ...ClientOption) (*LeaderHealthCheckOK, error) + Logs(params *LogsParams, writer io.Writer, opts ...ClientOption) (*LogsOK, error) Readiness(params *ReadinessParams, opts ...ClientOption) (*ReadinessOK, error) @@ -209,6 +211,45 @@ func (a *Client) GetSettings(params *GetSettingsParams, opts ...ClientOption) (* return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) } +/* +LeaderHealthCheck checks leadership + +Checks if the instance is the leader in a cluster. Returns an error if the instance isn't the leader. +*/ +func (a *Client) LeaderHealthCheck(params *LeaderHealthCheckParams, opts ...ClientOption) (*LeaderHealthCheckOK, error) { + // TODO: Validate the params before sending + if params == nil { + params = NewLeaderHealthCheckParams() + } + op := &runtime.ClientOperation{ + ID: "LeaderHealthCheck", + Method: "POST", + PathPattern: "/v1/leaderHealthCheck", + ProducesMediaTypes: []string{"application/json"}, + ConsumesMediaTypes: []string{"application/json"}, + Schemes: []string{"http", "https"}, + Params: params, + Reader: &LeaderHealthCheckReader{formats: a.formats}, + Context: params.Context, + Client: params.HTTPClient, + } + for _, opt := range opts { + opt(op) + } + + result, err := a.transport.Submit(op) + if err != nil { + return nil, err + } + success, ok := result.(*LeaderHealthCheckOK) + if ok { + return success, nil + } + // unexpected success response + unexpectedSuccess := result.(*LeaderHealthCheckDefault) + return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) +} + /* Logs logs diff --git a/api/serverpb/json/serverpb.json b/api/serverpb/json/serverpb.json index 4501cde473..2df13cdae7 100644 --- a/api/serverpb/json/serverpb.json +++ b/api/serverpb/json/serverpb.json @@ -1322,6 +1322,68 @@ } } }, + "/v1/leaderHealthCheck": { + "post": { + "description": "Checks if the instance is the leader in a cluster. Returns an error if the instance isn't the leader.", + "tags": [ + "Server" + ], + "summary": "Check Leadership", + "operationId": "LeaderHealthCheck", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "object" + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "description": "This probe is available without authentication, so it should not contain any data.", + "type": "object" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32", + "x-order": 0 + }, + "details": { + "type": "array", + "items": { + "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\nExample 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\nExample 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", + "type": "object", + "properties": { + "@type": { + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", + "type": "string", + "x-order": 0 + } + }, + "additionalProperties": false + }, + "x-order": 2 + }, + "message": { + "type": "string", + "x-order": 1 + } + } + } + } + } + } + }, "/v1/readyz": { "get": { "description": "Returns an error when Server components being restarted are not ready yet. Use this API for checking the health of Docker containers and for probing Kubernetes readiness.", diff --git a/api/serverpb/server.pb.go b/api/serverpb/server.pb.go index 3a3d7a99ec..edb7d7050a 100644 --- a/api/serverpb/server.pb.go +++ b/api/serverpb/server.pb.go @@ -351,6 +351,82 @@ func (*ReadinessResponse) Descriptor() ([]byte, []int) { return file_serverpb_server_proto_rawDescGZIP(), []int{4} } +type LeaderHealthCheckRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *LeaderHealthCheckRequest) Reset() { + *x = LeaderHealthCheckRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_serverpb_server_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LeaderHealthCheckRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LeaderHealthCheckRequest) ProtoMessage() {} + +func (x *LeaderHealthCheckRequest) ProtoReflect() protoreflect.Message { + mi := &file_serverpb_server_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LeaderHealthCheckRequest.ProtoReflect.Descriptor instead. +func (*LeaderHealthCheckRequest) Descriptor() ([]byte, []int) { + return file_serverpb_server_proto_rawDescGZIP(), []int{5} +} + +type LeaderHealthCheckResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *LeaderHealthCheckResponse) Reset() { + *x = LeaderHealthCheckResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_serverpb_server_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LeaderHealthCheckResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LeaderHealthCheckResponse) ProtoMessage() {} + +func (x *LeaderHealthCheckResponse) ProtoReflect() protoreflect.Message { + mi := &file_serverpb_server_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LeaderHealthCheckResponse.ProtoReflect.Descriptor instead. +func (*LeaderHealthCheckResponse) Descriptor() ([]byte, []int) { + return file_serverpb_server_proto_rawDescGZIP(), []int{6} +} + type CheckUpdatesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -365,7 +441,7 @@ type CheckUpdatesRequest struct { func (x *CheckUpdatesRequest) Reset() { *x = CheckUpdatesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[5] + mi := &file_serverpb_server_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -378,7 +454,7 @@ func (x *CheckUpdatesRequest) String() string { func (*CheckUpdatesRequest) ProtoMessage() {} func (x *CheckUpdatesRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[5] + mi := &file_serverpb_server_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -391,7 +467,7 @@ func (x *CheckUpdatesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckUpdatesRequest.ProtoReflect.Descriptor instead. func (*CheckUpdatesRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{5} + return file_serverpb_server_proto_rawDescGZIP(), []int{7} } func (x *CheckUpdatesRequest) GetForce() bool { @@ -428,7 +504,7 @@ type CheckUpdatesResponse struct { func (x *CheckUpdatesResponse) Reset() { *x = CheckUpdatesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[6] + mi := &file_serverpb_server_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -441,7 +517,7 @@ func (x *CheckUpdatesResponse) String() string { func (*CheckUpdatesResponse) ProtoMessage() {} func (x *CheckUpdatesResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[6] + mi := &file_serverpb_server_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -454,7 +530,7 @@ func (x *CheckUpdatesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckUpdatesResponse.ProtoReflect.Descriptor instead. func (*CheckUpdatesResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{6} + return file_serverpb_server_proto_rawDescGZIP(), []int{8} } func (x *CheckUpdatesResponse) GetInstalled() *VersionInfo { @@ -501,7 +577,7 @@ type StartUpdateRequest struct { func (x *StartUpdateRequest) Reset() { *x = StartUpdateRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[7] + mi := &file_serverpb_server_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -514,7 +590,7 @@ func (x *StartUpdateRequest) String() string { func (*StartUpdateRequest) ProtoMessage() {} func (x *StartUpdateRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[7] + mi := &file_serverpb_server_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -527,7 +603,7 @@ func (x *StartUpdateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StartUpdateRequest.ProtoReflect.Descriptor instead. func (*StartUpdateRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{7} + return file_serverpb_server_proto_rawDescGZIP(), []int{9} } type StartUpdateResponse struct { @@ -544,7 +620,7 @@ type StartUpdateResponse struct { func (x *StartUpdateResponse) Reset() { *x = StartUpdateResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[8] + mi := &file_serverpb_server_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -557,7 +633,7 @@ func (x *StartUpdateResponse) String() string { func (*StartUpdateResponse) ProtoMessage() {} func (x *StartUpdateResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[8] + mi := &file_serverpb_server_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -570,7 +646,7 @@ func (x *StartUpdateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StartUpdateResponse.ProtoReflect.Descriptor instead. func (*StartUpdateResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{8} + return file_serverpb_server_proto_rawDescGZIP(), []int{10} } func (x *StartUpdateResponse) GetAuthToken() string { @@ -601,7 +677,7 @@ type UpdateStatusRequest struct { func (x *UpdateStatusRequest) Reset() { *x = UpdateStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[9] + mi := &file_serverpb_server_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -614,7 +690,7 @@ func (x *UpdateStatusRequest) String() string { func (*UpdateStatusRequest) ProtoMessage() {} func (x *UpdateStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[9] + mi := &file_serverpb_server_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -627,7 +703,7 @@ func (x *UpdateStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateStatusRequest.ProtoReflect.Descriptor instead. func (*UpdateStatusRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{9} + return file_serverpb_server_proto_rawDescGZIP(), []int{11} } func (x *UpdateStatusRequest) GetAuthToken() string { @@ -660,7 +736,7 @@ type UpdateStatusResponse struct { func (x *UpdateStatusResponse) Reset() { *x = UpdateStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[10] + mi := &file_serverpb_server_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -673,7 +749,7 @@ func (x *UpdateStatusResponse) String() string { func (*UpdateStatusResponse) ProtoMessage() {} func (x *UpdateStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[10] + mi := &file_serverpb_server_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -686,7 +762,7 @@ func (x *UpdateStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateStatusResponse.ProtoReflect.Descriptor instead. func (*UpdateStatusResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{10} + return file_serverpb_server_proto_rawDescGZIP(), []int{12} } func (x *UpdateStatusResponse) GetLogLines() []string { @@ -727,7 +803,7 @@ type MetricsResolutions struct { func (x *MetricsResolutions) Reset() { *x = MetricsResolutions{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[11] + mi := &file_serverpb_server_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -740,7 +816,7 @@ func (x *MetricsResolutions) String() string { func (*MetricsResolutions) ProtoMessage() {} func (x *MetricsResolutions) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[11] + mi := &file_serverpb_server_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -753,7 +829,7 @@ func (x *MetricsResolutions) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricsResolutions.ProtoReflect.Descriptor instead. func (*MetricsResolutions) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{11} + return file_serverpb_server_proto_rawDescGZIP(), []int{13} } func (x *MetricsResolutions) GetHr() *durationpb.Duration { @@ -804,7 +880,7 @@ type EmailAlertingSettings struct { func (x *EmailAlertingSettings) Reset() { *x = EmailAlertingSettings{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[12] + mi := &file_serverpb_server_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -817,7 +893,7 @@ func (x *EmailAlertingSettings) String() string { func (*EmailAlertingSettings) ProtoMessage() {} func (x *EmailAlertingSettings) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[12] + mi := &file_serverpb_server_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -830,7 +906,7 @@ func (x *EmailAlertingSettings) ProtoReflect() protoreflect.Message { // Deprecated: Use EmailAlertingSettings.ProtoReflect.Descriptor instead. func (*EmailAlertingSettings) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{12} + return file_serverpb_server_proto_rawDescGZIP(), []int{14} } func (x *EmailAlertingSettings) GetFrom() string { @@ -902,7 +978,7 @@ type SlackAlertingSettings struct { func (x *SlackAlertingSettings) Reset() { *x = SlackAlertingSettings{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[13] + mi := &file_serverpb_server_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -915,7 +991,7 @@ func (x *SlackAlertingSettings) String() string { func (*SlackAlertingSettings) ProtoMessage() {} func (x *SlackAlertingSettings) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[13] + mi := &file_serverpb_server_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -928,7 +1004,7 @@ func (x *SlackAlertingSettings) ProtoReflect() protoreflect.Message { // Deprecated: Use SlackAlertingSettings.ProtoReflect.Descriptor instead. func (*SlackAlertingSettings) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{13} + return file_serverpb_server_proto_rawDescGZIP(), []int{15} } func (x *SlackAlertingSettings) GetUrl() string { @@ -955,7 +1031,7 @@ type STTCheckIntervals struct { func (x *STTCheckIntervals) Reset() { *x = STTCheckIntervals{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[14] + mi := &file_serverpb_server_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -968,7 +1044,7 @@ func (x *STTCheckIntervals) String() string { func (*STTCheckIntervals) ProtoMessage() {} func (x *STTCheckIntervals) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[14] + mi := &file_serverpb_server_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -981,7 +1057,7 @@ func (x *STTCheckIntervals) ProtoReflect() protoreflect.Message { // Deprecated: Use STTCheckIntervals.ProtoReflect.Descriptor instead. func (*STTCheckIntervals) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{14} + return file_serverpb_server_proto_rawDescGZIP(), []int{16} } func (x *STTCheckIntervals) GetStandardInterval() *durationpb.Duration { @@ -1056,7 +1132,7 @@ type Settings struct { func (x *Settings) Reset() { *x = Settings{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[15] + mi := &file_serverpb_server_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1069,7 +1145,7 @@ func (x *Settings) String() string { func (*Settings) ProtoMessage() {} func (x *Settings) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[15] + mi := &file_serverpb_server_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1082,7 +1158,7 @@ func (x *Settings) ProtoReflect() protoreflect.Message { // Deprecated: Use Settings.ProtoReflect.Descriptor instead. func (*Settings) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{15} + return file_serverpb_server_proto_rawDescGZIP(), []int{17} } func (x *Settings) GetUpdatesDisabled() bool { @@ -1248,7 +1324,7 @@ type GetSettingsRequest struct { func (x *GetSettingsRequest) Reset() { *x = GetSettingsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[16] + mi := &file_serverpb_server_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1261,7 +1337,7 @@ func (x *GetSettingsRequest) String() string { func (*GetSettingsRequest) ProtoMessage() {} func (x *GetSettingsRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[16] + mi := &file_serverpb_server_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1274,7 +1350,7 @@ func (x *GetSettingsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSettingsRequest.ProtoReflect.Descriptor instead. func (*GetSettingsRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{16} + return file_serverpb_server_proto_rawDescGZIP(), []int{18} } type GetSettingsResponse struct { @@ -1288,7 +1364,7 @@ type GetSettingsResponse struct { func (x *GetSettingsResponse) Reset() { *x = GetSettingsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[17] + mi := &file_serverpb_server_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1301,7 +1377,7 @@ func (x *GetSettingsResponse) String() string { func (*GetSettingsResponse) ProtoMessage() {} func (x *GetSettingsResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[17] + mi := &file_serverpb_server_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1314,7 +1390,7 @@ func (x *GetSettingsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSettingsResponse.ProtoReflect.Descriptor instead. func (*GetSettingsResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{17} + return file_serverpb_server_proto_rawDescGZIP(), []int{19} } func (x *GetSettingsResponse) GetSettings() *Settings { @@ -1388,7 +1464,7 @@ type ChangeSettingsRequest struct { func (x *ChangeSettingsRequest) Reset() { *x = ChangeSettingsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[18] + mi := &file_serverpb_server_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1401,7 +1477,7 @@ func (x *ChangeSettingsRequest) String() string { func (*ChangeSettingsRequest) ProtoMessage() {} func (x *ChangeSettingsRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[18] + mi := &file_serverpb_server_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1414,7 +1490,7 @@ func (x *ChangeSettingsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ChangeSettingsRequest.ProtoReflect.Descriptor instead. func (*ChangeSettingsRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{18} + return file_serverpb_server_proto_rawDescGZIP(), []int{20} } func (x *ChangeSettingsRequest) GetEnableUpdates() bool { @@ -1645,7 +1721,7 @@ type ChangeSettingsResponse struct { func (x *ChangeSettingsResponse) Reset() { *x = ChangeSettingsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[19] + mi := &file_serverpb_server_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1658,7 +1734,7 @@ func (x *ChangeSettingsResponse) String() string { func (*ChangeSettingsResponse) ProtoMessage() {} func (x *ChangeSettingsResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[19] + mi := &file_serverpb_server_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1671,7 +1747,7 @@ func (x *ChangeSettingsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ChangeSettingsResponse.ProtoReflect.Descriptor instead. func (*ChangeSettingsResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{19} + return file_serverpb_server_proto_rawDescGZIP(), []int{21} } func (x *ChangeSettingsResponse) GetSettings() *Settings { @@ -1695,7 +1771,7 @@ type TestEmailAlertingSettingsRequest struct { func (x *TestEmailAlertingSettingsRequest) Reset() { *x = TestEmailAlertingSettingsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[20] + mi := &file_serverpb_server_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1708,7 +1784,7 @@ func (x *TestEmailAlertingSettingsRequest) String() string { func (*TestEmailAlertingSettingsRequest) ProtoMessage() {} func (x *TestEmailAlertingSettingsRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[20] + mi := &file_serverpb_server_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1721,7 +1797,7 @@ func (x *TestEmailAlertingSettingsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TestEmailAlertingSettingsRequest.ProtoReflect.Descriptor instead. func (*TestEmailAlertingSettingsRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{20} + return file_serverpb_server_proto_rawDescGZIP(), []int{22} } func (x *TestEmailAlertingSettingsRequest) GetEmailAlertingSettings() *EmailAlertingSettings { @@ -1747,7 +1823,7 @@ type TestEmailAlertingSettingsResponse struct { func (x *TestEmailAlertingSettingsResponse) Reset() { *x = TestEmailAlertingSettingsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[21] + mi := &file_serverpb_server_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1760,7 +1836,7 @@ func (x *TestEmailAlertingSettingsResponse) String() string { func (*TestEmailAlertingSettingsResponse) ProtoMessage() {} func (x *TestEmailAlertingSettingsResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[21] + mi := &file_serverpb_server_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1773,7 +1849,7 @@ func (x *TestEmailAlertingSettingsResponse) ProtoReflect() protoreflect.Message // Deprecated: Use TestEmailAlertingSettingsResponse.ProtoReflect.Descriptor instead. func (*TestEmailAlertingSettingsResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{21} + return file_serverpb_server_proto_rawDescGZIP(), []int{23} } type AWSInstanceCheckRequest struct { @@ -1788,7 +1864,7 @@ type AWSInstanceCheckRequest struct { func (x *AWSInstanceCheckRequest) Reset() { *x = AWSInstanceCheckRequest{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[22] + mi := &file_serverpb_server_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1801,7 +1877,7 @@ func (x *AWSInstanceCheckRequest) String() string { func (*AWSInstanceCheckRequest) ProtoMessage() {} func (x *AWSInstanceCheckRequest) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[22] + mi := &file_serverpb_server_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1814,7 +1890,7 @@ func (x *AWSInstanceCheckRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AWSInstanceCheckRequest.ProtoReflect.Descriptor instead. func (*AWSInstanceCheckRequest) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{22} + return file_serverpb_server_proto_rawDescGZIP(), []int{24} } func (x *AWSInstanceCheckRequest) GetInstanceId() string { @@ -1833,7 +1909,7 @@ type AWSInstanceCheckResponse struct { func (x *AWSInstanceCheckResponse) Reset() { *x = AWSInstanceCheckResponse{} if protoimpl.UnsafeEnabled { - mi := &file_serverpb_server_proto_msgTypes[23] + mi := &file_serverpb_server_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1846,7 +1922,7 @@ func (x *AWSInstanceCheckResponse) String() string { func (*AWSInstanceCheckResponse) ProtoMessage() {} func (x *AWSInstanceCheckResponse) ProtoReflect() protoreflect.Message { - mi := &file_serverpb_server_proto_msgTypes[23] + mi := &file_serverpb_server_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1859,7 +1935,7 @@ func (x *AWSInstanceCheckResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AWSInstanceCheckResponse.ProtoReflect.Descriptor instead. func (*AWSInstanceCheckResponse) Descriptor() ([]byte, []int) { - return file_serverpb_server_proto_rawDescGZIP(), []int{23} + return file_serverpb_server_proto_rawDescGZIP(), []int{25} } var File_serverpb_server_proto protoreflect.FileDescriptor @@ -1904,181 +1980,101 @@ var file_serverpb_server_proto_rawDesc = []byte{ 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x22, 0x12, 0x0a, 0x10, 0x52, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x13, 0x0a, 0x11, 0x52, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x61, 0x0a, 0x13, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, - 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, - 0x72, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x14, 0x6f, 0x6e, 0x6c, 0x79, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, - 0x65, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x84, 0x02, 0x0a, 0x14, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x31, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x06, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x6c, 0x61, 0x74, 0x65, - 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x76, 0x61, - 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x26, 0x0a, - 0x0f, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x6e, 0x65, 0x77, 0x73, 0x5f, 0x75, 0x72, 0x6c, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4e, 0x65, - 0x77, 0x73, 0x55, 0x72, 0x6c, 0x12, 0x39, 0x0a, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x68, - 0x65, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x6c, 0x61, 0x73, 0x74, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x22, 0x14, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x72, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x53, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, - 0x0a, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, - 0x6c, 0x6f, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x09, 0x6c, 0x6f, 0x67, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x53, 0x0a, 0x13, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6c, 0x6f, 0x67, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, - 0x22, 0x66, 0x0a, 0x14, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, - 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, - 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x5f, 0x6f, 0x66, 0x66, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x18, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x22, 0x1b, 0x0a, 0x19, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x61, 0x0a, 0x13, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x16, + 0x6f, 0x6e, 0x6c, 0x79, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x6f, 0x6e, + 0x6c, 0x79, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x84, 0x02, 0x0a, 0x14, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x09, 0x69, + 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x12, 0x2b, + 0x0a, 0x06, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x76, 0x61, + 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, + 0x5f, 0x6e, 0x65, 0x77, 0x73, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4e, 0x65, 0x77, 0x73, 0x55, 0x72, 0x6c, 0x12, 0x39, + 0x0a, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x6c, 0x61, 0x73, 0x74, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x22, 0x14, 0x0a, 0x12, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, + 0x53, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6c, 0x6f, 0x67, 0x4f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x12, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x29, 0x0a, 0x02, 0x68, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x68, 0x72, 0x12, 0x29, 0x0a, 0x02, 0x6d, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x02, 0x6d, 0x72, 0x12, 0x29, 0x0a, 0x02, 0x6c, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x6c, 0x72, - 0x22, 0xfe, 0x01, 0x0a, 0x15, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x04, 0x66, 0x72, - 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, - 0x01, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x25, 0x0a, 0x09, 0x73, 0x6d, 0x61, 0x72, 0x74, - 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, - 0x02, 0x10, 0x01, 0x52, 0x09, 0x73, 0x6d, 0x61, 0x72, 0x74, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x14, - 0x0a, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x68, - 0x65, 0x6c, 0x6c, 0x6f, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x1a, 0x0a, 0x08, - 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x54, 0x6c, - 0x73, 0x22, 0x32, 0x0a, 0x15, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x19, 0x0a, 0x03, 0x75, 0x72, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, - 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xe3, 0x01, 0x0a, 0x11, 0x53, 0x54, 0x54, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x12, 0x46, 0x0a, 0x11, 0x73, - 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x10, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x76, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x61, 0x72, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x66, 0x73, 0x65, 0x74, 0x22, 0x53, 0x0a, 0x13, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, + 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, + 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x6c, 0x6f, 0x67, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x66, 0x0a, 0x14, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x1d, + 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x6c, 0x6f, 0x67, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x6f, 0x6e, + 0x65, 0x22, 0x95, 0x01, 0x0a, 0x12, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, + 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x02, 0x68, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x02, 0x68, 0x72, 0x12, 0x29, 0x0a, 0x02, 0x6d, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x6d, 0x72, 0x12, 0x29, + 0x0a, 0x02, 0x6c, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x72, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x76, 0x61, 0x6c, 0x12, 0x46, 0x0a, 0x11, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x66, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x22, 0xfe, 0x08, 0x0a, 0x08, - 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x73, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x44, 0x69, 0x73, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, - 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, - 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x12, 0x4b, 0x0a, 0x13, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x6f, - 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, - 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x40, 0x0a, - 0x0e, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x17, 0x0a, 0x07, 0x73, 0x73, 0x68, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x73, 0x68, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x77, 0x73, 0x5f, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0d, 0x61, 0x77, 0x73, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x65, 0x72, - 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x2e, 0x0a, 0x13, 0x61, - 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x72, 0x75, 0x6c, - 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x4d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x73, - 0x74, 0x74, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0a, 0x73, 0x74, 0x74, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x25, 0x0a, 0x0e, - 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x45, 0x6d, - 0x61, 0x69, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x62, 0x61, 0x61, 0x73, 0x5f, 0x65, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x64, 0x62, 0x61, 0x61, - 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x6c, 0x65, 0x72, - 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x12, 0x55, 0x0a, 0x17, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x6c, 0x65, - 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0d, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x45, 0x6d, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x6c, 0x72, 0x22, 0xfe, 0x01, 0x0a, 0x15, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x52, 0x15, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x55, 0x0a, 0x17, 0x73, 0x6c, - 0x61, 0x63, 0x6b, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, - 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x15, 0x73, 0x6c, 0x61, 0x63, - 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x6d, 0x6d, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, - 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, - 0x6d, 0x6d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x49, 0x0a, 0x13, 0x73, 0x74, 0x74, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x54, 0x54, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x52, 0x11, 0x73, 0x74, 0x74, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x62, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, - 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x62, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x33, 0x0a, 0x15, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, - 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, - 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, - 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x15, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x70, 0x6c, 0x61, 0x74, - 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x65, 0x64, 0x54, 0x6f, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, - 0x2f, 0x0a, 0x13, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x73, 0x75, 0x6d, - 0x6d, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x74, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x69, 0x65, 0x73, - 0x12, 0x32, 0x0a, 0x15, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x13, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x26, 0x0a, 0x0f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, - 0x72, 0x6f, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x6c, 0x65, 0x49, 0x64, 0x22, 0x14, 0x0a, 0x12, - 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x43, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x73, 0x65, 0x74, - 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x08, 0x73, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x22, 0xf0, 0x0c, 0x0a, 0x15, 0x43, 0x68, 0x61, 0x6e, - 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x73, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x65, 0x6e, 0x61, 0x62, 0x6c, - 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x61, - 0x62, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x1d, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0e, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x74, 0x65, 0x6c, 0x65, - 0x6d, 0x65, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x65, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x12, 0x2b, 0x0a, 0x11, - 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, - 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x12, 0x4b, 0x0a, 0x13, 0x6d, 0x65, 0x74, + 0x6e, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, + 0x12, 0x25, 0x0a, 0x09, 0x73, 0x6d, 0x61, 0x72, 0x74, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x09, 0x73, 0x6d, + 0x61, 0x72, 0x74, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x12, 0x1a, 0x0a, + 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, + 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, + 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x71, + 0x75, 0x69, 0x72, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x54, 0x6c, 0x73, 0x22, 0x32, 0x0a, 0x15, 0x53, 0x6c, + 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x12, 0x19, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xe3, + 0x01, 0x0a, 0x11, 0x53, 0x54, 0x54, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x76, 0x61, 0x6c, 0x73, 0x12, 0x46, 0x0a, 0x11, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, + 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x73, 0x74, 0x61, 0x6e, + 0x64, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0d, + 0x72, 0x61, 0x72, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, + 0x72, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x46, 0x0a, 0x11, + 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, + 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x10, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x76, 0x61, 0x6c, 0x22, 0xfe, 0x08, 0x0a, 0x08, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x73, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x69, 0x73, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x73, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x11, + 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, + 0x72, 0x79, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4b, 0x0a, 0x13, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, @@ -2094,222 +2090,321 @@ var file_serverpb_server_proto_rawDesc = []byte{ 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x55, 0x72, 0x6c, 0x12, 0x37, 0x0a, 0x18, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, - 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x6c, - 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x2e, 0x0a, - 0x13, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x72, - 0x75, 0x6c, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x6c, 0x65, 0x72, - 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x3b, 0x0a, - 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x17, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x4d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x74, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, - 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x73, - 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x74, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x0d, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0e, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x6c, 0x65, 0x72, 0x74, - 0x69, 0x6e, 0x67, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, - 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x64, - 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x55, - 0x0a, 0x17, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, - 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, - 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x15, - 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, - 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x43, 0x0a, 0x1e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, + 0x72, 0x55, 0x72, 0x6c, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x11, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x52, + 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x74, 0x5f, 0x65, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x74, 0x74, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, + 0x6d, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x23, 0x0a, 0x0d, + 0x64, 0x62, 0x61, 0x61, 0x73, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0c, 0x64, 0x62, 0x61, 0x61, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x61, 0x6c, 0x65, + 0x72, 0x74, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x55, 0x0a, 0x17, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1b, 0x72, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x55, 0x0a, 0x17, 0x73, 0x6c, - 0x61, 0x63, 0x6b, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, - 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x15, 0x73, 0x6c, 0x61, 0x63, - 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x12, 0x43, 0x0a, 0x1e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x73, 0x6c, 0x61, 0x63, - 0x6b, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1b, 0x72, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x6d, 0x6d, 0x5f, 0x70, 0x75, - 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x13, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x10, 0x70, 0x6d, 0x6d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x41, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x12, 0x39, 0x0a, 0x19, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x70, - 0x6d, 0x6d, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, - 0x6d, 0x6d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x49, 0x0a, 0x13, 0x73, 0x74, 0x74, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x54, 0x54, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x52, 0x11, 0x73, 0x74, 0x74, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x12, 0x31, 0x0a, 0x14, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, - 0x65, 0x72, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x41, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x12, 0x33, 0x0a, - 0x15, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, - 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, - 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, - 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x18, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x3a, 0x0a, 0x19, - 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x17, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x5f, 0x64, 0x62, 0x61, 0x61, 0x73, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, - 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x62, 0x61, 0x61, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x64, - 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, 0x62, 0x61, 0x61, 0x73, 0x18, 0x1b, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x62, 0x61, 0x61, 0x73, - 0x12, 0x32, 0x0a, 0x15, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x13, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x34, 0x0a, 0x16, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, - 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x1f, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, 0x46, 0x0a, 0x16, 0x43, 0x68, - 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, - 0x67, 0x73, 0x22, 0x94, 0x01, 0x0a, 0x20, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, - 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x55, 0x0a, 0x17, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, + 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x15, 0x65, 0x6d, + 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x12, 0x55, 0x0a, 0x17, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x5f, 0x61, 0x6c, 0x65, + 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0e, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x6c, + 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x52, 0x15, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, + 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x6d, + 0x6d, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, 0x6d, 0x6d, 0x50, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x49, 0x0a, 0x13, 0x73, 0x74, 0x74, 0x5f, + 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x18, + 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, + 0x54, 0x54, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, + 0x52, 0x11, 0x73, 0x74, 0x74, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, + 0x33, 0x0a, 0x15, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, + 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, + 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x15, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x13, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x54, 0x6f, + 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x2f, 0x0a, 0x13, 0x74, 0x65, 0x6c, 0x65, + 0x6d, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, + 0x14, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, + 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x26, 0x0a, + 0x0f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x72, 0x6f, 0x6c, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x16, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, + 0x6f, 0x6c, 0x65, 0x49, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x43, 0x0a, 0x13, 0x47, + 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x22, 0xf0, 0x0c, 0x0a, 0x15, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x1c, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0d, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x73, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x64, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x65, 0x6c, 0x65, + 0x6d, 0x65, 0x74, 0x72, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x10, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, + 0x72, 0x79, 0x12, 0x4b, 0x0a, 0x13, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x72, 0x65, + 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x40, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x73, 0x68, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x73, 0x73, 0x68, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x77, + 0x73, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0d, 0x61, 0x77, 0x73, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, + 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x37, 0x0a, + 0x18, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x15, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x72, + 0x75, 0x6c, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x72, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x52, 0x75, + 0x6c, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x74, + 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, + 0x74, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x74, + 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x53, 0x74, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x6c, + 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x29, 0x0a, 0x10, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, + 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, + 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x55, 0x0a, 0x17, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, - 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x67, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x15, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, - 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x19, - 0x0a, 0x08, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x54, 0x6f, 0x22, 0x23, 0x0a, 0x21, 0x54, 0x65, 0x73, - 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x43, - 0x0a, 0x17, 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x0b, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, - 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, - 0x66, 0x0a, 0x12, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x4d, - 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x1f, 0x0a, 0x1b, 0x44, 0x49, 0x53, 0x54, 0x52, 0x49, 0x42, - 0x55, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x48, 0x4f, 0x44, 0x5f, 0x49, 0x4e, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, - 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x4f, 0x56, 0x46, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x41, - 0x4d, 0x49, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x10, 0x04, 0x12, - 0x06, 0x0a, 0x02, 0x44, 0x4f, 0x10, 0x05, 0x32, 0xef, 0x0c, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x12, 0x79, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3d, - 0x92, 0x41, 0x27, 0x12, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x1c, 0x52, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0x73, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, - 0x12, 0x0b, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x9e, 0x02, - 0x0a, 0x09, 0x52, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x12, 0x18, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x52, - 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0xdb, 0x01, 0x92, 0x41, 0xc5, 0x01, 0x12, 0x16, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x20, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x72, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x1a, - 0xaa, 0x01, 0x52, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x20, 0x77, 0x68, 0x65, 0x6e, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x63, - 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x20, 0x62, 0x65, 0x69, 0x6e, 0x67, 0x20, - 0x72, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x20, 0x61, 0x72, 0x65, 0x20, 0x6e, 0x6f, - 0x74, 0x20, 0x72, 0x65, 0x61, 0x64, 0x79, 0x20, 0x79, 0x65, 0x74, 0x2e, 0x20, 0x55, 0x73, 0x65, - 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x41, 0x50, 0x49, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x63, 0x68, - 0x65, 0x63, 0x6b, 0x69, 0x6e, 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, 0x68, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x20, 0x6f, 0x66, 0x20, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x70, - 0x72, 0x6f, 0x62, 0x69, 0x6e, 0x67, 0x20, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, - 0x73, 0x20, 0x72, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, - 0x02, 0x0c, 0x12, 0x0a, 0x2f, 0x76, 0x31, 0x2f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x12, 0xa3, - 0x01, 0x0a, 0x0c, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, - 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x58, 0x92, 0x41, 0x39, 0x12, - 0x0d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x1a, 0x28, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x16, 0x3a, 0x01, - 0x2a, 0x22, 0x11, 0x2f, 0x76, 0x31, 0x2f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x12, 0x90, 0x01, 0x0a, 0x0b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x74, - 0x61, 0x72, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x48, 0x92, - 0x41, 0x29, 0x12, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x1a, 0x19, 0x53, 0x74, 0x61, 0x72, 0x74, 0x73, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x16, 0x3a, 0x01, 0x2a, 0x22, 0x11, 0x2f, 0x76, 0x31, 0x2f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x73, 0x2f, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x9d, 0x01, 0x0a, 0x0c, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x52, 0x92, 0x41, 0x32, 0x12, 0x0d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x20, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x21, 0x52, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x73, - 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x17, - 0x3a, 0x01, 0x2a, 0x22, 0x12, 0x2f, 0x76, 0x31, 0x2f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, - 0x2f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x9a, 0x01, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x53, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1a, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x52, 0x92, 0x41, 0x34, 0x12, 0x0c, 0x47, 0x65, 0x74, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x1a, 0x24, 0x52, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x20, 0x63, 0x75, 0x72, - 0x72, 0x65, 0x6e, 0x74, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, - 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x15, 0x3a, - 0x01, 0x2a, 0x22, 0x10, 0x2f, 0x76, 0x31, 0x2f, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, - 0x2f, 0x47, 0x65, 0x74, 0x12, 0xa1, 0x01, 0x0a, 0x0e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, - 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x92, 0x41, 0x2f, 0x12, 0x0f, 0x43, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x1c, 0x43, 0x68, - 0x61, 0x6e, 0x67, 0x65, 0x73, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x18, - 0x3a, 0x01, 0x2a, 0x22, 0x13, 0x2f, 0x76, 0x31, 0x2f, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x2f, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x81, 0x02, 0x0a, 0x19, 0x54, 0x65, 0x73, - 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x28, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, + 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x43, + 0x0a, 0x1e, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, + 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1b, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x6d, + 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x12, 0x55, 0x0a, 0x17, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x5f, 0x61, 0x6c, 0x65, + 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x11, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x6c, + 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x52, 0x15, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, + 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x43, 0x0a, 0x1e, 0x72, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, + 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x12, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x1b, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x41, + 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, + 0x2c, 0x0a, 0x12, 0x70, 0x6d, 0x6d, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x61, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, 0x6d, 0x6d, + 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x39, 0x0a, + 0x19, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x70, 0x6d, 0x6d, 0x5f, 0x70, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x6d, 0x6d, 0x50, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x49, 0x0a, 0x13, 0x73, 0x74, 0x74, 0x5f, + 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, 0x18, + 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, + 0x54, 0x54, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x73, + 0x52, 0x11, 0x73, 0x74, 0x74, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x73, 0x12, 0x31, 0x0a, 0x14, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x7a, + 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x18, 0x16, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x13, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, + 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x12, 0x33, 0x0a, 0x15, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, + 0x65, 0x5f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x18, + 0x17, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x7a, + 0x75, 0x72, 0x65, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x3a, 0x0a, 0x19, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, + 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, 0x62, 0x61, 0x61, + 0x73, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x44, + 0x62, 0x61, 0x61, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, + 0x64, 0x62, 0x61, 0x61, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x64, 0x69, 0x73, + 0x61, 0x62, 0x6c, 0x65, 0x44, 0x62, 0x61, 0x61, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x34, 0x0a, + 0x16, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, + 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x22, 0x46, 0x0a, 0x16, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, + 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, + 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x10, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x73, 0x52, 0x08, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x94, 0x01, 0x0a, 0x20, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x29, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, - 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, - 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8e, 0x01, 0x92, 0x41, - 0x5a, 0x12, 0x13, 0x54, 0x65, 0x73, 0x74, 0x20, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x20, 0x61, 0x6c, - 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x43, 0x53, 0x65, 0x6e, 0x64, 0x73, 0x20, 0x74, 0x65, - 0x73, 0x74, 0x20, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x63, 0x68, 0x65, 0x63, - 0x6b, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x53, 0x4d, 0x54, 0x50, 0x20, 0x73, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x65, 0x6d, 0x61, 0x69, - 0x6c, 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x2b, 0x3a, 0x01, 0x2a, 0x22, 0x26, 0x2f, 0x76, 0x31, 0x2f, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, - 0x67, 0x73, 0x2f, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, - 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0xaa, 0x01, 0x0a, - 0x10, 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x12, 0x1f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x41, 0x57, 0x53, 0x49, 0x6e, + 0x12, 0x55, 0x0a, 0x17, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, + 0x6e, 0x67, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x45, 0x6d, 0x61, 0x69, 0x6c, + 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x52, 0x15, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, + 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x5f, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x54, 0x6f, 0x22, 0x23, 0x0a, 0x21, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, + 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x43, 0x0a, 0x17, 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x41, 0x57, 0x53, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x92, 0x41, 0x31, 0x12, 0x12, 0x41, 0x57, 0x53, 0x20, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x1a, 0x1b, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x73, 0x20, 0x41, 0x57, 0x53, 0x20, 0x45, 0x43, 0x32, 0x20, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x20, 0x49, 0x44, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x19, - 0x3a, 0x01, 0x2a, 0x22, 0x14, 0x2f, 0x76, 0x31, 0x2f, 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x76, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x70, 0x65, 0x72, 0x63, 0x6f, 0x6e, 0x61, 0x2f, 0x70, 0x6d, 0x6d, 0x2f, 0x61, - 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0xa2, 0x02, 0x03, 0x53, 0x58, - 0x58, 0xaa, 0x02, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0xca, 0x02, 0x06, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0xe2, 0x02, 0x12, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5c, 0x47, 0x50, 0x42, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x74, 0x12, 0x28, 0x0a, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, + 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x1a, 0x0a, 0x18, + 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x66, 0x0a, 0x12, 0x44, 0x69, 0x73, 0x74, + 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x1f, + 0x0a, 0x1b, 0x44, 0x49, 0x53, 0x54, 0x52, 0x49, 0x42, 0x55, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, + 0x45, 0x54, 0x48, 0x4f, 0x44, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, + 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x4f, + 0x56, 0x46, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4d, 0x49, 0x10, 0x03, 0x12, 0x09, 0x0a, + 0x05, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x10, 0x04, 0x12, 0x06, 0x0a, 0x02, 0x44, 0x4f, 0x10, 0x05, + 0x32, 0xe9, 0x0e, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x79, 0x0a, 0x07, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3d, 0x92, 0x41, 0x27, 0x12, 0x07, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x1c, 0x52, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x20, 0x50, + 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x12, 0x0b, 0x2f, 0x76, 0x31, 0x2f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x9e, 0x02, 0x0a, 0x09, 0x52, 0x65, 0x61, 0x64, 0x69, + 0x6e, 0x65, 0x73, 0x73, 0x12, 0x18, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xdb, 0x01, 0x92, 0x41, 0xc5, 0x01, + 0x12, 0x16, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x72, + 0x65, 0x61, 0x64, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x1a, 0xaa, 0x01, 0x52, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0x73, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x77, 0x68, 0x65, 0x6e, + 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x73, 0x20, 0x62, 0x65, 0x69, 0x6e, 0x67, 0x20, 0x72, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x65, 0x64, 0x20, 0x61, 0x72, 0x65, 0x20, 0x6e, 0x6f, 0x74, 0x20, 0x72, 0x65, 0x61, 0x64, 0x79, + 0x20, 0x79, 0x65, 0x74, 0x2e, 0x20, 0x55, 0x73, 0x65, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x41, + 0x50, 0x49, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x69, 0x6e, 0x67, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x20, 0x6f, 0x66, 0x20, 0x44, 0x6f, + 0x63, 0x6b, 0x65, 0x72, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x20, + 0x61, 0x6e, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x70, 0x72, 0x6f, 0x62, 0x69, 0x6e, 0x67, 0x20, + 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x20, 0x72, 0x65, 0x61, 0x64, 0x69, + 0x6e, 0x65, 0x73, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x12, 0x0a, 0x2f, 0x76, 0x31, + 0x2f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x7a, 0x12, 0xf7, 0x01, 0x0a, 0x11, 0x4c, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x20, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x21, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x9c, 0x01, 0x92, 0x41, 0x79, 0x12, 0x10, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x20, + 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, 0x1a, 0x65, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x73, 0x20, 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x63, 0x65, 0x20, 0x69, 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x20, 0x69, 0x6e, 0x20, 0x61, 0x20, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x20, 0x52, + 0x65, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, + 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x20, + 0x69, 0x73, 0x6e, 0x27, 0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, 0x2f, 0x76, 0x31, 0x2f, + 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x12, 0xa3, 0x01, 0x0a, 0x0c, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x58, 0x92, + 0x41, 0x39, 0x12, 0x0d, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x73, 0x1a, 0x28, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x76, + 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, + 0x16, 0x3a, 0x01, 0x2a, 0x22, 0x11, 0x2f, 0x76, 0x31, 0x2f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x73, 0x2f, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x90, 0x01, 0x0a, 0x0b, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x48, 0x92, 0x41, 0x29, 0x12, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x20, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x1a, 0x19, 0x53, 0x74, 0x61, 0x72, 0x74, 0x73, 0x20, 0x50, 0x4d, 0x4d, 0x20, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x82, 0xd3, + 0xe4, 0x93, 0x02, 0x16, 0x3a, 0x01, 0x2a, 0x22, 0x11, 0x2f, 0x76, 0x31, 0x2f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x73, 0x2f, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x9d, 0x01, 0x0a, 0x0c, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x52, 0x92, 0x41, 0x32, 0x12, 0x0d, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x21, 0x52, 0x65, 0x74, 0x75, + 0x72, 0x6e, 0x73, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x17, 0x3a, 0x01, 0x2a, 0x22, 0x12, 0x2f, 0x76, 0x31, 0x2f, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x73, 0x2f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x9a, 0x01, 0x0a, 0x0b, 0x47, + 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1a, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x52, 0x92, 0x41, 0x34, 0x12, 0x0c, 0x47, 0x65, 0x74, 0x20, 0x73, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x24, 0x52, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x20, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x82, 0xd3, 0xe4, 0x93, + 0x02, 0x15, 0x3a, 0x01, 0x2a, 0x22, 0x10, 0x2f, 0x76, 0x31, 0x2f, 0x53, 0x65, 0x74, 0x74, 0x69, + 0x6e, 0x67, 0x73, 0x2f, 0x47, 0x65, 0x74, 0x12, 0xa1, 0x01, 0x0a, 0x0e, 0x43, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1d, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, + 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x92, 0x41, 0x2f, 0x12, 0x0f, + 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x1a, + 0x1c, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x20, 0x50, 0x4d, 0x4d, 0x20, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2e, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x18, 0x3a, 0x01, 0x2a, 0x22, 0x13, 0x2f, 0x76, 0x31, 0x2f, 0x53, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x2f, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x81, 0x02, 0x0a, 0x19, + 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, + 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x28, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, + 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x73, + 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, + 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8e, + 0x01, 0x92, 0x41, 0x5a, 0x12, 0x13, 0x54, 0x65, 0x73, 0x74, 0x20, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x43, 0x53, 0x65, 0x6e, 0x64, 0x73, + 0x20, 0x74, 0x65, 0x73, 0x74, 0x20, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x20, 0x74, 0x6f, 0x20, 0x63, + 0x68, 0x65, 0x63, 0x6b, 0x20, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x53, 0x4d, 0x54, + 0x50, 0x20, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x65, + 0x6d, 0x61, 0x69, 0x6c, 0x20, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x82, 0xd3, + 0xe4, 0x93, 0x02, 0x2b, 0x3a, 0x01, 0x2a, 0x22, 0x26, 0x2f, 0x76, 0x31, 0x2f, 0x53, 0x65, 0x74, + 0x74, 0x69, 0x6e, 0x67, 0x73, 0x2f, 0x54, 0x65, 0x73, 0x74, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x41, + 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x12, + 0xaa, 0x01, 0x0a, 0x10, 0x41, 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x41, 0x57, + 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x41, + 0x57, 0x53, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x92, 0x41, 0x31, 0x12, 0x12, 0x41, 0x57, + 0x53, 0x20, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6b, + 0x1a, 0x1b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x20, 0x41, 0x57, 0x53, 0x20, 0x45, 0x43, 0x32, + 0x20, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x20, 0x49, 0x44, 0x2e, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x19, 0x3a, 0x01, 0x2a, 0x22, 0x14, 0x2f, 0x76, 0x31, 0x2f, 0x41, 0x57, 0x53, 0x49, + 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x42, 0x76, 0x0a, 0x0a, + 0x63, 0x6f, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x0b, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x65, 0x72, 0x63, 0x6f, 0x6e, 0x61, 0x2f, 0x70, 0x6d, + 0x6d, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x70, 0x62, 0xa2, 0x02, + 0x03, 0x53, 0x58, 0x58, 0xaa, 0x02, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0xca, 0x02, 0x06, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0xe2, 0x02, 0x12, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5c, + 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x06, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2326,7 +2421,7 @@ func file_serverpb_server_proto_rawDescGZIP() []byte { var ( file_serverpb_server_proto_enumTypes = make([]protoimpl.EnumInfo, 1) - file_serverpb_server_proto_msgTypes = make([]protoimpl.MessageInfo, 24) + file_serverpb_server_proto_msgTypes = make([]protoimpl.MessageInfo, 26) file_serverpb_server_proto_goTypes = []interface{}{ (DistributionMethod)(0), // 0: server.DistributionMethod (*VersionInfo)(nil), // 1: server.VersionInfo @@ -2334,77 +2429,81 @@ var ( (*VersionResponse)(nil), // 3: server.VersionResponse (*ReadinessRequest)(nil), // 4: server.ReadinessRequest (*ReadinessResponse)(nil), // 5: server.ReadinessResponse - (*CheckUpdatesRequest)(nil), // 6: server.CheckUpdatesRequest - (*CheckUpdatesResponse)(nil), // 7: server.CheckUpdatesResponse - (*StartUpdateRequest)(nil), // 8: server.StartUpdateRequest - (*StartUpdateResponse)(nil), // 9: server.StartUpdateResponse - (*UpdateStatusRequest)(nil), // 10: server.UpdateStatusRequest - (*UpdateStatusResponse)(nil), // 11: server.UpdateStatusResponse - (*MetricsResolutions)(nil), // 12: server.MetricsResolutions - (*EmailAlertingSettings)(nil), // 13: server.EmailAlertingSettings - (*SlackAlertingSettings)(nil), // 14: server.SlackAlertingSettings - (*STTCheckIntervals)(nil), // 15: server.STTCheckIntervals - (*Settings)(nil), // 16: server.Settings - (*GetSettingsRequest)(nil), // 17: server.GetSettingsRequest - (*GetSettingsResponse)(nil), // 18: server.GetSettingsResponse - (*ChangeSettingsRequest)(nil), // 19: server.ChangeSettingsRequest - (*ChangeSettingsResponse)(nil), // 20: server.ChangeSettingsResponse - (*TestEmailAlertingSettingsRequest)(nil), // 21: server.TestEmailAlertingSettingsRequest - (*TestEmailAlertingSettingsResponse)(nil), // 22: server.TestEmailAlertingSettingsResponse - (*AWSInstanceCheckRequest)(nil), // 23: server.AWSInstanceCheckRequest - (*AWSInstanceCheckResponse)(nil), // 24: server.AWSInstanceCheckResponse - (*timestamppb.Timestamp)(nil), // 25: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 26: google.protobuf.Duration + (*LeaderHealthCheckRequest)(nil), // 6: server.LeaderHealthCheckRequest + (*LeaderHealthCheckResponse)(nil), // 7: server.LeaderHealthCheckResponse + (*CheckUpdatesRequest)(nil), // 8: server.CheckUpdatesRequest + (*CheckUpdatesResponse)(nil), // 9: server.CheckUpdatesResponse + (*StartUpdateRequest)(nil), // 10: server.StartUpdateRequest + (*StartUpdateResponse)(nil), // 11: server.StartUpdateResponse + (*UpdateStatusRequest)(nil), // 12: server.UpdateStatusRequest + (*UpdateStatusResponse)(nil), // 13: server.UpdateStatusResponse + (*MetricsResolutions)(nil), // 14: server.MetricsResolutions + (*EmailAlertingSettings)(nil), // 15: server.EmailAlertingSettings + (*SlackAlertingSettings)(nil), // 16: server.SlackAlertingSettings + (*STTCheckIntervals)(nil), // 17: server.STTCheckIntervals + (*Settings)(nil), // 18: server.Settings + (*GetSettingsRequest)(nil), // 19: server.GetSettingsRequest + (*GetSettingsResponse)(nil), // 20: server.GetSettingsResponse + (*ChangeSettingsRequest)(nil), // 21: server.ChangeSettingsRequest + (*ChangeSettingsResponse)(nil), // 22: server.ChangeSettingsResponse + (*TestEmailAlertingSettingsRequest)(nil), // 23: server.TestEmailAlertingSettingsRequest + (*TestEmailAlertingSettingsResponse)(nil), // 24: server.TestEmailAlertingSettingsResponse + (*AWSInstanceCheckRequest)(nil), // 25: server.AWSInstanceCheckRequest + (*AWSInstanceCheckResponse)(nil), // 26: server.AWSInstanceCheckResponse + (*timestamppb.Timestamp)(nil), // 27: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 28: google.protobuf.Duration } ) var file_serverpb_server_proto_depIdxs = []int32{ - 25, // 0: server.VersionInfo.timestamp:type_name -> google.protobuf.Timestamp + 27, // 0: server.VersionInfo.timestamp:type_name -> google.protobuf.Timestamp 1, // 1: server.VersionResponse.server:type_name -> server.VersionInfo 1, // 2: server.VersionResponse.managed:type_name -> server.VersionInfo 0, // 3: server.VersionResponse.distribution_method:type_name -> server.DistributionMethod 1, // 4: server.CheckUpdatesResponse.installed:type_name -> server.VersionInfo 1, // 5: server.CheckUpdatesResponse.latest:type_name -> server.VersionInfo - 25, // 6: server.CheckUpdatesResponse.last_check:type_name -> google.protobuf.Timestamp - 26, // 7: server.MetricsResolutions.hr:type_name -> google.protobuf.Duration - 26, // 8: server.MetricsResolutions.mr:type_name -> google.protobuf.Duration - 26, // 9: server.MetricsResolutions.lr:type_name -> google.protobuf.Duration - 26, // 10: server.STTCheckIntervals.standard_interval:type_name -> google.protobuf.Duration - 26, // 11: server.STTCheckIntervals.rare_interval:type_name -> google.protobuf.Duration - 26, // 12: server.STTCheckIntervals.frequent_interval:type_name -> google.protobuf.Duration - 12, // 13: server.Settings.metrics_resolutions:type_name -> server.MetricsResolutions - 26, // 14: server.Settings.data_retention:type_name -> google.protobuf.Duration - 13, // 15: server.Settings.email_alerting_settings:type_name -> server.EmailAlertingSettings - 14, // 16: server.Settings.slack_alerting_settings:type_name -> server.SlackAlertingSettings - 15, // 17: server.Settings.stt_check_intervals:type_name -> server.STTCheckIntervals - 16, // 18: server.GetSettingsResponse.settings:type_name -> server.Settings - 12, // 19: server.ChangeSettingsRequest.metrics_resolutions:type_name -> server.MetricsResolutions - 26, // 20: server.ChangeSettingsRequest.data_retention:type_name -> google.protobuf.Duration - 13, // 21: server.ChangeSettingsRequest.email_alerting_settings:type_name -> server.EmailAlertingSettings - 14, // 22: server.ChangeSettingsRequest.slack_alerting_settings:type_name -> server.SlackAlertingSettings - 15, // 23: server.ChangeSettingsRequest.stt_check_intervals:type_name -> server.STTCheckIntervals - 16, // 24: server.ChangeSettingsResponse.settings:type_name -> server.Settings - 13, // 25: server.TestEmailAlertingSettingsRequest.email_alerting_settings:type_name -> server.EmailAlertingSettings + 27, // 6: server.CheckUpdatesResponse.last_check:type_name -> google.protobuf.Timestamp + 28, // 7: server.MetricsResolutions.hr:type_name -> google.protobuf.Duration + 28, // 8: server.MetricsResolutions.mr:type_name -> google.protobuf.Duration + 28, // 9: server.MetricsResolutions.lr:type_name -> google.protobuf.Duration + 28, // 10: server.STTCheckIntervals.standard_interval:type_name -> google.protobuf.Duration + 28, // 11: server.STTCheckIntervals.rare_interval:type_name -> google.protobuf.Duration + 28, // 12: server.STTCheckIntervals.frequent_interval:type_name -> google.protobuf.Duration + 14, // 13: server.Settings.metrics_resolutions:type_name -> server.MetricsResolutions + 28, // 14: server.Settings.data_retention:type_name -> google.protobuf.Duration + 15, // 15: server.Settings.email_alerting_settings:type_name -> server.EmailAlertingSettings + 16, // 16: server.Settings.slack_alerting_settings:type_name -> server.SlackAlertingSettings + 17, // 17: server.Settings.stt_check_intervals:type_name -> server.STTCheckIntervals + 18, // 18: server.GetSettingsResponse.settings:type_name -> server.Settings + 14, // 19: server.ChangeSettingsRequest.metrics_resolutions:type_name -> server.MetricsResolutions + 28, // 20: server.ChangeSettingsRequest.data_retention:type_name -> google.protobuf.Duration + 15, // 21: server.ChangeSettingsRequest.email_alerting_settings:type_name -> server.EmailAlertingSettings + 16, // 22: server.ChangeSettingsRequest.slack_alerting_settings:type_name -> server.SlackAlertingSettings + 17, // 23: server.ChangeSettingsRequest.stt_check_intervals:type_name -> server.STTCheckIntervals + 18, // 24: server.ChangeSettingsResponse.settings:type_name -> server.Settings + 15, // 25: server.TestEmailAlertingSettingsRequest.email_alerting_settings:type_name -> server.EmailAlertingSettings 2, // 26: server.Server.Version:input_type -> server.VersionRequest 4, // 27: server.Server.Readiness:input_type -> server.ReadinessRequest - 6, // 28: server.Server.CheckUpdates:input_type -> server.CheckUpdatesRequest - 8, // 29: server.Server.StartUpdate:input_type -> server.StartUpdateRequest - 10, // 30: server.Server.UpdateStatus:input_type -> server.UpdateStatusRequest - 17, // 31: server.Server.GetSettings:input_type -> server.GetSettingsRequest - 19, // 32: server.Server.ChangeSettings:input_type -> server.ChangeSettingsRequest - 21, // 33: server.Server.TestEmailAlertingSettings:input_type -> server.TestEmailAlertingSettingsRequest - 23, // 34: server.Server.AWSInstanceCheck:input_type -> server.AWSInstanceCheckRequest - 3, // 35: server.Server.Version:output_type -> server.VersionResponse - 5, // 36: server.Server.Readiness:output_type -> server.ReadinessResponse - 7, // 37: server.Server.CheckUpdates:output_type -> server.CheckUpdatesResponse - 9, // 38: server.Server.StartUpdate:output_type -> server.StartUpdateResponse - 11, // 39: server.Server.UpdateStatus:output_type -> server.UpdateStatusResponse - 18, // 40: server.Server.GetSettings:output_type -> server.GetSettingsResponse - 20, // 41: server.Server.ChangeSettings:output_type -> server.ChangeSettingsResponse - 22, // 42: server.Server.TestEmailAlertingSettings:output_type -> server.TestEmailAlertingSettingsResponse - 24, // 43: server.Server.AWSInstanceCheck:output_type -> server.AWSInstanceCheckResponse - 35, // [35:44] is the sub-list for method output_type - 26, // [26:35] is the sub-list for method input_type + 6, // 28: server.Server.LeaderHealthCheck:input_type -> server.LeaderHealthCheckRequest + 8, // 29: server.Server.CheckUpdates:input_type -> server.CheckUpdatesRequest + 10, // 30: server.Server.StartUpdate:input_type -> server.StartUpdateRequest + 12, // 31: server.Server.UpdateStatus:input_type -> server.UpdateStatusRequest + 19, // 32: server.Server.GetSettings:input_type -> server.GetSettingsRequest + 21, // 33: server.Server.ChangeSettings:input_type -> server.ChangeSettingsRequest + 23, // 34: server.Server.TestEmailAlertingSettings:input_type -> server.TestEmailAlertingSettingsRequest + 25, // 35: server.Server.AWSInstanceCheck:input_type -> server.AWSInstanceCheckRequest + 3, // 36: server.Server.Version:output_type -> server.VersionResponse + 5, // 37: server.Server.Readiness:output_type -> server.ReadinessResponse + 7, // 38: server.Server.LeaderHealthCheck:output_type -> server.LeaderHealthCheckResponse + 9, // 39: server.Server.CheckUpdates:output_type -> server.CheckUpdatesResponse + 11, // 40: server.Server.StartUpdate:output_type -> server.StartUpdateResponse + 13, // 41: server.Server.UpdateStatus:output_type -> server.UpdateStatusResponse + 20, // 42: server.Server.GetSettings:output_type -> server.GetSettingsResponse + 22, // 43: server.Server.ChangeSettings:output_type -> server.ChangeSettingsResponse + 24, // 44: server.Server.TestEmailAlertingSettings:output_type -> server.TestEmailAlertingSettingsResponse + 26, // 45: server.Server.AWSInstanceCheck:output_type -> server.AWSInstanceCheckResponse + 36, // [36:46] is the sub-list for method output_type + 26, // [26:36] is the sub-list for method input_type 26, // [26:26] is the sub-list for extension type_name 26, // [26:26] is the sub-list for extension extendee 0, // [0:26] is the sub-list for field type_name @@ -2477,7 +2576,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CheckUpdatesRequest); i { + switch v := v.(*LeaderHealthCheckRequest); i { case 0: return &v.state case 1: @@ -2489,7 +2588,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CheckUpdatesResponse); i { + switch v := v.(*LeaderHealthCheckResponse); i { case 0: return &v.state case 1: @@ -2501,7 +2600,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartUpdateRequest); i { + switch v := v.(*CheckUpdatesRequest); i { case 0: return &v.state case 1: @@ -2513,7 +2612,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartUpdateResponse); i { + switch v := v.(*CheckUpdatesResponse); i { case 0: return &v.state case 1: @@ -2525,7 +2624,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpdateStatusRequest); i { + switch v := v.(*StartUpdateRequest); i { case 0: return &v.state case 1: @@ -2537,7 +2636,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpdateStatusResponse); i { + switch v := v.(*StartUpdateResponse); i { case 0: return &v.state case 1: @@ -2549,7 +2648,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MetricsResolutions); i { + switch v := v.(*UpdateStatusRequest); i { case 0: return &v.state case 1: @@ -2561,7 +2660,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmailAlertingSettings); i { + switch v := v.(*UpdateStatusResponse); i { case 0: return &v.state case 1: @@ -2573,7 +2672,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SlackAlertingSettings); i { + switch v := v.(*MetricsResolutions); i { case 0: return &v.state case 1: @@ -2585,7 +2684,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*STTCheckIntervals); i { + switch v := v.(*EmailAlertingSettings); i { case 0: return &v.state case 1: @@ -2597,7 +2696,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Settings); i { + switch v := v.(*SlackAlertingSettings); i { case 0: return &v.state case 1: @@ -2609,7 +2708,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSettingsRequest); i { + switch v := v.(*STTCheckIntervals); i { case 0: return &v.state case 1: @@ -2621,7 +2720,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSettingsResponse); i { + switch v := v.(*Settings); i { case 0: return &v.state case 1: @@ -2633,7 +2732,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ChangeSettingsRequest); i { + switch v := v.(*GetSettingsRequest); i { case 0: return &v.state case 1: @@ -2645,7 +2744,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ChangeSettingsResponse); i { + switch v := v.(*GetSettingsResponse); i { case 0: return &v.state case 1: @@ -2657,7 +2756,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TestEmailAlertingSettingsRequest); i { + switch v := v.(*ChangeSettingsRequest); i { case 0: return &v.state case 1: @@ -2669,7 +2768,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TestEmailAlertingSettingsResponse); i { + switch v := v.(*ChangeSettingsResponse); i { case 0: return &v.state case 1: @@ -2681,7 +2780,7 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AWSInstanceCheckRequest); i { + switch v := v.(*TestEmailAlertingSettingsRequest); i { case 0: return &v.state case 1: @@ -2693,6 +2792,30 @@ func file_serverpb_server_proto_init() { } } file_serverpb_server_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestEmailAlertingSettingsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_serverpb_server_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AWSInstanceCheckRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_serverpb_server_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AWSInstanceCheckResponse); i { case 0: return &v.state @@ -2711,7 +2834,7 @@ func file_serverpb_server_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_serverpb_server_proto_rawDesc, NumEnums: 1, - NumMessages: 24, + NumMessages: 26, NumExtensions: 0, NumServices: 1, }, diff --git a/api/serverpb/server.pb.gw.go b/api/serverpb/server.pb.gw.go index 6b7c5f0f11..4ffe35ac64 100644 --- a/api/serverpb/server.pb.gw.go +++ b/api/serverpb/server.pb.gw.go @@ -81,6 +81,38 @@ func local_request_Server_Readiness_0(ctx context.Context, marshaler runtime.Mar return msg, metadata, err } +func request_Server_LeaderHealthCheck_0(ctx context.Context, marshaler runtime.Marshaler, client ServerClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq LeaderHealthCheckRequest + var metadata runtime.ServerMetadata + + newReader, berr := utilities.IOReaderFactory(req.Body) + if berr != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) + } + if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.LeaderHealthCheck(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err +} + +func local_request_Server_LeaderHealthCheck_0(ctx context.Context, marshaler runtime.Marshaler, server ServerServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq LeaderHealthCheckRequest + var metadata runtime.ServerMetadata + + newReader, berr := utilities.IOReaderFactory(req.Body) + if berr != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) + } + if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := server.LeaderHealthCheck(ctx, &protoReq) + return msg, metadata, err +} + func request_Server_CheckUpdates_0(ctx context.Context, marshaler runtime.Marshaler, client ServerClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq CheckUpdatesRequest var metadata runtime.ServerMetadata @@ -358,6 +390,30 @@ func RegisterServerHandlerServer(ctx context.Context, mux *runtime.ServeMux, ser forward_Server_Readiness_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) + mux.Handle("POST", pattern_Server_LeaderHealthCheck_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + var err error + var annotatedContext context.Context + annotatedContext, err = runtime.AnnotateIncomingContext(ctx, mux, req, "/server.Server/LeaderHealthCheck", runtime.WithHTTPPathPattern("/v1/leaderHealthCheck")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_Server_LeaderHealthCheck_0(annotatedContext, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + + forward_Server_LeaderHealthCheck_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) + mux.Handle("POST", pattern_Server_CheckUpdates_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -608,6 +664,27 @@ func RegisterServerHandlerClient(ctx context.Context, mux *runtime.ServeMux, cli forward_Server_Readiness_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) + mux.Handle("POST", pattern_Server_LeaderHealthCheck_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + var err error + var annotatedContext context.Context + annotatedContext, err = runtime.AnnotateContext(ctx, mux, req, "/server.Server/LeaderHealthCheck", runtime.WithHTTPPathPattern("/v1/leaderHealthCheck")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_Server_LeaderHealthCheck_0(annotatedContext, inboundMarshaler, client, req, pathParams) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + + forward_Server_LeaderHealthCheck_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) + mux.Handle("POST", pattern_Server_CheckUpdates_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -763,6 +840,8 @@ var ( pattern_Server_Readiness_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v1", "readyz"}, "")) + pattern_Server_LeaderHealthCheck_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v1", "leaderHealthCheck"}, "")) + pattern_Server_CheckUpdates_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "Updates", "Check"}, "")) pattern_Server_StartUpdate_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v1", "Updates", "Start"}, "")) @@ -783,6 +862,8 @@ var ( forward_Server_Readiness_0 = runtime.ForwardResponseMessage + forward_Server_LeaderHealthCheck_0 = runtime.ForwardResponseMessage + forward_Server_CheckUpdates_0 = runtime.ForwardResponseMessage forward_Server_StartUpdate_0 = runtime.ForwardResponseMessage diff --git a/api/serverpb/server.pb.validate.go b/api/serverpb/server.pb.validate.go index de4247a314..15e62b9f2f 100644 --- a/api/serverpb/server.pb.validate.go +++ b/api/serverpb/server.pb.validate.go @@ -633,6 +633,210 @@ var _ interface { ErrorName() string } = ReadinessResponseValidationError{} +// Validate checks the field values on LeaderHealthCheckRequest with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *LeaderHealthCheckRequest) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LeaderHealthCheckRequest with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// LeaderHealthCheckRequestMultiError, or nil if none found. +func (m *LeaderHealthCheckRequest) ValidateAll() error { + return m.validate(true) +} + +func (m *LeaderHealthCheckRequest) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return LeaderHealthCheckRequestMultiError(errors) + } + + return nil +} + +// LeaderHealthCheckRequestMultiError is an error wrapping multiple validation +// errors returned by LeaderHealthCheckRequest.ValidateAll() if the designated +// constraints aren't met. +type LeaderHealthCheckRequestMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LeaderHealthCheckRequestMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LeaderHealthCheckRequestMultiError) AllErrors() []error { return m } + +// LeaderHealthCheckRequestValidationError is the validation error returned by +// LeaderHealthCheckRequest.Validate if the designated constraints aren't met. +type LeaderHealthCheckRequestValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LeaderHealthCheckRequestValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LeaderHealthCheckRequestValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LeaderHealthCheckRequestValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LeaderHealthCheckRequestValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LeaderHealthCheckRequestValidationError) ErrorName() string { + return "LeaderHealthCheckRequestValidationError" +} + +// Error satisfies the builtin error interface +func (e LeaderHealthCheckRequestValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLeaderHealthCheckRequest.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LeaderHealthCheckRequestValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LeaderHealthCheckRequestValidationError{} + +// Validate checks the field values on LeaderHealthCheckResponse with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *LeaderHealthCheckResponse) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LeaderHealthCheckResponse with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// LeaderHealthCheckResponseMultiError, or nil if none found. +func (m *LeaderHealthCheckResponse) ValidateAll() error { + return m.validate(true) +} + +func (m *LeaderHealthCheckResponse) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return LeaderHealthCheckResponseMultiError(errors) + } + + return nil +} + +// LeaderHealthCheckResponseMultiError is an error wrapping multiple validation +// errors returned by LeaderHealthCheckResponse.ValidateAll() if the +// designated constraints aren't met. +type LeaderHealthCheckResponseMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LeaderHealthCheckResponseMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LeaderHealthCheckResponseMultiError) AllErrors() []error { return m } + +// LeaderHealthCheckResponseValidationError is the validation error returned by +// LeaderHealthCheckResponse.Validate if the designated constraints aren't met. +type LeaderHealthCheckResponseValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LeaderHealthCheckResponseValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LeaderHealthCheckResponseValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LeaderHealthCheckResponseValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LeaderHealthCheckResponseValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LeaderHealthCheckResponseValidationError) ErrorName() string { + return "LeaderHealthCheckResponseValidationError" +} + +// Error satisfies the builtin error interface +func (e LeaderHealthCheckResponseValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLeaderHealthCheckResponse.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LeaderHealthCheckResponseValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LeaderHealthCheckResponseValidationError{} + // Validate checks the field values on CheckUpdatesRequest with the rules // defined in the proto definition for this message. If any rules are // violated, the first error encountered is returned, or nil if there are no violations. diff --git a/api/serverpb/server.proto b/api/serverpb/server.proto index 616b91fd43..4b254b34aa 100644 --- a/api/serverpb/server.proto +++ b/api/serverpb/server.proto @@ -53,6 +53,12 @@ message ReadinessResponse { // This probe is available without authentication, so it should not contain any data. } +message LeaderHealthCheckRequest {} + +message LeaderHealthCheckResponse { + // This probe is available without authentication, so it should not contain any data. +} + message CheckUpdatesRequest { // If false, cached information may be returned. bool force = 1; @@ -296,6 +302,17 @@ service Server { description: "Returns an error when Server components being restarted are not ready yet. Use this API for checking the health of Docker containers and for probing Kubernetes readiness." }; } + // LeaderHealthCheck checks if the instance is the leader in a cluster. + rpc LeaderHealthCheck(LeaderHealthCheckRequest) returns (LeaderHealthCheckResponse) { + option (google.api.http) = { + post: "/v1/leaderHealthCheck" + body: "*" + }; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { + summary: "Check Leadership" + description: "Checks if the instance is the leader in a cluster. Returns an error if the instance isn't the leader." + }; + } // CheckUpdates checks for available PMM Server updates. rpc CheckUpdates(CheckUpdatesRequest) returns (CheckUpdatesResponse) { option (google.api.http) = { diff --git a/api/serverpb/server_grpc.pb.go b/api/serverpb/server_grpc.pb.go index 63197e8537..de69a805f0 100644 --- a/api/serverpb/server_grpc.pb.go +++ b/api/serverpb/server_grpc.pb.go @@ -22,6 +22,7 @@ const _ = grpc.SupportPackageIsVersion7 const ( Server_Version_FullMethodName = "/server.Server/Version" Server_Readiness_FullMethodName = "/server.Server/Readiness" + Server_LeaderHealthCheck_FullMethodName = "/server.Server/LeaderHealthCheck" Server_CheckUpdates_FullMethodName = "/server.Server/CheckUpdates" Server_StartUpdate_FullMethodName = "/server.Server/StartUpdate" Server_UpdateStatus_FullMethodName = "/server.Server/UpdateStatus" @@ -40,6 +41,8 @@ type ServerClient interface { // Readiness returns an error when Server components being restarted are not ready yet. // Use this API for checking the health of Docker containers and for probing Kubernetes readiness. Readiness(ctx context.Context, in *ReadinessRequest, opts ...grpc.CallOption) (*ReadinessResponse, error) + // LeaderHealthCheck checks if the instance is the leader in a cluster. + LeaderHealthCheck(ctx context.Context, in *LeaderHealthCheckRequest, opts ...grpc.CallOption) (*LeaderHealthCheckResponse, error) // CheckUpdates checks for available PMM Server updates. CheckUpdates(ctx context.Context, in *CheckUpdatesRequest, opts ...grpc.CallOption) (*CheckUpdatesResponse, error) // StartUpdate starts PMM Server update. @@ -82,6 +85,15 @@ func (c *serverClient) Readiness(ctx context.Context, in *ReadinessRequest, opts return out, nil } +func (c *serverClient) LeaderHealthCheck(ctx context.Context, in *LeaderHealthCheckRequest, opts ...grpc.CallOption) (*LeaderHealthCheckResponse, error) { + out := new(LeaderHealthCheckResponse) + err := c.cc.Invoke(ctx, Server_LeaderHealthCheck_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *serverClient) CheckUpdates(ctx context.Context, in *CheckUpdatesRequest, opts ...grpc.CallOption) (*CheckUpdatesResponse, error) { out := new(CheckUpdatesResponse) err := c.cc.Invoke(ctx, Server_CheckUpdates_FullMethodName, in, out, opts...) @@ -154,6 +166,8 @@ type ServerServer interface { // Readiness returns an error when Server components being restarted are not ready yet. // Use this API for checking the health of Docker containers and for probing Kubernetes readiness. Readiness(context.Context, *ReadinessRequest) (*ReadinessResponse, error) + // LeaderHealthCheck checks if the instance is the leader in a cluster. + LeaderHealthCheck(context.Context, *LeaderHealthCheckRequest) (*LeaderHealthCheckResponse, error) // CheckUpdates checks for available PMM Server updates. CheckUpdates(context.Context, *CheckUpdatesRequest) (*CheckUpdatesResponse, error) // StartUpdate starts PMM Server update. @@ -182,6 +196,10 @@ func (UnimplementedServerServer) Readiness(context.Context, *ReadinessRequest) ( return nil, status.Errorf(codes.Unimplemented, "method Readiness not implemented") } +func (UnimplementedServerServer) LeaderHealthCheck(context.Context, *LeaderHealthCheckRequest) (*LeaderHealthCheckResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LeaderHealthCheck not implemented") +} + func (UnimplementedServerServer) CheckUpdates(context.Context, *CheckUpdatesRequest) (*CheckUpdatesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method CheckUpdates not implemented") } @@ -258,6 +276,24 @@ func _Server_Readiness_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } +func _Server_LeaderHealthCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LeaderHealthCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServerServer).LeaderHealthCheck(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Server_LeaderHealthCheck_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServerServer).LeaderHealthCheck(ctx, req.(*LeaderHealthCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Server_CheckUpdates_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CheckUpdatesRequest) if err := dec(in); err != nil { @@ -399,6 +435,10 @@ var Server_ServiceDesc = grpc.ServiceDesc{ MethodName: "Readiness", Handler: _Server_Readiness_Handler, }, + { + MethodName: "LeaderHealthCheck", + Handler: _Server_LeaderHealthCheck_Handler, + }, { MethodName: "CheckUpdates", Handler: _Server_CheckUpdates_Handler, diff --git a/api/swagger/swagger-dev.json b/api/swagger/swagger-dev.json index 926ca7fa3f..02a24796a5 100644 --- a/api/swagger/swagger-dev.json +++ b/api/swagger/swagger-dev.json @@ -16865,6 +16865,68 @@ } } }, + "/v1/leaderHealthCheck": { + "post": { + "description": "Checks if the instance is the leader in a cluster. Returns an error if the instance isn't the leader.", + "tags": [ + "Server" + ], + "summary": "Check Leadership", + "operationId": "LeaderHealthCheck", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "object" + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "description": "This probe is available without authentication, so it should not contain any data.", + "type": "object" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32", + "x-order": 0 + }, + "message": { + "type": "string", + "x-order": 1 + }, + "details": { + "type": "array", + "items": { + "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\nExample 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\nExample 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", + "type": "object", + "properties": { + "@type": { + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", + "type": "string", + "x-order": 0 + } + }, + "additionalProperties": false + }, + "x-order": 2 + } + } + } + } + } + } + }, "/v1/management/Actions/Cancel": { "post": { "description": "Stops an Action.", diff --git a/api/swagger/swagger.json b/api/swagger/swagger.json index 24f7b6c588..cfa1dddb91 100644 --- a/api/swagger/swagger.json +++ b/api/swagger/swagger.json @@ -14098,6 +14098,68 @@ } } }, + "/v1/leaderHealthCheck": { + "post": { + "description": "Checks if the instance is the leader in a cluster. Returns an error if the instance isn't the leader.", + "tags": [ + "Server" + ], + "summary": "Check Leadership", + "operationId": "LeaderHealthCheck", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "object" + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "description": "This probe is available without authentication, so it should not contain any data.", + "type": "object" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32", + "x-order": 0 + }, + "message": { + "type": "string", + "x-order": 1 + }, + "details": { + "type": "array", + "items": { + "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\nExample 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\nExample 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", + "type": "object", + "properties": { + "@type": { + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", + "type": "string", + "x-order": 0 + } + }, + "additionalProperties": false + }, + "x-order": 2 + } + } + } + } + } + } + }, "/v1/management/Actions/Cancel": { "post": { "description": "Stops an Action.", diff --git a/docker-compose.yml b/docker-compose.yml index 159d1fb415..03e842f9f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,7 +171,16 @@ services: - ./managed/testdata/pg/conf/:/conf/ - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ - pmm-managed-server-active: + haproxy: + image: haproxy:latest + volumes: + - ./managed/testdata/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + - ./mydomain.pem:/etc/ssl/private/mydomain.pem + ports: + - 443:443 + - 443:443 + + pmm-server-active: profiles: - pmm-ha depends_on: @@ -179,8 +188,8 @@ services: - victoriametrics - pg image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} - container_name: pmm-server - hostname: pmm-server + container_name: pmm-server-active + hostname: pmm-server-active networks: ha: ipv4_address: 172.20.0.5 @@ -218,11 +227,12 @@ services: - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - PERCONA_TEST_HA_ENABLE=1 - PERCONA_TEST_HA_BOOTSTRAP=1 - - PERCONA_TEST_HA_NODE_ID=pmm-managed-server + - PERCONA_TEST_HA_NODE_ID=pmm-server-active - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 - PERCONA_TEST_HA_GOSSIP_PORT=9096 + - PERCONA_TEST_HA_GF_PORT=9095 - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.5:9095 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9095,pmm-managed-server-passive:9095 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway @@ -255,7 +265,7 @@ services: - root-cache:/root/.cache - ./managed/testdata/pg/certs/:/certs/ - pmm-managed-server-passive: + pmm-server-passive: profiles: - pmm-ha depends_on: @@ -263,8 +273,8 @@ services: - pg - victoriametrics image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} - container_name: pmm-managed-server-passive - hostname: pmm-managed-server-passive + container_name: pmm-server-passive + hostname: pmm-server-passive networks: ha: ipv4_address: 172.20.0.6 @@ -299,14 +309,13 @@ services: - GO_VERSION=1.20 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - - PERCONA_TEST_HA_PASSIVE=true - PERCONA_TEST_HA_ENABLE=1 - - PERCONA_TEST_HA_NODE_ID=pmm-managed-server-passive + - PERCONA_TEST_HA_NODE_ID=pmm-server-passive - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - - PERCONA_TEST_HA_PEERS=pmm-managed-server,pmm-managed-server-passive + - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive - PERCONA_TEST_HA_GOSSIP_PORT=9096 - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.6:9095 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-managed-server:9095,pmm-managed-server-passive:9095 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 92280c6ea8..aeaea25fb9 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -245,7 +245,7 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { ) if l.Logger.GetLevel() >= logrus.DebugLevel { - l.Debug("Reflection and channelz are enabled.") + l.Debug("Reflection and channels are enabled.") reflection.Register(gRPCServer) channelz.RegisterChannelzServiceToServer(gRPCServer) @@ -880,6 +880,12 @@ func main() { //nolint:cyclop,maintidx highavailability.NewChannel() agentsRegistry := agents.NewRegistry(db, vmParams) + + //TODO remove once PMM cluster will be Active-Active + ha.AddLeaderService(highavailability.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() error { + return agentsRegistry.KickAll() + })) + pbmPITRService := backup.NewPBMPITRService() backupRemovalService := backup.NewRemovalService(db, pbmPITRService) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 49a3e27e11..20ccb0784c 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -440,6 +440,13 @@ func (r *Registry) Collect(ch chan<- prom.Metric) { r.mClockDrift.Collect(ch) } +func (r *Registry) KickAll() error { + for _, agentInfo := range r.agents { + r.Kick(context.Background(), agentInfo.ID()) + } + return nil +} + // check interfaces. var ( _ prom.Collector = (*Registry)(nil) diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index 48e26bf29c..d3ade46254 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -57,6 +57,7 @@ type Service struct { l *logrus.Entry wg *sync.WaitGroup + rw sync.RWMutex raftNode *raft.Raft memberlist *memberlist.Memberlist tm *transport.Manager @@ -138,11 +139,16 @@ func (s *Service) Run(ctx context.Context) error { } // Create a new Raft node + s.rw.Lock() s.raftNode, err = raft.NewRaft(raftConfig, s, raft.NewInmemStore(), raft.NewInmemStore(), raft.NewInmemSnapshotStore(), raftTrans) + s.rw.Unlock() if err != nil { return err } - defer s.raftNode.Shutdown().Error() + defer func() { + err := s.raftNode.Shutdown().Error() + s.l.Errorln(err) + }() // Create the memberlist configuration memberlistConfig := memberlist.DefaultWANConfig() @@ -208,12 +214,14 @@ func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { continue } node := event.Node + s.rw.RLock() switch event.Event { case memberlist.NodeJoin: s.raftNode.AddVoter(raft.ServerID(node.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", node.Addr.String(), s.params.RaftPort)), s.raftNode.AppliedIndex(), 10*time.Second).Error() case memberlist.NodeLeave: s.raftNode.RemoveServer(raft.ServerID(node.Name), 0, 10*time.Second).Error() } + s.rw.RUnlock() case <-ctx.Done(): return } @@ -222,8 +230,11 @@ func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { func (s *Service) runLeaderObserver(ctx context.Context) { for { + s.rw.RLock() + node := s.raftNode + s.rw.RUnlock() select { - case isLeader := <-s.raftNode.LeaderCh(): + case isLeader := <-node.LeaderCh(): if isLeader { s.services.StartAllServices(ctx) // This node is the leader @@ -233,7 +244,7 @@ func (s *Service) runLeaderObserver(ctx context.Context) { if peer.Name == s.params.NodeID { continue } - if err := s.raftNode.AddVoter(raft.ServerID(peer.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", peer.Addr.String(), s.params.RaftPort)), s.raftNode.AppliedIndex(), 10*time.Second).Error(); err != nil { + if err := node.AddVoter(raft.ServerID(peer.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", peer.Addr.String(), s.params.RaftPort)), node.AppliedIndex(), 10*time.Second).Error(); err != nil { s.l.Warnf("Failed to add Raft member: %v", err) } } @@ -256,6 +267,8 @@ func (s *Service) AddLeaderService(leaderService LeaderService) { func (s *Service) BroadcastMessage(message []byte) { if s.params.Enabled { + s.rw.RLock() + defer s.rw.RUnlock() s.raftNode.Apply(message, 3*time.Second) } else { s.receivedMessages <- message @@ -263,6 +276,8 @@ func (s *Service) BroadcastMessage(message []byte) { } func (s *Service) IsLeader() bool { + s.rw.RLock() + defer s.rw.RUnlock() return (s.raftNode != nil && s.raftNode.State() == raft.Leader) || !s.params.Enabled } diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 6da2f77a93..4db4c1bbeb 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -7,10 +7,36 @@ import ( type LeaderService interface { Start(ctx context.Context) error - Stop() + Stop() error ID() string } +type StandardService struct { + id string + + startFunc func(context.Context) error + stopFunc func() error +} + +func NewStandardService(id string, startFunc func(context.Context) error, stopFunc func() error) *StandardService { + return &StandardService{ + id: id, + startFunc: startFunc, + stopFunc: stopFunc, + } +} + +func (s *StandardService) ID() string { + return s.id +} + +func (s *StandardService) Start(ctx context.Context) error { + return s.startFunc(ctx) +} +func (s *StandardService) Stop() error { + return s.stopFunc() +} + type ContextService struct { id string @@ -21,6 +47,7 @@ type ContextService struct { func NewContextService(id string, startFunc func(context.Context) error) *ContextService { return &ContextService{ + id: id, startFunc: startFunc, } } @@ -34,8 +61,9 @@ func (s *ContextService) Start(ctx context.Context) error { return s.startFunc(ctx) } -func (s *ContextService) Stop() { +func (s *ContextService) Stop() error { s.cancel() + return nil } type RunOnceService struct { @@ -47,6 +75,7 @@ type RunOnceService struct { func NewRunOnceService(id string, startFunc func(context.Context) error) *RunOnceService { return &RunOnceService{ + id: id, startFunc: startFunc, } } @@ -56,9 +85,12 @@ func (s *RunOnceService) ID() string { } func (s *RunOnceService) Start(ctx context.Context) error { + var err error s.o.Do(func() { - s.startFunc(ctx) + err = s.startFunc(ctx) }) + return err +} +func (s *RunOnceService) Stop() error { return nil } -func (s *RunOnceService) Stop() {} diff --git a/managed/services/highavailability/memberlist.go b/managed/services/highavailability/memberlist.go deleted file mode 100644 index b04c4eb320..0000000000 --- a/managed/services/highavailability/memberlist.go +++ /dev/null @@ -1,22 +0,0 @@ -package highavailability - -import ( - "github.com/hashicorp/memberlist" - "github.com/sirupsen/logrus" -) - -type eventDelegate struct { - nodeCh chan *memberlist.Node -} - -func (e *eventDelegate) NotifyJoin(node *memberlist.Node) { - e.nodeCh <- node -} - -func (e *eventDelegate) NotifyLeave(node *memberlist.Node) { - e.nodeCh <- node -} - -func (e *eventDelegate) NotifyUpdate(node *memberlist.Node) { - logrus.Printf("NotifyUpdate: %v", node) -} diff --git a/managed/services/server/deps.go b/managed/services/server/deps.go index 3eef78b54a..57a95dceea 100644 --- a/managed/services/server/deps.go +++ b/managed/services/server/deps.go @@ -127,8 +127,14 @@ type emailer interface { Send(ctx context.Context, settings *models.EmailAlertingSettings, emailTo string) error } -// rulesService is a subset of methods of ia.TemplatesService used by this package. +// templatesService is a subset of methods of ia.TemplatesService used by this package. // We use it instead of real type for testing and to avoid dependency cycle. type templatesService interface { CollectTemplates(ctx context.Context) } + +// haService is a subset of methods of highavailability.Service used by this package. +// We use it instead of real type for testing and to avoid dependency cycle. +type haService interface { + IsLeader() bool +} diff --git a/managed/services/server/server.go b/managed/services/server/server.go index 258fb7647f..03327a299f 100644 --- a/managed/services/server/server.go +++ b/managed/services/server/server.go @@ -64,6 +64,7 @@ type Server struct { rulesService rulesService dbaasInitializer dbaasInitializer emailer emailer + haService haService l *logrus.Entry @@ -104,6 +105,7 @@ type Params struct { RulesService rulesService DBaaSInitializer dbaasInitializer Emailer emailer + HAService haService } // NewServer returns new server for Server service. @@ -240,6 +242,16 @@ func (s *Server) Readiness(ctx context.Context, req *serverpb.ReadinessRequest) return &serverpb.ReadinessResponse{}, nil } +// LeaderHealthCheck checks if the instance is the leader in a cluster. +// Returns an error if the instance isn't the leader. +// It's used for HA purpose +func (s *Server) LeaderHealthCheck(ctx context.Context, req *serverpb.LeaderHealthCheckRequest) (*serverpb.LeaderHealthCheckResponse, error) { + if s.haService.IsLeader() { + return &serverpb.LeaderHealthCheckResponse{}, nil + } + return nil, status.Error(codes.FailedPrecondition, "this PMM Server isn't the leader") +} + func (s *Server) onlyInstalledVersionResponse(ctx context.Context) *serverpb.CheckUpdatesResponse { v := s.supervisord.InstalledPMMVersion(ctx) r := &serverpb.CheckUpdatesResponse{ diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index f46bcb2e1c..490ea4423b 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -579,7 +579,7 @@ func (s *Service) UpdateConfiguration(settings *models.Settings, ssoDetails *mod } for _, tmpl := range templates.Templates() { - if tmpl.Name() == "" { + if tmpl.Name() == "" || (tmpl.Name() == "victoriametrics" && s.vmParams.ExternalVM()) { continue } diff --git a/managed/testdata/haproxy/haproxy.cfg b/managed/testdata/haproxy/haproxy.cfg new file mode 100644 index 0000000000..4606d3e296 --- /dev/null +++ b/managed/testdata/haproxy/haproxy.cfg @@ -0,0 +1,33 @@ +global + log /dev/log local0 + log /dev/log local1 notice + daemon + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 50000 + timeout server 50000 + +frontend http_front + bind *:80 + default_backend http_back + +frontend https_front + bind *:443 ssl crt /etc/ssl/private/mydomain.pem + default_backend https_back + +backend http_back + option httpchk POST /v1/Leader/HealthCheck HTTP/1.1\r\nHost:\ www + http-check expect status 200 + server pmm-server-active-http pmm-server-active:80 check + server pmm-server-passive-http pmm-server-passive:80 check + +backend https_back + option httpchk POST /v1/Leader/HealthCheck HTTP/1.1\r\nHost:\ www + http-check expect status 200 + server pmm-server-active-https pmm-server-active:443 ssl verify none + server pmm-server-passive-https pmm-server-passive:443 ssl verify none From fe61c4488c340719c1364099e6b7ba00331feb80 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 6 Jul 2023 15:56:19 +0300 Subject: [PATCH 35/77] PMM-12078 Fix docker-compose config --- docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 03e842f9f5..494e81cda5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -177,7 +177,7 @@ services: - ./managed/testdata/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg - ./mydomain.pem:/etc/ssl/private/mydomain.pem ports: - - 443:443 + - 80:80 - 443:443 pmm-server-active: @@ -252,8 +252,8 @@ services: memlock: 67108864 ports: - - ${PMM_PORT_HTTP:-80}:80 - - ${PMM_PORT_HTTPS:-443}:443 + - ${PMM_PORT_HTTP:-8081}:80 + - ${PMM_PORT_HTTPS:-8441}:443 # For headless delve - ${PMM_PORT_DELVE:-2345}:2345 volumes: @@ -335,8 +335,8 @@ services: memlock: 67108864 ports: - - ${PMM_PORT_HTTP:-8080}:80 - - ${PMM_PORT_HTTPS:-4433}:443 + - ${PMM_PORT_HTTP:-8082}:80 + - ${PMM_PORT_HTTPS:-8432}:443 # For headless delve - ${PMM_PORT_DELVE:-12345}:2345 volumes: From 464f44120f0c21100dec77668653bf09d46c71e5 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 6 Jul 2023 23:32:17 +0300 Subject: [PATCH 36/77] PMM-12078 HAProxy configuration --- docker-compose.yml | 94 ++++++++++++++++++- managed/cmd/pmm-managed/main.go | 1 + .../services/agents/channel/channel_test.go | 2 +- managed/services/agents/registry.go | 2 +- managed/services/grafana/auth_server.go | 5 +- managed/services/server/server.go | 1 + managed/testdata/haproxy/haproxy.cfg | 10 +- 7 files changed, 107 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 494e81cda5..d179412099 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -172,10 +172,17 @@ services: - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ haproxy: + profiles: + - pmm-ha image: haproxy:latest + container_name: haproxy + hostname: haproxy + networks: + ha: + ipv4_address: 172.20.0.10 volumes: - ./managed/testdata/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg - - ./mydomain.pem:/etc/ssl/private/mydomain.pem + - ./managed/testdata/haproxy/localhost.pem:/etc/ssl/private/localhost.pem ports: - 80:80 - 443:443 @@ -314,6 +321,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive - PERCONA_TEST_HA_GOSSIP_PORT=9096 + - PERCONA_TEST_HA_GF_PORT=9095 - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.6:9095 - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 @@ -348,6 +356,90 @@ services: - root-cache:/root/.cache - ./managed/testdata/pg/certs/:/certs/ + pmm-server-passive-2: + profiles: + - pmm-ha + depends_on: + - ch + - pg + - victoriametrics + image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} + container_name: pmm-server-passive-2 + hostname: pmm-server-passive-2 + networks: + ha: + ipv4_address: 172.20.0.11 + environment: + - PMM_RELEASE_PATH=/root/go/bin + - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} + - ENABLE_DBAAS=${ENABLE_DBAAS:-0} + - AWS_ACCESS_KEY=${AWS_ACCESS_KEY} + - AWS_SECRET_KEY=${AWS_SECRET_KEY} + # - PERCONA_TEST_PLATFORM_ADDRESS=https://check.localhost + # - PERCONA_TEST_PLATFORM_INSECURE=1 + # - PERCONA_TEST_PLATFORM_PUBLIC_KEY= + # - PERCONA_TEST_TELEMETRY_INTERVAL=10s + # - PERCONA_TEST_TELEMETRY_RETRY_BACKOFF=10s + # - PERCONA_TEST_TELEMETRY_DISABLE_START_DELAY=1 + - PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000 + - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm + - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 + - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 + - PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 + - PERCONA_TEST_POSTGRES_ADDR=pg:5432 + - PERCONA_TEST_POSTGRES_USERNAME=pmm-managed + - PERCONA_TEST_POSTGRES_DBPASSWORD=pmm-managed + - PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 + # - PERCONA_TEST_POSTGRES_SSL_MODE=require + # - PERCONA_TEST_POSTGRES_SSL_CA_PATH=/certs/root.crt + # - PERCONA_TEST_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key + # - PERCONA_TEST_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt + - GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana + # - GF_DATABASE_SSL_MODE=require + # - PMM_DEBUG=1 + - GO_VERSION=1.20 + - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} + - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest + - PERCONA_TEST_HA_ENABLE=1 + - PERCONA_TEST_HA_NODE_ID=pmm-server-passive + - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11 + - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive + - PERCONA_TEST_HA_GOSSIP_PORT=9096 + - PERCONA_TEST_HA_GF_PORT=9095 + - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.11:9095 + - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 + + extra_hosts: + - host.docker.internal:host-gateway + # - portal.localhost:${PORTAL_HOST:-host-gateway} + # - check.localhost:${PORTAL_CHECK_HOST:-host-gateway} + # - pmm.localhost:${PORTAL_PMM_HOST:-host-gateway} + # - check-dev.percona.com:${PORTAL_PMM_HOST:-host-gateway} + + # for delve + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + + # see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do + ulimits: + memlock: 67108864 + + ports: + - ${PMM_PORT_HTTP:-8083}:80 + - ${PMM_PORT_HTTPS:-8433}:443 + # For headless delve +# - ${PMM_PORT_DELVE:-12345}:2345 + volumes: + - ./:/root/go/src/github.com/percona/pmm + # - "../grafana/public:/usr/share/grafana/public" + - ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # change Makefile in devcontainer + # caching + - go-modules:/root/go/pkg/mod + - root-cache:/root/.cache + - ./managed/testdata/pg/certs/:/certs/ + volumes: go-modules: vmdata: {} diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index aeaea25fb9..ae37bfb583 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -1013,6 +1013,7 @@ func main() { //nolint:cyclop,maintidx RulesService: rulesService, DBaaSInitializer: dbaasInitializer, Emailer: emailer, + HAService: ha, } server, err := server.NewServer(serverParams) diff --git a/managed/services/agents/channel/channel_test.go b/managed/services/agents/channel/channel_test.go index e43b9f1677..9b70ff94e0 100644 --- a/managed/services/agents/channel/channel_test.go +++ b/managed/services/agents/channel/channel_test.go @@ -70,7 +70,7 @@ func setup(t *testing.T, connect func(*Channel) error, expected ...error) (agent agentpb.RegisterAgentServer(server, &testServer{ connectFunc: func(stream agentpb.Agent_ConnectServer) error { - channel = New(nil, stream) + channel = New(stream.Context(), stream) return connect(channel) }, }) diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 20ccb0784c..d7de332438 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -385,7 +385,7 @@ func removeVMAgentFromPMMAgent(q *reform.Querier, pmmAgentID string) error { // Kick unregisters and forcefully disconnects pmm-agent with given ID. func (r *Registry) Kick(ctx context.Context, pmmAgentID string) { - agent := r.unregister(pmmAgentID, "kickChan") + agent := r.unregister(pmmAgentID, "kick") if agent == nil { return } diff --git a/managed/services/grafana/auth_server.go b/managed/services/grafana/auth_server.go index 5f39ba3d8c..6d87ce56bc 100644 --- a/managed/services/grafana/auth_server.go +++ b/managed/services/grafana/auth_server.go @@ -73,8 +73,9 @@ var rules = map[string]role{ "/v1/user": viewer, // must be available without authentication for health checking - "/v1/readyz": none, - "/ping": none, // PMM 1.x variant + "/v1/readyz": none, + "/v1/leaderHealthCheck": none, + "/ping": none, // PMM 1.x variant // must not be available without authentication as it can leak data "/v1/version": viewer, diff --git a/managed/services/server/server.go b/managed/services/server/server.go index 03327a299f..5cbd011335 100644 --- a/managed/services/server/server.go +++ b/managed/services/server/server.go @@ -132,6 +132,7 @@ func NewServer(params *Params) (*Server, error) { rulesService: params.RulesService, dbaasInitializer: params.DBaaSInitializer, emailer: params.Emailer, + haService: params.HAService, l: logrus.WithField("component", "server"), pmmUpdateAuthFile: path, envSettings: &models.ChangeSettingsParams{}, diff --git a/managed/testdata/haproxy/haproxy.cfg b/managed/testdata/haproxy/haproxy.cfg index 4606d3e296..e7bc5c0c2c 100644 --- a/managed/testdata/haproxy/haproxy.cfg +++ b/managed/testdata/haproxy/haproxy.cfg @@ -17,17 +17,21 @@ frontend http_front default_backend http_back frontend https_front - bind *:443 ssl crt /etc/ssl/private/mydomain.pem + bind *:443 ssl crt /etc/ssl/private/localhost.pem default_backend https_back backend http_back - option httpchk POST /v1/Leader/HealthCheck HTTP/1.1\r\nHost:\ www + option httpchk + http-check send meth POST uri /v1/leaderHealthCheck ver HTTP/1.1 hdr Host www http-check expect status 200 server pmm-server-active-http pmm-server-active:80 check server pmm-server-passive-http pmm-server-passive:80 check + server pmm-server-passive-2-http pmm-server-passive-2:80 check backend https_back - option httpchk POST /v1/Leader/HealthCheck HTTP/1.1\r\nHost:\ www + option httpchk + http-check send meth POST uri /v1/leaderHealthCheck ver HTTP/1.1 hdr Host www http-check expect status 200 server pmm-server-active-https pmm-server-active:443 ssl verify none server pmm-server-passive-https pmm-server-passive:443 ssl verify none + server pmm-server-passive-2-https pmm-server-passive-2:443 ssl verify none From 28de62483877f1633bae6489595a5679ed93d4b7 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 6 Jul 2023 23:47:10 +0300 Subject: [PATCH 37/77] PMM-12078 Format --- managed/cmd/pmm-managed/main.go | 2 +- managed/services/highavailability/leaderservice.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index ae37bfb583..4ec1b060d7 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -881,7 +881,7 @@ func main() { //nolint:cyclop,maintidx highavailability.NewChannel() agentsRegistry := agents.NewRegistry(db, vmParams) - //TODO remove once PMM cluster will be Active-Active + // TODO remove once PMM cluster will be Active-Active ha.AddLeaderService(highavailability.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() error { return agentsRegistry.KickAll() })) diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 4db4c1bbeb..bc0d28debe 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -33,6 +33,7 @@ func (s *StandardService) ID() string { func (s *StandardService) Start(ctx context.Context) error { return s.startFunc(ctx) } + func (s *StandardService) Stop() error { return s.stopFunc() } @@ -91,6 +92,7 @@ func (s *RunOnceService) Start(ctx context.Context) error { }) return err } + func (s *RunOnceService) Stop() error { return nil } From 446f32e865c89cca557305a3c880bb040cc783f9 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 6 Jul 2023 23:47:29 +0300 Subject: [PATCH 38/77] PMM-12078 haproxy certificates --- managed/testdata/haproxy/localhost.crt | 20 +++++++++++ managed/testdata/haproxy/localhost.csr | 17 ++++++++++ managed/testdata/haproxy/localhost.key | 27 +++++++++++++++ managed/testdata/haproxy/localhost.pem | 47 ++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 managed/testdata/haproxy/localhost.crt create mode 100644 managed/testdata/haproxy/localhost.csr create mode 100644 managed/testdata/haproxy/localhost.key create mode 100644 managed/testdata/haproxy/localhost.pem diff --git a/managed/testdata/haproxy/localhost.crt b/managed/testdata/haproxy/localhost.crt new file mode 100644 index 0000000000..1b601b09c1 --- /dev/null +++ b/managed/testdata/haproxy/localhost.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPjCCAiYCCQC8Y6/8ayWo6DANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJU +UjEQMA4GA1UECgwHUGVyY29uYTESMBAGA1UEAwwJbG9jYWxob3N0MSwwKgYJKoZI +hvcNAQkBFh1udXJsYW4ubW9sZG9tdXJvdkBwZXJjb25hLmNvbTAeFw0yMzA3MDYy +MDA2NDNaFw0yNDA3MDUyMDA2NDNaMGExCzAJBgNVBAYTAlRSMRAwDgYDVQQKDAdQ +ZXJjb25hMRIwEAYDVQQDDAlsb2NhbGhvc3QxLDAqBgkqhkiG9w0BCQEWHW51cmxh +bi5tb2xkb211cm92QHBlcmNvbmEuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAmp+Xbi1C79S2l7IawE0dIBaKlx4vO/baQBDi+SXuhUonw8dPTqvD +DZy96/irc7SudvsJdcpcFn9tGfASDez56uZqt/39wsA+uUsym9yV39gcZPRzeoiV +nxDny1dcNJSqlGkcDp7BqXaj2/e6bK5RW3cpUnnRk4M7weDsdblBJLYPAIqTGMmZ +Chf/iKAz6i9E+FRuXi3rhFJKUEGv+5nh7Pjd/9BxmVyjl9f9SWhe+AkimOs+CYrh +lcGAHJ6XKq5KAMB43elZxSXgv6X5eTVhQsqf7X0e8n0OrXKqP+fqrQMPnkOsVvEj +q90mcG1mvvPrMoXN0XN1dfJEhyRB/hwi5QIDAQABMA0GCSqGSIb3DQEBCwUAA4IB +AQB9BKnOT2KiKTdnydorEpuMgzD1RZ9bfX8mGiucjPh5lcjO6L9haUbFN/6PupZP +x1WRKNwYm+R2vP/Q1tlkOwVDfBtycAhyx5NMRyHkmG90ap/hJUThF2D9Q+5A81Ma +tnpg5jbxPEBeMHujGZmDEiBNPHc9oP7HNuPXMzZWxAOjRAg2WtaqjyJi8ExnCP1t +4ELKdjojtSefhxQzZmdHNBKWa0kUJhDGfhvSo0//H9n8Q7VMmtVS94Klu/H+IG88 +EYmEzgkmty2eie+Jiv+S2WGDEUuopAReifGscFI3tYvNBbeU4GbtUCXKoNX8N6hO +1POaPPj84EK2ncLJXffk0XYq +-----END CERTIFICATE----- diff --git a/managed/testdata/haproxy/localhost.csr b/managed/testdata/haproxy/localhost.csr new file mode 100644 index 0000000000..84d654a289 --- /dev/null +++ b/managed/testdata/haproxy/localhost.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpjCCAY4CAQAwYTELMAkGA1UEBhMCVFIxEDAOBgNVBAoMB1BlcmNvbmExEjAQ +BgNVBAMMCWxvY2FsaG9zdDEsMCoGCSqGSIb3DQEJARYdbnVybGFuLm1vbGRvbXVy +b3ZAcGVyY29uYS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCa +n5duLULv1LaXshrATR0gFoqXHi879tpAEOL5Je6FSifDx09Oq8MNnL3r+KtztK52 ++wl1ylwWf20Z8BIN7Pnq5mq3/f3CwD65SzKb3JXf2Bxk9HN6iJWfEOfLV1w0lKqU +aRwOnsGpdqPb97psrlFbdylSedGTgzvB4Ox1uUEktg8AipMYyZkKF/+IoDPqL0T4 +VG5eLeuEUkpQQa/7meHs+N3/0HGZXKOX1/1JaF74CSKY6z4JiuGVwYAcnpcqrkoA +wHjd6VnFJeC/pfl5NWFCyp/tfR7yfQ6tcqo/5+qtAw+eQ6xW8SOr3SZwbWa+8+sy +hc3Rc3V18kSHJEH+HCLlAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEALWj/APq2 +xfNqyRM4cf8uSpRiIe7OjE9HABFXWowLfFMJ6E337n9TnV/srCNxgdBPZ78pPJLR +EWFRJUtB/cwYqxSauWYg7+x1HtNn2yQnyKX1Fep8LBREs2ykXPQAmiTaUrxja0+W +D880Ck8uy+C8HKF/cBQA3ZCrdkrV9Q6829WG3FNtRdIu72SDZb9opxvufiOxCRgX +6E1CiZL4fTcgUXVcR9MxJfSNj+HgsO3mU5DiyvbsOpXxCfWDy2O0/CmonfhyDL8o +2EF870bTHMJ4sxMOIB9ZFj4TFwoVUnl08G+XEx2mFR1Hb2ooUDUO5LXt80lftJR0 +qBGJW/RYyCRrPA== +-----END CERTIFICATE REQUEST----- diff --git a/managed/testdata/haproxy/localhost.key b/managed/testdata/haproxy/localhost.key new file mode 100644 index 0000000000..81e8700b34 --- /dev/null +++ b/managed/testdata/haproxy/localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAmp+Xbi1C79S2l7IawE0dIBaKlx4vO/baQBDi+SXuhUonw8dP +TqvDDZy96/irc7SudvsJdcpcFn9tGfASDez56uZqt/39wsA+uUsym9yV39gcZPRz +eoiVnxDny1dcNJSqlGkcDp7BqXaj2/e6bK5RW3cpUnnRk4M7weDsdblBJLYPAIqT +GMmZChf/iKAz6i9E+FRuXi3rhFJKUEGv+5nh7Pjd/9BxmVyjl9f9SWhe+AkimOs+ +CYrhlcGAHJ6XKq5KAMB43elZxSXgv6X5eTVhQsqf7X0e8n0OrXKqP+fqrQMPnkOs +VvEjq90mcG1mvvPrMoXN0XN1dfJEhyRB/hwi5QIDAQABAoIBAEk7zT0hstJkrRas +BH+QBntsMbfhU/3SrQwq81WN4aq/tJXFkIpyT6/izRE2df4XVYqE27YuYe9F6yad +ze9KjhPzjhgW9FmJNCwOsamgkFu0v74RCaC/kB4Go8JrXgCJaUFhhyhliNP6nSFR +87oF1gK8LZYinGCBh4wMO/KGC5SW6X9W6xf7f6RJYSPUH7h80XwnDRvsSnldH/3e +QPciQXP8fPUUW9EP9WERjpWTBZ8YqUG5P2w6ZlCrJ7L+/STMbscD3dYTUXxQUsfh +oxjsi9BQKTPFqiLH7gstCfdoufQJkfWMLSxpn8bBiHf5nhehuRRd9hKwz8+S1jdE +Lv0FYAECgYEAy8h9t5QwW3NSD1WclIFcxQhUeHIWHjLB8KJZi9eYJ4oZ7zWa5duV +0DTs3vJfVvxJ8XLbjnaap7ULWDZETbm2m6d5zJBaEGqLYShzlr15vNp0ZzDZghw+ +9hG/TqAB1jj76RYbI9h4TvzSI3+mbq3nl5Sykz1Envuznz2JiKbefIUCgYEAwj5h +fS0wAa5zjR8Lgs1nRFmg37qH6gn6w40yDDYwxDtM2L1l96ONPROvm6RZkM6hkHge +dKzabpHrh7eViQOgbUO+tyxttOdBspdK1ubi7UZTG6xq4zuzqiITP+BnhSw2mzDR +J276DMNalsmzQdI8v+eIMK0yxqOobcgRK979iuECgYBw/NP/onGBcxpPmEc968/1 +Cx5SvebXjYsMkeeWas5ZNfAVOqKMychx7bZcEwSbpTyWW/myLr6nN/F3UndipRLD +kQMuUect7PUkxJn6PUovVOxvfp1Kz8B1DPgGbx81mNjLrs8Te+WQ3grhVdiAy3l6 +CR9OFg1jHOnF5AfKtcLsRQKBgGL800WtX4eb1XsXVRBliLjGTDt3nYfhag95xwV+ +IEAAUFsruekHShTUEWvpx1MKWj97V1nyNKagaj0Ri3z1gi3sliZW19mW+F4Ax7zY +kNCGRBgYN6hxZk/Paavluhudun4/1HaaEYerjmDFjTp/30GUxky4FuYvxMeda1LG +IsNBAoGBAIeBiXXaB8QhlP4vJu6HT9IDZKRRiYTNjiS1o8CQ2U86FfyO7OVQ+19R +DJCYOc10foiwv+HEsEVXAjux79Z2h2dqqxtVoWPNh6yGs0SDKObmPWOEZsFABeF4 +RFTKlWMq0tvbXmnGwciA3Oy9DbsHp5jTo6qbuewVvE5PL7GTnokF +-----END RSA PRIVATE KEY----- diff --git a/managed/testdata/haproxy/localhost.pem b/managed/testdata/haproxy/localhost.pem new file mode 100644 index 0000000000..e401361481 --- /dev/null +++ b/managed/testdata/haproxy/localhost.pem @@ -0,0 +1,47 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAmp+Xbi1C79S2l7IawE0dIBaKlx4vO/baQBDi+SXuhUonw8dP +TqvDDZy96/irc7SudvsJdcpcFn9tGfASDez56uZqt/39wsA+uUsym9yV39gcZPRz +eoiVnxDny1dcNJSqlGkcDp7BqXaj2/e6bK5RW3cpUnnRk4M7weDsdblBJLYPAIqT +GMmZChf/iKAz6i9E+FRuXi3rhFJKUEGv+5nh7Pjd/9BxmVyjl9f9SWhe+AkimOs+ +CYrhlcGAHJ6XKq5KAMB43elZxSXgv6X5eTVhQsqf7X0e8n0OrXKqP+fqrQMPnkOs +VvEjq90mcG1mvvPrMoXN0XN1dfJEhyRB/hwi5QIDAQABAoIBAEk7zT0hstJkrRas +BH+QBntsMbfhU/3SrQwq81WN4aq/tJXFkIpyT6/izRE2df4XVYqE27YuYe9F6yad +ze9KjhPzjhgW9FmJNCwOsamgkFu0v74RCaC/kB4Go8JrXgCJaUFhhyhliNP6nSFR +87oF1gK8LZYinGCBh4wMO/KGC5SW6X9W6xf7f6RJYSPUH7h80XwnDRvsSnldH/3e +QPciQXP8fPUUW9EP9WERjpWTBZ8YqUG5P2w6ZlCrJ7L+/STMbscD3dYTUXxQUsfh +oxjsi9BQKTPFqiLH7gstCfdoufQJkfWMLSxpn8bBiHf5nhehuRRd9hKwz8+S1jdE +Lv0FYAECgYEAy8h9t5QwW3NSD1WclIFcxQhUeHIWHjLB8KJZi9eYJ4oZ7zWa5duV +0DTs3vJfVvxJ8XLbjnaap7ULWDZETbm2m6d5zJBaEGqLYShzlr15vNp0ZzDZghw+ +9hG/TqAB1jj76RYbI9h4TvzSI3+mbq3nl5Sykz1Envuznz2JiKbefIUCgYEAwj5h +fS0wAa5zjR8Lgs1nRFmg37qH6gn6w40yDDYwxDtM2L1l96ONPROvm6RZkM6hkHge +dKzabpHrh7eViQOgbUO+tyxttOdBspdK1ubi7UZTG6xq4zuzqiITP+BnhSw2mzDR +J276DMNalsmzQdI8v+eIMK0yxqOobcgRK979iuECgYBw/NP/onGBcxpPmEc968/1 +Cx5SvebXjYsMkeeWas5ZNfAVOqKMychx7bZcEwSbpTyWW/myLr6nN/F3UndipRLD +kQMuUect7PUkxJn6PUovVOxvfp1Kz8B1DPgGbx81mNjLrs8Te+WQ3grhVdiAy3l6 +CR9OFg1jHOnF5AfKtcLsRQKBgGL800WtX4eb1XsXVRBliLjGTDt3nYfhag95xwV+ +IEAAUFsruekHShTUEWvpx1MKWj97V1nyNKagaj0Ri3z1gi3sliZW19mW+F4Ax7zY +kNCGRBgYN6hxZk/Paavluhudun4/1HaaEYerjmDFjTp/30GUxky4FuYvxMeda1LG +IsNBAoGBAIeBiXXaB8QhlP4vJu6HT9IDZKRRiYTNjiS1o8CQ2U86FfyO7OVQ+19R +DJCYOc10foiwv+HEsEVXAjux79Z2h2dqqxtVoWPNh6yGs0SDKObmPWOEZsFABeF4 +RFTKlWMq0tvbXmnGwciA3Oy9DbsHp5jTo6qbuewVvE5PL7GTnokF +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDPjCCAiYCCQC8Y6/8ayWo6DANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJU +UjEQMA4GA1UECgwHUGVyY29uYTESMBAGA1UEAwwJbG9jYWxob3N0MSwwKgYJKoZI +hvcNAQkBFh1udXJsYW4ubW9sZG9tdXJvdkBwZXJjb25hLmNvbTAeFw0yMzA3MDYy +MDA2NDNaFw0yNDA3MDUyMDA2NDNaMGExCzAJBgNVBAYTAlRSMRAwDgYDVQQKDAdQ +ZXJjb25hMRIwEAYDVQQDDAlsb2NhbGhvc3QxLDAqBgkqhkiG9w0BCQEWHW51cmxh +bi5tb2xkb211cm92QHBlcmNvbmEuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAmp+Xbi1C79S2l7IawE0dIBaKlx4vO/baQBDi+SXuhUonw8dPTqvD +DZy96/irc7SudvsJdcpcFn9tGfASDez56uZqt/39wsA+uUsym9yV39gcZPRzeoiV +nxDny1dcNJSqlGkcDp7BqXaj2/e6bK5RW3cpUnnRk4M7weDsdblBJLYPAIqTGMmZ +Chf/iKAz6i9E+FRuXi3rhFJKUEGv+5nh7Pjd/9BxmVyjl9f9SWhe+AkimOs+CYrh +lcGAHJ6XKq5KAMB43elZxSXgv6X5eTVhQsqf7X0e8n0OrXKqP+fqrQMPnkOsVvEj +q90mcG1mvvPrMoXN0XN1dfJEhyRB/hwi5QIDAQABMA0GCSqGSIb3DQEBCwUAA4IB +AQB9BKnOT2KiKTdnydorEpuMgzD1RZ9bfX8mGiucjPh5lcjO6L9haUbFN/6PupZP +x1WRKNwYm+R2vP/Q1tlkOwVDfBtycAhyx5NMRyHkmG90ap/hJUThF2D9Q+5A81Ma +tnpg5jbxPEBeMHujGZmDEiBNPHc9oP7HNuPXMzZWxAOjRAg2WtaqjyJi8ExnCP1t +4ELKdjojtSefhxQzZmdHNBKWa0kUJhDGfhvSo0//H9n8Q7VMmtVS94Klu/H+IG88 +EYmEzgkmty2eie+Jiv+S2WGDEUuopAReifGscFI3tYvNBbeU4GbtUCXKoNX8N6hO +1POaPPj84EK2ncLJXffk0XYq +-----END CERTIFICATE----- From 2980ec8033fcf554f8111d1e898b3a41cc7ab7ba Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 7 Jul 2023 18:43:18 +0300 Subject: [PATCH 39/77] PMM-12078 fix configuration --- docker-compose.yml | 31 +++--- managed/cmd/pmm-managed/main.go | 50 +++++---- managed/models/params.go | 18 ++++ managed/services/agents/registry.go | 4 +- .../highavailability/highavailability.go | 101 ++++++++++++------ .../highavailability/leaderservice.go | 6 ++ managed/services/highavailability/services.go | 46 ++++---- managed/services/supervisord/supervisord.go | 28 ++++- managed/testdata/haproxy/haproxy.cfg | 16 +-- 9 files changed, 198 insertions(+), 102 deletions(-) create mode 100644 managed/models/params.go diff --git a/docker-compose.yml b/docker-compose.yml index d179412099..764ee68048 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,17 +73,17 @@ services: memlock: 67108864 ports: - - ${PMM_PORT_HTTP:-80}:80 - - ${PMM_PORT_HTTPS:-443}:443 +# - ${PMM_PORT_HTTP:-80}:80 +# - ${PMM_PORT_HTTPS:-443}:443 # For headless delve - - ${PMM_PORT_DELVE:-2345}:2345 +# - ${PMM_PORT_DELVE:-2345}:2345 # PG - ${PMM_PORT_PG:-15432}:5432 # VM - - ${PMM_PORT_VM:-9090}:9090 +# - ${PMM_PORT_VM:-9090}:9090 # CH - - ${PMM_PORT_CH_TCP:-11000}:9000 - - ${PMM_PORT_CH_HTTP:-11123}:8123 +# - ${PMM_PORT_CH_TCP:-11000}:9000 +# - ${PMM_PORT_CH_HTTP:-11123}:8123 volumes: - ./:/root/go/src/github.com/percona/pmm - ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # substitute Makefile in devcontainer @@ -186,6 +186,10 @@ services: ports: - 80:80 - 443:443 + depends_on: + - pmm-server-active + - pmm-server-passive + - pmm-server-passive-2 pmm-server-active: profiles: @@ -236,10 +240,9 @@ services: - PERCONA_TEST_HA_BOOTSTRAP=1 - PERCONA_TEST_HA_NODE_ID=pmm-server-active - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 + - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - PERCONA_TEST_HA_GF_PORT=9095 - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.5:9095 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway @@ -279,6 +282,7 @@ services: - ch - pg - victoriametrics + - pmm-server-active image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-server-passive hostname: pmm-server-passive @@ -319,11 +323,9 @@ services: - PERCONA_TEST_HA_ENABLE=1 - PERCONA_TEST_HA_NODE_ID=pmm-server-passive - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive + - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - PERCONA_TEST_HA_GF_PORT=9095 - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.6:9095 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway @@ -363,6 +365,7 @@ services: - ch - pg - victoriametrics + - pmm-server-active image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-server-passive-2 hostname: pmm-server-passive-2 @@ -401,13 +404,11 @@ services: - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - PERCONA_TEST_HA_ENABLE=1 - - PERCONA_TEST_HA_NODE_ID=pmm-server-passive + - PERCONA_TEST_HA_NODE_ID=pmm-server-passive-2 - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11 - - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive + - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - PERCONA_TEST_HA_GF_PORT=9095 - - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.11:9095 - - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 extra_hosts: - host.docker.internal:host-gateway diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 4ec1b060d7..93211f9d56 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -252,7 +252,6 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { l.Debug("RPC response latency histogram enabled.") grpcMetrics.EnableHandlingTimeHistogram() } - deps.ha.RegisterGRPC(gRPCServer) serverpb.RegisterServerServer(gRPCServer, deps.server) agentpb.RegisterAgentServer(gRPCServer, agentgrpc.NewAgentServer(deps.handler)) @@ -744,6 +743,10 @@ func main() { //nolint:cyclop,maintidx Envar("PERCONA_TEST_HA_GOSSIP_PORT"). Default("9761"). Int() + grafanaGossipPort := kingpin.Flag("grafana-gossip-port", "Grafana gossip port"). + Envar("PERCONA_TEST_GRAFANA_GOSSIP_PORT"). + Default("9762"). + Int() // haGrafanaPort := kingpin.Flag("ha-grafana-port", "HA grafana port"). // Envar("PERCONA_TEST_HA_GRAFANA_PORT"). // Default("9760"). @@ -778,14 +781,19 @@ func main() { //nolint:cyclop,maintidx ctx = logger.Set(ctx, "main") defer l.Info("Done.") - haParams := &highavailability.Params{ - Enabled: *haEnabled, - Bootstrap: *haBootstrap, - NodeID: *haNodeID, - AdvertiseAddress: *haAdvertiseAddress, - Nodes: strings.Split(*haGossipNodes, ","), - RaftPort: *haRaftPort, - GossipPort: *haGossipPort, + var nodes []string + if *haGossipNodes != "" { + nodes = strings.Split(*haGossipNodes, ",") + } + haParams := &models.HAParams{ + Enabled: *haEnabled, + Bootstrap: *haBootstrap, + NodeID: *haNodeID, + AdvertiseAddress: *haAdvertiseAddress, + Nodes: nodes, + RaftPort: *haRaftPort, + GossipPort: *haGossipPort, + GrafanaGossipPort: *grafanaGossipPort, } ha := highavailability.New(haParams) @@ -883,7 +891,7 @@ func main() { //nolint:cyclop,maintidx // TODO remove once PMM cluster will be Active-Active ha.AddLeaderService(highavailability.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() error { - return agentsRegistry.KickAll() + return agentsRegistry.KickAll(ctx) })) pbmPITRService := backup.NewPBMPITRService() @@ -909,16 +917,18 @@ func main() { //nolint:cyclop,maintidx supervisord := supervisord.New( *supervisordConfigDirF, pmmUpdateCheck, - vmParams, - models.PGParams{ - Addr: *postgresAddrF, - DBName: *postgresDBNameF, - DBUsername: *postgresDBUsernameF, - DBPassword: *postgresDBPasswordF, - SSLMode: *postgresSSLModeF, - SSLCAPath: *postgresSSLCAPathF, - SSLKeyPath: *postgresSSLKeyPathF, - SSLCertPath: *postgresSSLCertPathF, + &models.Params{ + VMParams: vmParams, + PGParams: &models.PGParams{ + Addr: *postgresAddrF, + DBName: *postgresDBNameF, + DBUsername: *postgresDBUsernameF, + DBPassword: *postgresDBPasswordF, + SSLMode: *postgresSSLModeF, + SSLCAPath: *postgresSSLCAPathF, + SSLKeyPath: *postgresSSLKeyPathF, + SSLCertPath: *postgresSSLCertPathF, + }, }, gRPCMessageMaxSize) diff --git a/managed/models/params.go b/managed/models/params.go new file mode 100644 index 0000000000..c6c7852284 --- /dev/null +++ b/managed/models/params.go @@ -0,0 +1,18 @@ +package models + +type HAParams struct { + GrafanaGossipPort int + Enabled bool + Bootstrap bool + NodeID string + AdvertiseAddress string + Nodes []string + RaftPort int + GossipPort int +} + +type Params struct { + HAParams *HAParams + VMParams *VictoriaMetricsParams + PGParams *PGParams +} diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index d7de332438..278c0f3bdf 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -440,9 +440,9 @@ func (r *Registry) Collect(ch chan<- prom.Metric) { r.mClockDrift.Collect(ch) } -func (r *Registry) KickAll() error { +func (r *Registry) KickAll(ctx context.Context) error { for _, agentInfo := range r.agents { - r.Kick(context.Background(), agentInfo.ID()) + r.Kick(ctx, agentInfo.ID()) } return nil } diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index d3ade46254..ac81b8ddb5 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -19,6 +19,7 @@ package highavailability import ( "context" "fmt" + "github.com/percona/pmm/managed/models" "io" "net" "sync" @@ -28,24 +29,13 @@ import ( "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/sirupsen/logrus" - "google.golang.org/grpc" "google.golang.org/protobuf/proto" "github.com/percona/pmm/api/hapb" ) -type Params struct { - Enabled bool - Bootstrap bool - NodeID string - AdvertiseAddress string - Nodes []string - RaftPort int - GossipPort int -} - type Service struct { - params *Params + params *models.HAParams bootstrapCluster bool services *services @@ -89,7 +79,7 @@ func (s *Service) Restore(snapshot io.ReadCloser) error { return nil } -func New(params *Params) *Service { +func New(params *models.HAParams) *Service { return &Service{ params: params, bootstrapCluster: params.Bootstrap, @@ -99,7 +89,6 @@ func New(params *Params) *Service { receivedMessages: make(chan []byte), l: logrus.WithField("component", "ha"), wg: &sync.WaitGroup{}, - tm: transport.New(raft.ServerAddress(fmt.Sprintf("%s:443", params.AdvertiseAddress)), []grpc.DialOption{grpc.WithInsecure()}), } } @@ -114,6 +103,7 @@ func (s *Service) Run(ctx context.Context) error { s.services.StartAllServices(ctx) } case <-ctx.Done(): + s.services.StopRunningServices() return } } @@ -123,10 +113,13 @@ func (s *Service) Run(ctx context.Context) error { return nil } + s.l.Infoln("Starting...") + defer s.l.Infoln("Done.") + // Create the Raft configuration raftConfig := raft.DefaultConfig() raftConfig.LocalID = raft.ServerID(s.params.NodeID) - raftConfig.LogLevel = "INFO" + raftConfig.LogLevel = "DEBUG" // Create a new Raft transport raa, err := net.ResolveTCPAddr("", fmt.Sprintf("%s:%d", s.params.AdvertiseAddress, s.params.RaftPort)) @@ -146,8 +139,13 @@ func (s *Service) Run(ctx context.Context) error { return err } defer func() { + if s.IsLeader() { + s.raftNode.LeadershipTransfer() + } err := s.raftNode.Shutdown().Error() - s.l.Errorln(err) + if err != nil { + s.l.Errorf("error during the shutdown of raft node: %q", err) + } }() // Create the memberlist configuration @@ -164,7 +162,16 @@ func (s *Service) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to create memberlist: %q", err) } - defer s.memberlist.Leave(5 * time.Second) + defer func() { + err := s.memberlist.Leave(5 * time.Second) + if err != nil { + s.l.Errorf("couldn't leave memberlist cluster: %q", err) + } + err = s.memberlist.Shutdown() + if err != nil { + s.l.Errorf("couldn't shutdown memberlist listeners: %q", err) + } + }() if s.bootstrapCluster { // Start the Raft node @@ -180,7 +187,8 @@ func (s *Service) Run(ctx context.Context) error { if err := s.raftNode.BootstrapCluster(cfg).Error(); err != nil { return fmt.Errorf("failed to bootstrap Raft cluster: %q", err) } - } else { + } + if len(s.params.Nodes) > 0 { _, err := s.memberlist.Join(s.params.Nodes) if err != nil { return fmt.Errorf("failed to join memberlist cluster: %q", err) @@ -207,6 +215,8 @@ func (s *Service) Run(ctx context.Context) error { } func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { + t := time.NewTicker(5 * time.Second) + for { select { case event := <-s.nodeCh: @@ -214,21 +224,55 @@ func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { continue } node := event.Node - s.rw.RLock() switch event.Event { case memberlist.NodeJoin: - s.raftNode.AddVoter(raft.ServerID(node.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", node.Addr.String(), s.params.RaftPort)), s.raftNode.AppliedIndex(), 10*time.Second).Error() + s.addMemberlistNodeToRaft(node) case memberlist.NodeLeave: - s.raftNode.RemoveServer(raft.ServerID(node.Name), 0, 10*time.Second).Error() + s.removeMemberlistNodeFromRaft(node) + } + case <-t.C: + if !s.IsLeader() { + continue + } + servers := s.raftNode.GetConfiguration().Configuration().Servers + raftServers := make(map[string]struct{}) + for _, server := range servers { + raftServers[string(server.ID)] = struct{}{} + } + members := s.memberlist.Members() + s.l.Infoln("memberlist members: %v", members) + for _, node := range members { + if _, ok := raftServers[node.Name]; !ok { + s.addMemberlistNodeToRaft(node) + } } - s.rw.RUnlock() case <-ctx.Done(): + t.Stop() return } } } +func (s *Service) removeMemberlistNodeFromRaft(node *memberlist.Node) { + s.rw.RLock() + defer s.rw.RUnlock() + err := s.raftNode.RemoveServer(raft.ServerID(node.Name), 0, 10*time.Second).Error() + if err != nil { + s.l.Errorln(err) + } +} + +func (s *Service) addMemberlistNodeToRaft(node *memberlist.Node) { + s.rw.RLock() + defer s.rw.RUnlock() + err := s.raftNode.AddVoter(raft.ServerID(node.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", node.Addr.String(), s.params.RaftPort)), 0, 10*time.Second).Error() + if err != nil { + s.l.Errorf("couldn't add a server node %s: %q", node.Name, err) + } +} + func (s *Service) runLeaderObserver(ctx context.Context) { + t := time.NewTicker(5 * time.Second) for { s.rw.RLock() node := s.raftNode @@ -244,14 +288,15 @@ func (s *Service) runLeaderObserver(ctx context.Context) { if peer.Name == s.params.NodeID { continue } - if err := node.AddVoter(raft.ServerID(peer.Name), raft.ServerAddress(fmt.Sprintf("%s:%d", peer.Addr.String(), s.params.RaftPort)), node.AppliedIndex(), 10*time.Second).Error(); err != nil { - s.l.Warnf("Failed to add Raft member: %v", err) - } + s.addMemberlistNodeToRaft(peer) } } else { s.l.Printf("I am not a leader!") s.services.StopRunningServices() } + case <-t.C: + address, serverID := s.raftNode.LeaderWithID() + s.l.Infof("Leader is %s on %s", serverID, address) case <-ctx.Done(): return } @@ -284,9 +329,3 @@ func (s *Service) IsLeader() bool { func (s *Service) Bootstrap() bool { return s.params.Bootstrap || !s.params.Enabled } - -func (s *Service) RegisterGRPC(server *grpc.Server) { - if s.params.Enabled { - s.tm.Register(server) - } -} diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index bc0d28debe..14e3eeb221 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -44,6 +44,8 @@ type ContextService struct { startFunc func(context.Context) error ctx context.Context cancel context.CancelFunc + + m sync.Mutex } func NewContextService(id string, startFunc func(context.Context) error) *ContextService { @@ -58,11 +60,15 @@ func (s *ContextService) ID() string { } func (s *ContextService) Start(ctx context.Context) error { + s.m.Lock() s.ctx, s.cancel = context.WithCancel(ctx) + s.m.Unlock() return s.startFunc(ctx) } func (s *ContextService) Stop() error { + s.m.Lock() + defer s.m.Unlock() s.cancel() return nil } diff --git a/managed/services/highavailability/services.go b/managed/services/highavailability/services.go index 8b9e267b5b..7daabeb980 100644 --- a/managed/services/highavailability/services.go +++ b/managed/services/highavailability/services.go @@ -3,6 +3,7 @@ package highavailability import ( "context" "fmt" + "github.com/sirupsen/logrus" "sync" ) @@ -13,6 +14,8 @@ type services struct { running map[string]LeaderService serviceAdded chan struct{} + + l *logrus.Entry } func newServices() *services { @@ -21,6 +24,7 @@ func newServices() *services { all: make(map[string]LeaderService), running: make(map[string]LeaderService), serviceAdded: make(chan struct{}), + l: logrus.WithField("component", "ha-services"), } } @@ -47,38 +51,34 @@ func (s *services) StartAllServices(ctx context.Context) { for id, service := range s.all { if _, ok := s.running[id]; !ok { s.wg.Add(1) - go func(ls LeaderService) { - defer s.wg.Done() - ls.Start(ctx) - s.rw.Lock() - defer s.rw.Unlock() - delete(s.running, id) - }(service) s.running[id] = service + ls := service + go func() { + s.l.Infoln("Starting ", ls.ID()) + err := ls.Start(ctx) + if err != nil { + s.l.Errorln(err) + } + }() } } } -func (s *services) StartService(ctx context.Context, id string) error { - s.rw.Lock() - defer s.rw.Unlock() - service, ok := s.all[id] - if !ok { - return fmt.Errorf("service with ID %s not found", id) - } - if _, ok := s.running[id]; !ok { - go service.Start(ctx) - s.running[id] = service - } - return nil -} - func (s *services) StopRunningServices() { s.rw.Lock() defer s.rw.Unlock() - for _, service := range s.running { - go service.Stop() + for id, service := range s.running { + id := id + ls := service + go func() { + defer s.wg.Done() + s.l.Infoln("Stopping ", ls) + ls.Stop() + s.rw.Lock() + defer s.rw.Unlock() + delete(s.running, id) + }() } } diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 490ea4423b..2f5b2d13de 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -68,7 +68,8 @@ type Service struct { supervisordConfigsM sync.Mutex vmParams *models.VictoriaMetricsParams - pgParams models.PGParams + pgParams *models.PGParams + haParams *models.HAParams } type sub struct { @@ -84,7 +85,7 @@ const ( ) // New creates new service. -func New(configDir string, pmmUpdateCheck *PMMUpdateChecker, vmParams *models.VictoriaMetricsParams, pgParams models.PGParams, gRPCMessageMaxSize uint32) *Service { +func New(configDir string, pmmUpdateCheck *PMMUpdateChecker, params *models.Params, gRPCMessageMaxSize uint32) *Service { path, _ := exec.LookPath("supervisorctl") return &Service{ configDir: configDir, @@ -94,8 +95,9 @@ func New(configDir string, pmmUpdateCheck *PMMUpdateChecker, vmParams *models.Vi pmmUpdateCheck: pmmUpdateCheck, subs: make(map[chan *event]sub), lastEvents: make(map[string]eventType), - vmParams: vmParams, - pgParams: pgParams, + vmParams: params.VMParams, + pgParams: params.PGParams, + haParams: params.HAParams, } } @@ -447,6 +449,7 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin } s.addPostgresParams(templateParams) + s.addClusterParams(templateParams) if ssoDetails != nil { u, err := url.Parse(ssoDetails.IssuerURL) @@ -516,6 +519,19 @@ func (s *Service) addPostgresParams(templateParams map[string]interface{}) { templateParams["PostgresSSLCertPath"] = s.pgParams.SSLCertPath } +func (s *Service) addClusterParams(templateParams map[string]interface{}) { + templateParams["GrafanaGossipPort"] = s.haParams.GrafanaGossipPort + templateParams["HAAdvertiseAddress"] = s.haParams.AdvertiseAddress + nodes := make([]string, len(s.haParams.Nodes)) + for i, node := range s.haParams.Nodes { + nodes[i] = fmt.Sprintf("%s:%d", node, s.haParams.GrafanaGossipPort) + } + templateParams["HANodes"] = strings.Join(nodes, ",") + templateParams["HAEnabled"] = s.haParams.Enabled + //- GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.5:9095 + //- GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 +} + // saveConfigAndReload saves given supervisord program configuration to file and reloads it. // If configuration can't be reloaded for some reason, old file is restored, and configuration is reloaded again. // Returns true if configuration was changed. @@ -818,6 +834,10 @@ environment = {{- if .PerconaSSODetails}} GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} + {{- if .HAEnabled}} + GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS={{ .HAAdvertiseAddress }}:{{ .GrafanaGossipPort }} + GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 + {{- end}} user = grafana directory = /usr/share/grafana autorestart = true diff --git a/managed/testdata/haproxy/haproxy.cfg b/managed/testdata/haproxy/haproxy.cfg index e7bc5c0c2c..57cf72dfb0 100644 --- a/managed/testdata/haproxy/haproxy.cfg +++ b/managed/testdata/haproxy/haproxy.cfg @@ -1,6 +1,7 @@ global - log /dev/log local0 - log /dev/log local1 notice + log stdout local0 debug + log stdout local1 info + log stdout local2 info daemon defaults @@ -25,13 +26,14 @@ backend http_back http-check send meth POST uri /v1/leaderHealthCheck ver HTTP/1.1 hdr Host www http-check expect status 200 server pmm-server-active-http pmm-server-active:80 check - server pmm-server-passive-http pmm-server-passive:80 check - server pmm-server-passive-2-http pmm-server-passive-2:80 check + server pmm-server-passive-http pmm-server-passive:80 check backup + server pmm-server-passive-2-http pmm-server-passive-2:80 check backup backend https_back option httpchk http-check send meth POST uri /v1/leaderHealthCheck ver HTTP/1.1 hdr Host www http-check expect status 200 - server pmm-server-active-https pmm-server-active:443 ssl verify none - server pmm-server-passive-https pmm-server-passive:443 ssl verify none - server pmm-server-passive-2-https pmm-server-passive-2:443 ssl verify none + server pmm-server-active-https pmm-server-active:443 check ssl verify none backup + server pmm-server-passive-https pmm-server-passive:443 check ssl verify none backup + server pmm-server-passive-2-https pmm-server-passive-2:443 check ssl verify none backup + From e86414c16ddc54b900ca0732a2dff9f3aed9b0b7 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 7 Jul 2023 23:20:46 +0300 Subject: [PATCH 40/77] PMM-12078 fix configuration --- managed/cmd/pmm-managed/main.go | 1 + .../services/supervisord/devcontainer_test.go | 4 ++-- managed/services/supervisord/supervisord.go | 19 ++++++++++++------- .../services/supervisord/supervisord_test.go | 6 +++--- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 93211f9d56..c3998e0d51 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -929,6 +929,7 @@ func main() { //nolint:cyclop,maintidx SSLKeyPath: *postgresSSLKeyPathF, SSLCertPath: *postgresSSLCertPathF, }, + HAParams: haParams, }, gRPCMessageMaxSize) diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index 0110cfb793..bb8fc17c05 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -115,7 +115,7 @@ func TestDevContainer(t *testing.T) { vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) - s := New("/etc/supervisord.d", checker, vmParams, models.PGParams{}, gRPCMessageMaxSize) + s := New("/etc/supervisord.d", checker, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}}, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) ctx, cancel := context.WithCancel(context.Background()) @@ -168,7 +168,7 @@ func TestDevContainer(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) vmParams := &models.VictoriaMetricsParams{} - s := New("/etc/supervisord.d", checker, vmParams, models.PGParams{}, gRPCMessageMaxSize) + s := New("/etc/supervisord.d", checker, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}}, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) ctx, cancel := context.WithCancel(context.Background()) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 2f5b2d13de..82879ff368 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -509,6 +509,9 @@ func addAlertManagerParams(alertManagerURL string, templateParams map[string]int // addPostgresParams adds pmm-server postgres database params to template config for grafana. func (s *Service) addPostgresParams(templateParams map[string]interface{}) { + if s.pgParams == nil { + return + } templateParams["PostgresAddr"] = s.pgParams.Addr templateParams["PostgresDBName"] = s.pgParams.DBName templateParams["PostgresDBUsername"] = s.pgParams.DBUsername @@ -520,14 +523,16 @@ func (s *Service) addPostgresParams(templateParams map[string]interface{}) { } func (s *Service) addClusterParams(templateParams map[string]interface{}) { - templateParams["GrafanaGossipPort"] = s.haParams.GrafanaGossipPort - templateParams["HAAdvertiseAddress"] = s.haParams.AdvertiseAddress - nodes := make([]string, len(s.haParams.Nodes)) - for i, node := range s.haParams.Nodes { - nodes[i] = fmt.Sprintf("%s:%d", node, s.haParams.GrafanaGossipPort) - } - templateParams["HANodes"] = strings.Join(nodes, ",") templateParams["HAEnabled"] = s.haParams.Enabled + if s.haParams.Enabled { + templateParams["GrafanaGossipPort"] = s.haParams.GrafanaGossipPort + templateParams["HAAdvertiseAddress"] = s.haParams.AdvertiseAddress + nodes := make([]string, len(s.haParams.Nodes)) + for i, node := range s.haParams.Nodes { + nodes[i] = fmt.Sprintf("%s:%d", node, s.haParams.GrafanaGossipPort) + } + templateParams["HANodes"] = strings.Join(nodes, ",") + } //- GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS=172.20.0.5:9095 //- GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 } diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index e700f03751..f60f2ebbef 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -38,7 +38,7 @@ func TestConfig(t *testing.T) { configDir := filepath.Join("..", "..", "testdata", "supervisord.d") vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) - pgParams := models.PGParams{ + pgParams := &models.PGParams{ Addr: "127.0.0.1:5432", DBName: "postgres", DBUsername: "db_username", @@ -48,7 +48,7 @@ func TestConfig(t *testing.T) { SSLKeyPath: "path-to-key", SSLCertPath: "path-to-cert", } - s := New(configDir, pmmUpdateCheck, vmParams, pgParams, gRPCMessageMaxSize) + s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: pgParams}, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, AlertManagerURL: "https://external-user:passw!,ord@external-alertmanager:6443/alerts", @@ -81,7 +81,7 @@ func TestDBaaSController(t *testing.T) { configDir := filepath.Join("..", "..", "testdata", "supervisord.d") vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) - s := New(configDir, pmmUpdateCheck, vmParams, models.PGParams{}, gRPCMessageMaxSize) + s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}}, gRPCMessageMaxSize) var tp *template.Template for _, tmpl := range templates.Templates() { From d32960c8b6221973e039b35f9f963b41507e4048 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 7 Jul 2023 23:22:15 +0300 Subject: [PATCH 41/77] PMM-12078 fix tests --- managed/services/supervisord/devcontainer_test.go | 4 ++-- managed/services/supervisord/supervisord_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index bb8fc17c05..2495da119e 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -115,7 +115,7 @@ func TestDevContainer(t *testing.T) { vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) - s := New("/etc/supervisord.d", checker, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}}, gRPCMessageMaxSize) + s := New("/etc/supervisord.d", checker, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}, HAParams: &models.HAParams{}}, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) ctx, cancel := context.WithCancel(context.Background()) @@ -168,7 +168,7 @@ func TestDevContainer(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) vmParams := &models.VictoriaMetricsParams{} - s := New("/etc/supervisord.d", checker, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}}, gRPCMessageMaxSize) + s := New("/etc/supervisord.d", checker, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}, HAParams: &models.HAParams{}}, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) ctx, cancel := context.WithCancel(context.Background()) diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index f60f2ebbef..2d7ab2087b 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -48,7 +48,7 @@ func TestConfig(t *testing.T) { SSLKeyPath: "path-to-key", SSLCertPath: "path-to-cert", } - s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: pgParams}, gRPCMessageMaxSize) + s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: pgParams, HAParams: &models.HAParams{}}, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, AlertManagerURL: "https://external-user:passw!,ord@external-alertmanager:6443/alerts", @@ -81,7 +81,7 @@ func TestDBaaSController(t *testing.T) { configDir := filepath.Join("..", "..", "testdata", "supervisord.d") vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) require.NoError(t, err) - s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}}, gRPCMessageMaxSize) + s := New(configDir, pmmUpdateCheck, &models.Params{VMParams: vmParams, PGParams: &models.PGParams{}, HAParams: &models.HAParams{}}, gRPCMessageMaxSize) var tp *template.Template for _, tmpl := range templates.Templates() { From 285b327f52e30b18536a3b86053cc29fbfc72794 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 7 Jul 2023 23:53:06 +0300 Subject: [PATCH 42/77] PMM-12078 fix tests --- .../services/supervisord/pmm_config_test.go | 47 +++++++++++++++++++ .../services/supervisord/supervisord_test.go | 37 --------------- 2 files changed, 47 insertions(+), 37 deletions(-) create mode 100644 managed/services/supervisord/pmm_config_test.go diff --git a/managed/services/supervisord/pmm_config_test.go b/managed/services/supervisord/pmm_config_test.go new file mode 100644 index 0000000000..14abdb0cce --- /dev/null +++ b/managed/services/supervisord/pmm_config_test.go @@ -0,0 +1,47 @@ +package supervisord + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSavePMMConfig(t *testing.T) { + t.Parallel() + configDir := filepath.Join("..", "..", "testdata", "supervisord.d") + tests := []struct { + description string + params map[string]any + file string + }{ + { + description: "disable internal postgresql db", + params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "DisableInternalClickhouse": false, "PassivePMM": false}, + file: "pmm-db_disabled", + }, + { + description: "enable internal postgresql db", + params: map[string]any{"DisableInternalDB": false, "DisableSupervisor": false, "DisableInternalClickhouse": false, "PassivePMM": false}, + file: "pmm-db_enabled", + }, + { + description: "passive pmm", + params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "DisableInternalClickhouse": false, "PassivePMM": true}, + file: "pmm-passive", + }, + } + for _, test := range tests { + test := test + t.Run(test.description, func(t *testing.T) { + t.Parallel() + expected, err := os.ReadFile(filepath.Join(configDir, test.file+".ini")) //nolint:gosec + require.NoError(t, err) + actual, err := marshalConfig(test.params) + require.NoError(t, err) + assert.Equal(t, string(expected), string(actual)) + }) + } +} diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index 2d7ab2087b..c6343103e3 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -171,40 +171,3 @@ func TestAddAlertManagerParam(t *testing.T) { require.Equal(t, "http://127.0.0.1:9093/alertmanager", params["AlertmanagerURL"]) }) } - -func TestSavePMMConfig(t *testing.T) { - t.Parallel() - configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - tests := []struct { - description string - params map[string]any - file string - }{ - { - description: "disable internal postgresql db", - params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "PassivePMM": false}, - file: "pmm-db_disabled", - }, - { - description: "enable internal postgresql db", - params: map[string]any{"DisableInternalDB": false, "DisableSupervisor": false, "PassivePMM": false}, - file: "pmm-db_enabled", - }, - { - description: "passive pmm", - params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "PassivePMM": true}, - file: "pmm-passive", - }, - } - for _, test := range tests { - test := test - t.Run(test.description, func(t *testing.T) { - t.Parallel() - expected, err := os.ReadFile(filepath.Join(configDir, test.file+".ini")) //nolint:gosec - require.NoError(t, err) - actual, err := marshalConfig(test.params) - require.NoError(t, err) - assert.Equal(t, string(expected), string(actual)) - }) - } -} From 1a25b5adb21af33d8217843fbc035a14d8f354c4 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Jul 2023 00:38:38 +0300 Subject: [PATCH 43/77] PMM-12078 fix supervisord --- managed/services/supervisord/supervisord.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 82879ff368..b9c766d4fd 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -840,8 +840,8 @@ environment = GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} {{- if .HAEnabled}} - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS={{ .HAAdvertiseAddress }}:{{ .GrafanaGossipPort }} - GF_UNIFIED_ALERTING_HA_PEERS=pmm-server-active:9095,pmm-server-passive:9095 + GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS="{{ .HAAdvertiseAddress }}:{{ .GrafanaGossipPort }}" + GF_UNIFIED_ALERTING_HA_PEERS="{{ .HANodes }}" {{- end}} user = grafana directory = /usr/share/grafana From affcfbec49feed869c5e7232aa9a8d325747c4af Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Jul 2023 01:01:40 +0300 Subject: [PATCH 44/77] PMM-12078 fix supervisord --- docker-compose.yml | 24 ++++++++++----------- managed/services/supervisord/supervisord.go | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 764ee68048..a03e7cfa56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,17 +73,17 @@ services: memlock: 67108864 ports: -# - ${PMM_PORT_HTTP:-80}:80 -# - ${PMM_PORT_HTTPS:-443}:443 + - ${PMM_PORT_HTTP:-80}:80 + - ${PMM_PORT_HTTPS:-443}:443 # For headless delve -# - ${PMM_PORT_DELVE:-2345}:2345 + - ${PMM_PORT_DELVE:-2345}:2345 # PG - ${PMM_PORT_PG:-15432}:5432 # VM -# - ${PMM_PORT_VM:-9090}:9090 + - ${PMM_PORT_VM:-9090}:9090 # CH -# - ${PMM_PORT_CH_TCP:-11000}:9000 -# - ${PMM_PORT_CH_HTTP:-11123}:8123 + - ${PMM_PORT_CH_TCP:-11000}:9000 + - ${PMM_PORT_CH_HTTP:-11123}:8123 volumes: - ./:/root/go/src/github.com/percona/pmm - ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # substitute Makefile in devcontainer @@ -205,7 +205,7 @@ services: ha: ipv4_address: 172.20.0.5 environment: - - PMM_RELEASE_PATH=/root/go/bin + - PMM_RELEASE_PATH=/root/go/src/github.com/percona/pmm/bin - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} - ENABLE_DBAAS=${ENABLE_DBAAS:-0} - AWS_ACCESS_KEY=${AWS_ACCESS_KEY} @@ -242,7 +242,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_HA_GF_PORT=9095 + - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9095 extra_hosts: - host.docker.internal:host-gateway @@ -290,7 +290,7 @@ services: ha: ipv4_address: 172.20.0.6 environment: - - PMM_RELEASE_PATH=/root/go/bin + - PMM_RELEASE_PATH=/root/go/src/github.com/percona/pmm/bin - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} - ENABLE_DBAAS=${ENABLE_DBAAS:-0} - AWS_ACCESS_KEY=${AWS_ACCESS_KEY} @@ -325,7 +325,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_HA_GF_PORT=9095 + - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9095 extra_hosts: - host.docker.internal:host-gateway @@ -373,7 +373,7 @@ services: ha: ipv4_address: 172.20.0.11 environment: - - PMM_RELEASE_PATH=/root/go/bin + - PMM_RELEASE_PATH=/root/go/src/github.com/percona/pmm/bin - REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN} - ENABLE_DBAAS=${ENABLE_DBAAS:-0} - AWS_ACCESS_KEY=${AWS_ACCESS_KEY} @@ -408,7 +408,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_HA_GF_PORT=9095 + - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9095 extra_hosts: - host.docker.internal:host-gateway diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index b9c766d4fd..9320f732dd 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -840,8 +840,8 @@ environment = GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} {{- if .HAEnabled}} - GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS="{{ .HAAdvertiseAddress }}:{{ .GrafanaGossipPort }}" - GF_UNIFIED_ALERTING_HA_PEERS="{{ .HANodes }}" + GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS="{{ .HAAdvertiseAddress }}:{{ .GrafanaGossipPort }}", + GF_UNIFIED_ALERTING_HA_PEERS="{{ .HANodes }}" {{- end}} user = grafana directory = /usr/share/grafana From 520150276aaea41d6dddb67dea80a8d9b33da23d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Jul 2023 01:57:49 +0300 Subject: [PATCH 45/77] PMM-12078 fix linters --- managed/services/highavailability/highavailability.go | 4 ++-- managed/services/highavailability/services.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index ac81b8ddb5..ca33ef86cb 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -19,7 +19,6 @@ package highavailability import ( "context" "fmt" - "github.com/percona/pmm/managed/models" "io" "net" "sync" @@ -32,6 +31,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/percona/pmm/api/hapb" + "github.com/percona/pmm/managed/models" ) type Service struct { @@ -240,7 +240,7 @@ func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { raftServers[string(server.ID)] = struct{}{} } members := s.memberlist.Members() - s.l.Infoln("memberlist members: %v", members) + s.l.Infof("memberlist members: %v", members) for _, node := range members { if _, ok := raftServers[node.Name]; !ok { s.addMemberlistNodeToRaft(node) diff --git a/managed/services/highavailability/services.go b/managed/services/highavailability/services.go index 7daabeb980..429bdb9537 100644 --- a/managed/services/highavailability/services.go +++ b/managed/services/highavailability/services.go @@ -3,8 +3,9 @@ package highavailability import ( "context" "fmt" - "github.com/sirupsen/logrus" "sync" + + "github.com/sirupsen/logrus" ) type services struct { From 0e7d22fe59a45382f948c7725536412c6ec99f16 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Sat, 8 Jul 2023 02:32:31 +0300 Subject: [PATCH 46/77] PMM-12078 fix listen port for grafana gossip protocol --- docker-compose.yml | 6 +++--- managed/services/supervisord/supervisord.go | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a03e7cfa56..6fd3ae5379 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -242,7 +242,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9095 + - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9094 extra_hosts: - host.docker.internal:host-gateway @@ -325,7 +325,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9095 + - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9094 extra_hosts: - host.docker.internal:host-gateway @@ -408,7 +408,7 @@ services: - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11 - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9095 + - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9094 extra_hosts: - host.docker.internal:host-gateway diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 9320f732dd..b40ff1d081 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -840,6 +840,7 @@ environment = GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} {{- if .HAEnabled}} + GF_UNIFIED_ALERTING_HA_LISTEN_ADDRESS="0.0.0.0:{{ .GrafanaGossipPort }}", GF_UNIFIED_ALERTING_HA_ADVERTISE_ADDRESS="{{ .HAAdvertiseAddress }}:{{ .GrafanaGossipPort }}", GF_UNIFIED_ALERTING_HA_PEERS="{{ .HANodes }}" {{- end}} From 1552bde401e29f51964b961a18a97f36c296171c Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 10 Jul 2023 03:26:31 +0300 Subject: [PATCH 47/77] PMM-12078 Some cleanup. --- managed/cmd/pmm-managed/main.go | 1 - managed/services/highavailability/channel.go | 19 ----- .../highavailability/leaderservice.go | 15 ++++ managed/services/highavailability/raft.go | 78 ------------------- managed/services/highavailability/services.go | 15 ++++ 5 files changed, 30 insertions(+), 98 deletions(-) delete mode 100644 managed/services/highavailability/channel.go delete mode 100644 managed/services/highavailability/raft.go diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index c3998e0d51..3e6963a956 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -886,7 +886,6 @@ func main() { //nolint:cyclop,maintidx qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) - highavailability.NewChannel() agentsRegistry := agents.NewRegistry(db, vmParams) // TODO remove once PMM cluster will be Active-Active diff --git a/managed/services/highavailability/channel.go b/managed/services/highavailability/channel.go deleted file mode 100644 index c643593f04..0000000000 --- a/managed/services/highavailability/channel.go +++ /dev/null @@ -1,19 +0,0 @@ -package highavailability - -import ( - "github.com/percona/pmm/api/agentpb" -) - -type Channel struct{} - -func NewChannel() *Channel { - return &Channel{} -} - -func (c *Channel) Send(*agentpb.ServerMessage) error { - return nil -} - -func (c *Channel) Recv() (*agentpb.AgentMessage, error) { - return nil, nil -} diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 14e3eeb221..2e42b01671 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -1,3 +1,18 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package highavailability import ( diff --git a/managed/services/highavailability/raft.go b/managed/services/highavailability/raft.go deleted file mode 100644 index 06676f073c..0000000000 --- a/managed/services/highavailability/raft.go +++ /dev/null @@ -1,78 +0,0 @@ -package highavailability - -import ( - "context" - "io" - "sync" - - "github.com/hashicorp/memberlist" - "github.com/hashicorp/raft" - "github.com/sirupsen/logrus" -) - -// raftFSM is a simple example implementation of a Raft FSM. -type raftFSM struct { - m *memberlist.Memberlist - mu sync.Mutex - nodeID raft.ServerID - leaderID raft.ServerID - - l *logrus.Entry -} - -func newFSM(m *memberlist.Memberlist, nodeID raft.ServerID) *raftFSM { - return &raftFSM{ - m: m, - nodeID: nodeID, - l: logrus.WithField("component", "raftFSM"), - } -} - -func (f *raftFSM) Run(ctx context.Context, leaderCh chan raft.Observation) { - for { - select { - case o := <-leaderCh: - l := o.Data.(raft.LeaderObservation) - f.LeaderChange(l.LeaderID) - case <-ctx.Done(): - return - } - } -} - -func (f *raftFSM) Apply(logEntry *raft.Log) interface{} { - f.mu.Lock() - defer f.mu.Unlock() - f.l.Printf("raft: got a message: %s", string(logEntry.Data)) - // If this node is the leader, broadcast the message to all the other nodes - if f.nodeID == f.leaderID { - f.l.Printf("Broadcasting message: %s", string(logEntry.Data)) - for _, peer := range f.m.Members() { - if peer.Name == string(f.nodeID) { - continue - } - f.l.Printf("Sending message to: %s", peer.Name) - err := f.m.SendReliable(peer, logEntry.Data) - if err != nil { - f.l.Printf("Failed to send message to node %s: %v", peer.Name, err) - } - } - } - - return nil -} - -func (f *raftFSM) Snapshot() (raft.FSMSnapshot, error) { - return nil, nil -} - -func (f *raftFSM) Restore(rc io.ReadCloser) error { - return nil -} - -func (f *raftFSM) LeaderChange(leader raft.ServerID) { - f.mu.Lock() - defer f.mu.Unlock() - f.l.Printf("Leader changed from %s to %s", f.leaderID, leader) - f.leaderID = leader -} diff --git a/managed/services/highavailability/services.go b/managed/services/highavailability/services.go index 429bdb9537..e3772d1a02 100644 --- a/managed/services/highavailability/services.go +++ b/managed/services/highavailability/services.go @@ -1,3 +1,18 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package highavailability import ( From ff7a30577e0c41116eb3c65c2f2289e3543a348b Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 10 Jul 2023 11:44:20 +0300 Subject: [PATCH 48/77] PMM-12078 fix license headers. --- managed/models/params.go | 15 +++++++++++++++ managed/services/supervisord/pmm_config_test.go | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/managed/models/params.go b/managed/models/params.go index c6c7852284..2be506c3c5 100644 --- a/managed/models/params.go +++ b/managed/models/params.go @@ -1,3 +1,18 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package models type HAParams struct { diff --git a/managed/services/supervisord/pmm_config_test.go b/managed/services/supervisord/pmm_config_test.go index 14abdb0cce..e67ed56c88 100644 --- a/managed/services/supervisord/pmm_config_test.go +++ b/managed/services/supervisord/pmm_config_test.go @@ -1,3 +1,18 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package supervisord import ( From cea1eda8372be42e21cac975ba94c9534656d210 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 11 Jul 2023 17:46:56 +0300 Subject: [PATCH 49/77] PMM-12293 fix linters. --- admin/cli/cli.go | 1 - admin/commands/reload.go | 56 ------------------- managed/cmd/pmm-managed/main.go | 4 +- managed/models/victoriametrics_params.go | 2 +- managed/services/agents/registry.go | 13 ++--- .../highavailability/highavailability.go | 14 ++--- .../highavailability/leaderservice.go | 45 +++------------ managed/utils/envvars/parser.go | 6 +- 8 files changed, 26 insertions(+), 115 deletions(-) delete mode 100644 admin/commands/reload.go diff --git a/admin/cli/cli.go b/admin/cli/cli.go index 42b600604f..36e5d4615f 100644 --- a/admin/cli/cli.go +++ b/admin/cli/cli.go @@ -49,7 +49,6 @@ type PMMAdminCommands struct { flags.GlobalFlags Status commands.StatusCommand `cmd:"" help:"Show information about local pmm-agent"` - Reload commands.ReloadCommand `cmd:"" help:"Show information about local pmm-agent"` Summary commands.SummaryCommand `cmd:"" help:"Fetch system data for diagnostics"` List commands.ListCommand `cmd:"" help:"Show Services and Agents running on this Node"` Config commands.ConfigCommand `cmd:"" help:"Configure local pmm-agent"` diff --git a/admin/commands/reload.go b/admin/commands/reload.go deleted file mode 100644 index fd990c81ed..0000000000 --- a/admin/commands/reload.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "time" - - "github.com/percona/pmm/api/agentlocalpb/json/client" - "github.com/percona/pmm/api/agentlocalpb/json/client/agent_local" -) - -var reloadResultT = ParseTemplate(` -Reloaded. -`) - -type reloadResult struct{} - -func (res *reloadResult) Result() {} - -func (res *reloadResult) String() string { - return RenderTemplate(reloadResultT, res) -} - -// ReloadCommand is used by Kong for CLI flags and commands. -type ReloadCommand struct { - Timeout time.Duration `name:"wait" help:"Time to wait for a successful response from pmm-agent"` -} - -// BeforeApply is run before the command is applied. -func (cmd *ReloadCommand) BeforeApply() error { - return nil -} - -// RunCmd runs the ReloadCommand. -func (cmd *ReloadCommand) RunCmd() (Result, error) { - _, err := client.Default.AgentLocal.Reload(&agent_local.ReloadParams{ - Context: Ctx, - }) - if err != nil { - return nil, err - } - - return &reloadResult{}, nil -} diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 3e6963a956..658d66979f 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -889,8 +889,8 @@ func main() { //nolint:cyclop,maintidx agentsRegistry := agents.NewRegistry(db, vmParams) // TODO remove once PMM cluster will be Active-Active - ha.AddLeaderService(highavailability.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() error { - return agentsRegistry.KickAll(ctx) + ha.AddLeaderService(highavailability.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() { + agentsRegistry.KickAll(ctx) })) pbmPITRService := backup.NewPBMPITRService() diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 30a38f8b27..f033b5a4f4 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -45,7 +45,7 @@ type VictoriaMetricsParams struct { // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { if !strings.HasSuffix(vmURL, "/") { - vmURL = vmURL + "/" + vmURL += "/" } URL, err := url.Parse(vmURL) diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 278c0f3bdf..cac3bae82e 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -72,7 +72,7 @@ type pmmAgentInfo interface { } type localPMMAgentInfo struct { - channel communicationChannel + channel *channel.Channel id string stateChangeChan chan struct{} kickChan chan struct{} @@ -99,7 +99,7 @@ type Registry struct { db *reform.DB rw sync.RWMutex - agents map[string]pmmAgentInfo // id -> info + agents map[string]*localPMMAgentInfo // id -> info roster *roster @@ -114,7 +114,7 @@ type Registry struct { // NewRegistry creates a new registry with given database connection. func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry { - agents := make(map[string]pmmAgentInfo) + agents := make(map[string]*localPMMAgentInfo) r := &Registry{ db: db, @@ -176,7 +176,7 @@ func (r *Registry) IsConnected(pmmAgentID string) bool { return err == nil } -func (r *Registry) register(stream agentpb.Agent_ConnectServer) (pmmAgentInfo, error) { +func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*localPMMAgentInfo, error) { ctx := stream.Context() l := logger.Get(ctx) r.mConnects.Inc() @@ -278,7 +278,7 @@ func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Quer } // unregister removes pmm-agent with given ID from the registry. -func (r *Registry) unregister(pmmAgentID, disconnectReason string) pmmAgentInfo { +func (r *Registry) unregister(pmmAgentID, disconnectReason string) *localPMMAgentInfo { r.mDisconnects.WithLabelValues(disconnectReason).Inc() r.rw.Lock() @@ -440,11 +440,10 @@ func (r *Registry) Collect(ch chan<- prom.Metric) { r.mClockDrift.Collect(ch) } -func (r *Registry) KickAll(ctx context.Context) error { +func (r *Registry) KickAll(ctx context.Context) { for _, agentInfo := range r.agents { r.Kick(ctx, agentInfo.ID()) } - return nil } // check interfaces. diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index ca33ef86cb..c2e729d3fa 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -24,7 +24,6 @@ import ( "sync" "time" - transport "github.com/Jille/raft-grpc-transport" "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/sirupsen/logrus" @@ -50,7 +49,6 @@ type Service struct { rw sync.RWMutex raftNode *raft.Raft memberlist *memberlist.Memberlist - tm *transport.Manager } func (s *Service) Apply(logEntry *raft.Log) interface{} { @@ -70,12 +68,10 @@ func (s *Service) Apply(logEntry *raft.Log) interface{} { } func (s *Service) Snapshot() (raft.FSMSnapshot, error) { - // TODO implement me return nil, nil } -func (s *Service) Restore(snapshot io.ReadCloser) error { - // TODO implement me +func (s *Service) Restore(_ io.ReadCloser) error { return nil } @@ -160,7 +156,7 @@ func (s *Service) Run(ctx context.Context) error { // Create the memberlist s.memberlist, err = memberlist.Create(memberlistConfig) if err != nil { - return fmt.Errorf("failed to create memberlist: %q", err) + return fmt.Errorf("failed to create memberlist: %w", err) } defer func() { err := s.memberlist.Leave(5 * time.Second) @@ -185,13 +181,13 @@ func (s *Service) Run(ctx context.Context) error { }, } if err := s.raftNode.BootstrapCluster(cfg).Error(); err != nil { - return fmt.Errorf("failed to bootstrap Raft cluster: %q", err) + return fmt.Errorf("failed to bootstrap Raft cluster: %w", err) } } if len(s.params.Nodes) > 0 { _, err := s.memberlist.Join(s.params.Nodes) if err != nil { - return fmt.Errorf("failed to join memberlist cluster: %q", err) + return fmt.Errorf("failed to join memberlist cluster: %w", err) } } s.wg.Add(1) @@ -229,6 +225,8 @@ func (s *Service) runRaftNodesSynchronizer(ctx context.Context) { s.addMemberlistNodeToRaft(node) case memberlist.NodeLeave: s.removeMemberlistNodeFromRaft(node) + case memberlist.NodeUpdate: + continue } case <-t.C: if !s.IsLeader() { diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 2e42b01671..0582daaf61 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -22,7 +22,7 @@ import ( type LeaderService interface { Start(ctx context.Context) error - Stop() error + Stop() ID() string } @@ -30,10 +30,10 @@ type StandardService struct { id string startFunc func(context.Context) error - stopFunc func() error + stopFunc func() } -func NewStandardService(id string, startFunc func(context.Context) error, stopFunc func() error) *StandardService { +func NewStandardService(id string, startFunc func(context.Context) error, stopFunc func()) *StandardService { return &StandardService{ id: id, startFunc: startFunc, @@ -49,8 +49,8 @@ func (s *StandardService) Start(ctx context.Context) error { return s.startFunc(ctx) } -func (s *StandardService) Stop() error { - return s.stopFunc() +func (s *StandardService) Stop() { + s.stopFunc() } type ContextService struct { @@ -78,42 +78,11 @@ func (s *ContextService) Start(ctx context.Context) error { s.m.Lock() s.ctx, s.cancel = context.WithCancel(ctx) s.m.Unlock() - return s.startFunc(ctx) + return s.startFunc(s.ctx) } -func (s *ContextService) Stop() error { +func (s *ContextService) Stop() { s.m.Lock() defer s.m.Unlock() s.cancel() - return nil -} - -type RunOnceService struct { - id string - - startFunc func(context.Context) error - o sync.Once -} - -func NewRunOnceService(id string, startFunc func(context.Context) error) *RunOnceService { - return &RunOnceService{ - id: id, - startFunc: startFunc, - } -} - -func (s *RunOnceService) ID() string { - return s.id -} - -func (s *RunOnceService) Start(ctx context.Context) error { - var err error - s.o.Do(func() { - err = s.startFunc(ctx) - }) - return err -} - -func (s *RunOnceService) Stop() error { - return nil } diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 2f70388254..42483155b2 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -69,8 +69,10 @@ func (e InvalidDurationError) Error() string { return string(e) } // - the environment variables prefixed with GF_ passed as related to Grafana. // - the environment variables relating to proxies // - the environment variable set by podman -func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs []error, warns []string) { //nolint:cyclop,nonamedreturns - envSettings = &models.ChangeSettingsParams{} +func ParseEnvVars(envs []string) (*models.ChangeSettingsParams, []error, []string) { //nolint:cyclop,nonamedreturns,maintidx + envSettings := &models.ChangeSettingsParams{} + var errs []error + var warns []string for _, env := range envs { p := strings.SplitN(env, "=", 2) From 77ae9c45a64b6cd8a0674b9f84dd9158c5205147 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 11 Jul 2023 21:32:39 +0300 Subject: [PATCH 50/77] PMM-12078 go mod tidy --- go.mod | 1 - go.sum | 27 --------------------------- 2 files changed, 28 deletions(-) diff --git a/go.mod b/go.mod index 49db0db898..0c38c0edb4 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000 github.com/ClickHouse/clickhouse-go/v2 v2.10.0 github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/Jille/raft-grpc-transport v1.4.0 github.com/alecthomas/kong v0.8.0 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 diff --git a/go.sum b/go.sum index 4016d99b9e..11f2e8eda7 100644 --- a/go.sum +++ b/go.sum @@ -76,12 +76,9 @@ github.com/ClickHouse/clickhouse-go/v2 v2.10.0 h1:0w/A50D5MfsRUYBaV6rLKwZ4LXWKLZ github.com/ClickHouse/clickhouse-go/v2 v2.10.0/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/Jille/raft-grpc-transport v1.4.0 h1:Kwk+IceQD8MpLKOulBu2ignX+aZAEjOhffEhN44sdzQ= -github.com/Jille/raft-grpc-transport v1.4.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -112,8 +109,6 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -222,7 +217,6 @@ github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= @@ -435,8 +429,6 @@ github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -460,12 +452,10 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= -github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.5.0 h1:uNs9EfJ4FwiArZRxxfd/dQ5d33nV31/CdCHArH89hT8= github.com/hashicorp/raft v1.5.0/go.mod h1:pKHB2mf/Y25u3AHNSXVRv+yT+WAnmeTX0BwVppVQV+M= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -551,9 +541,7 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -561,8 +549,6 @@ github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqf github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -706,7 +692,6 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/alertmanager v0.25.0 h1:vbXKUR6PYRiZPRIKfmXaG+dmCKG52RtPL4Btl8hQGvg= github.com/prometheus/alertmanager v0.25.0/go.mod h1:MEZ3rFVHqKZsw7IcNS/m4AWZeXThmJhumpiWR4eHU/w= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -719,7 +704,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -733,7 +717,6 @@ github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57J github.com/prometheus/exporter-toolkit v0.8.2 h1:sbJAfBXQFkG6sUkbwBun8MNdzW9+wd5YfPYofbmj0YM= github.com/prometheus/exporter-toolkit v0.8.2/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -861,8 +844,6 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= @@ -895,7 +876,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -912,7 +892,6 @@ golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -946,7 +925,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= @@ -980,7 +958,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -994,7 +971,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,7 +1003,6 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1120,7 +1095,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= @@ -1190,7 +1164,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= From d7267be186509dc9d79757abab5eb36889266c6a Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 18 Jul 2023 09:47:41 +0300 Subject: [PATCH 51/77] PMM-12078 Fix linters --- agent/config/config.go | 2 +- managed/services/agents/registry.go | 4 ++-- managed/services/highavailability/highavailability.go | 3 +-- managed/services/highavailability/leaderservice.go | 5 ++--- managed/utils/envvars/parser.go | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/agent/config/config.go b/agent/config/config.go index eebca14167..afa49335be 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -182,7 +182,7 @@ func getFromCmdLine(cfg *Config, l *logrus.Entry) (string, error) { func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err error) { //nolint:nonamedreturns // tweak configuration on exit to cover all return points defer func() { - if cfg == nil { + if cfg == nil || err != nil { return } diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index cac3bae82e..96e2f38d98 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -78,7 +78,7 @@ type localPMMAgentInfo struct { kickChan chan struct{} } -func (p *localPMMAgentInfo) Channel() communicationChannel { +func (p *localPMMAgentInfo) Channel() communicationChannel { //nolint:ireturn return p.channel } @@ -400,7 +400,7 @@ func (r *Registry) Kick(ctx context.Context, pmmAgentID string) { // closing agent.kickChan is enough to exit runStateChangeHandler goroutine. } -func (r *Registry) get(pmmAgentID string) (pmmAgentInfo, error) { +func (r *Registry) get(pmmAgentID string) (pmmAgentInfo, error) { //nolint:ireturn r.rw.RLock() pmmAgent := r.agents[pmmAgentID] r.rw.RUnlock() diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index c2e729d3fa..5be2ba4699 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -61,13 +61,12 @@ func (s *Service) Apply(logEntry *raft.Log) interface{} { switch m.Payload.(type) { case *hapb.ClusterMessage_ServerMessage: case *hapb.ClusterMessage_AgentMessage: - } s.receivedMessages <- logEntry.Data return nil } -func (s *Service) Snapshot() (raft.FSMSnapshot, error) { +func (s *Service) Snapshot() (raft.FSMSnapshot, error) { //nolint:ireturn return nil, nil } diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 0582daaf61..05d909e4ca 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -57,7 +57,6 @@ type ContextService struct { id string startFunc func(context.Context) error - ctx context.Context cancel context.CancelFunc m sync.Mutex @@ -76,9 +75,9 @@ func (s *ContextService) ID() string { func (s *ContextService) Start(ctx context.Context) error { s.m.Lock() - s.ctx, s.cancel = context.WithCancel(ctx) + ctx, s.cancel = context.WithCancel(ctx) s.m.Unlock() - return s.startFunc(s.ctx) + return s.startFunc(ctx) } func (s *ContextService) Stop() { diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 42483155b2..d620bcf77e 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -69,7 +69,7 @@ func (e InvalidDurationError) Error() string { return string(e) } // - the environment variables prefixed with GF_ passed as related to Grafana. // - the environment variables relating to proxies // - the environment variable set by podman -func ParseEnvVars(envs []string) (*models.ChangeSettingsParams, []error, []string) { //nolint:cyclop,nonamedreturns,maintidx +func ParseEnvVars(envs []string) (*models.ChangeSettingsParams, []error, []string) { //nolint:cyclop,maintidx envSettings := &models.ChangeSettingsParams{} var errs []error var warns []string From 3e9448660803c98210a13612c47587b247df5cfa Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 18 Jul 2023 10:12:51 +0300 Subject: [PATCH 52/77] PMM-12078 Fix linters. --- managed/services/highavailability/highavailability.go | 2 +- managed/services/highavailability/services.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index 5be2ba4699..10eb1a1703 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -183,7 +183,7 @@ func (s *Service) Run(ctx context.Context) error { return fmt.Errorf("failed to bootstrap Raft cluster: %w", err) } } - if len(s.params.Nodes) > 0 { + if len(s.params.Nodes) != 0 { _, err := s.memberlist.Join(s.params.Nodes) if err != nil { return fmt.Errorf("failed to join memberlist cluster: %w", err) diff --git a/managed/services/highavailability/services.go b/managed/services/highavailability/services.go index e3772d1a02..90b3e2d04f 100644 --- a/managed/services/highavailability/services.go +++ b/managed/services/highavailability/services.go @@ -36,7 +36,7 @@ type services struct { func newServices() *services { return &services{ - wg: new(sync.WaitGroup), + wg: &sync.WaitGroup{}, all: make(map[string]LeaderService), running: make(map[string]LeaderService), serviceAdded: make(chan struct{}), From 1948914ff0645eec7f28852acd26aea78c1cbdd7 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 18 Jul 2023 15:43:00 +0300 Subject: [PATCH 53/77] PMM-12078 revert changes. --- agent/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/config/config.go b/agent/config/config.go index afa49335be..eebca14167 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -182,7 +182,7 @@ func getFromCmdLine(cfg *Config, l *logrus.Entry) (string, error) { func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err error) { //nolint:nonamedreturns // tweak configuration on exit to cover all return points defer func() { - if cfg == nil || err != nil { + if cfg == nil { return } From d709b76402e5297c551bc87c446f1d89db5689e0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 19 Jul 2023 00:20:58 +0300 Subject: [PATCH 54/77] PMM-12078 don't run pmm-agent on passive nodes. --- .devcontainer/setup.py | 2 +- agent/agents/process/process_logger_test.go | 14 --- managed/cmd/pmm-managed-init/main.go | 1 - managed/cmd/pmm-managed/main.go | 14 +-- managed/models/params.go | 2 + managed/services/agents/registry.go | 4 +- managed/services/supervisord/pmm_config.go | 4 +- .../services/supervisord/pmm_config_test.go | 9 +- managed/services/supervisord/supervisord.go | 12 +++ managed/testdata/pg/queries/grafana.sql | 2 +- .../testdata/supervisord.d/pmm-passive.ini | 89 ------------------- 11 files changed, 30 insertions(+), 123 deletions(-) delete mode 100644 managed/testdata/supervisord.d/pmm-passive.ini diff --git a/.devcontainer/setup.py b/.devcontainer/setup.py index bb2c043a67..f57267ac07 100755 --- a/.devcontainer/setup.py +++ b/.devcontainer/setup.py @@ -96,7 +96,7 @@ def setup(): def main(): install_packages() install_go() - # make_init() + make_init() # do basic setup # TODO: fix the setup and revert diff --git a/agent/agents/process/process_logger_test.go b/agent/agents/process/process_logger_test.go index 9117496d12..ccf7a53cf2 100644 --- a/agent/agents/process/process_logger_test.go +++ b/agent/agents/process/process_logger_test.go @@ -127,20 +127,6 @@ func TestProcessLogger(t *testing.T) { 0, 0, }, - { - "redact keywords with special symbols", - 3, - []string{ - "text\nsecond ", - "line\nthird r.o,w line\n", - "fourth ", - "line\nlast r.o,w\n", - }, - []string{"r.o,w"}, - []string{"third *** line", "fourth line", "last ***"}, - 0, - 0, - }, } for _, tt := range tests { t.Run(tt.testName, func(t *testing.T) { diff --git a/managed/cmd/pmm-managed-init/main.go b/managed/cmd/pmm-managed-init/main.go index 3e61334248..bed09643bb 100644 --- a/managed/cmd/pmm-managed-init/main.go +++ b/managed/cmd/pmm-managed-init/main.go @@ -55,7 +55,6 @@ func main() { pmmConfigParams := make(map[string]any) pmmConfigParams["DisableInternalDB"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES")) pmmConfigParams["DisableInternalClickhouse"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE")) - pmmConfigParams["PassivePMM"], _ = strconv.ParseBool(os.Getenv("PERCONA_TEST_HA_PASSIVE")) if err := supervisord.SavePMMConfig(pmmConfigParams); err != nil { logrus.Errorf("PMM Server configuration error: %s.", err) os.Exit(1) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 658d66979f..dc4752ca29 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -252,7 +252,6 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { l.Debug("RPC response latency histogram enabled.") grpcMetrics.EnableHandlingTimeHistogram() } - serverpb.RegisterServerServer(gRPCServer, deps.server) agentpb.RegisterAgentServer(gRPCServer, agentgrpc.NewAgentServer(deps.handler)) @@ -747,10 +746,6 @@ func main() { //nolint:cyclop,maintidx Envar("PERCONA_TEST_GRAFANA_GOSSIP_PORT"). Default("9762"). Int() - // haGrafanaPort := kingpin.Flag("ha-grafana-port", "HA grafana port"). - // Envar("PERCONA_TEST_HA_GRAFANA_PORT"). - // Default("9760"). - // Int() supervisordConfigDirF := kingpin.Flag("supervisord-config-dir", "Supervisord configuration directory").Required().String() @@ -932,6 +927,15 @@ func main() { //nolint:cyclop,maintidx }, gRPCMessageMaxSize) + ha.AddLeaderService(highavailability.NewStandardService("pmm-agent-runner", func(ctx context.Context) error { + return supervisord.StartSupervisedService("pmm-agent") + }, func() { + err := supervisord.StopSupervisedService("pmm-agent") + if err != nil { + l.Warnf("couldn't stop pmm-agent: %q", err) + } + })) + platformAddress, err := envvars.GetPlatformAddress() if err != nil { l.Fatal(err) diff --git a/managed/models/params.go b/managed/models/params.go index 2be506c3c5..588ae8efa9 100644 --- a/managed/models/params.go +++ b/managed/models/params.go @@ -15,6 +15,7 @@ package models +// HAParams defines parameters related to High Availability. type HAParams struct { GrafanaGossipPort int Enabled bool @@ -26,6 +27,7 @@ type HAParams struct { GossipPort int } +// Params defines parameters for supervisor. type Params struct { HAParams *HAParams VMParams *VictoriaMetricsParams diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 96e2f38d98..b3ca08631f 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -65,7 +65,7 @@ var ( ) type pmmAgentInfo interface { - Channel() communicationChannel + Channel() *channel.Channel ID() string StateChangeChan() chan struct{} KickChan() chan struct{} @@ -78,7 +78,7 @@ type localPMMAgentInfo struct { kickChan chan struct{} } -func (p *localPMMAgentInfo) Channel() communicationChannel { //nolint:ireturn +func (p *localPMMAgentInfo) Channel() *channel.Channel { //nolint:ireturn return p.channel } diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index c62becd409..e42db15e93 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -179,13 +179,12 @@ stdout_logfile = /srv/logs/pmm-managed.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true -{{- if not .PassivePMM }} [program:pmm-agent] priority = 15 command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml autorestart = true -autostart = true +autostart = false startretries = 1000 startsecs = 1 stopsignal = TERM @@ -194,7 +193,6 @@ stdout_logfile = /srv/logs/pmm-agent.log stdout_logfile_maxbytes = 50MB stdout_logfile_backups = 2 redirect_stderr = true -{{- end }} [program:pmm-update-perform] command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml diff --git a/managed/services/supervisord/pmm_config_test.go b/managed/services/supervisord/pmm_config_test.go index e67ed56c88..ff95d36cd4 100644 --- a/managed/services/supervisord/pmm_config_test.go +++ b/managed/services/supervisord/pmm_config_test.go @@ -34,19 +34,14 @@ func TestSavePMMConfig(t *testing.T) { }{ { description: "disable internal postgresql db", - params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "DisableInternalClickhouse": false, "PassivePMM": false}, + params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "DisableInternalClickhouse": false}, file: "pmm-db_disabled", }, { description: "enable internal postgresql db", - params: map[string]any{"DisableInternalDB": false, "DisableSupervisor": false, "DisableInternalClickhouse": false, "PassivePMM": false}, + params: map[string]any{"DisableInternalDB": false, "DisableSupervisor": false, "DisableInternalClickhouse": false}, file: "pmm-db_enabled", }, - { - description: "passive pmm", - params: map[string]any{"DisableInternalDB": true, "DisableSupervisor": false, "DisableInternalClickhouse": false, "PassivePMM": true}, - file: "pmm-passive", - }, } for _, test := range tests { test := test diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index b40ff1d081..3ca50828b6 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -625,6 +625,18 @@ func (s *Service) RestartSupervisedService(serviceName string) error { return err } +// StartSupervisedService starts given service. +func (s *Service) StartSupervisedService(serviceName string) error { + _, err := s.supervisorctl("start", serviceName) + return err +} + +// StopSupervisedService stops given service. +func (s *Service) StopSupervisedService(serviceName string) error { + _, err := s.supervisorctl("stop", serviceName) + return err +} + //nolint:lll var templates = template.Must(template.New("").Option("missingkey=error").Parse(` {{define "dbaas-controller"}} diff --git a/managed/testdata/pg/queries/grafana.sql b/managed/testdata/pg/queries/grafana.sql index 44b1d4bf72..5bdbb0b4c8 100644 --- a/managed/testdata/pg/queries/grafana.sql +++ b/managed/testdata/pg/queries/grafana.sql @@ -1,3 +1,3 @@ CREATE DATABASE "grafana"; CREATE USER "grafana" WITH PASSWORD 'grafana'; -GRANT ALL PRIVILEGES ON DATABASE "grafana" TO "grafana" +GRANT ALL PRIVILEGES ON DATABASE "grafana" TO "grafana"; \ No newline at end of file diff --git a/managed/testdata/supervisord.d/pmm-passive.ini b/managed/testdata/supervisord.d/pmm-passive.ini deleted file mode 100644 index c89b8bc9e3..0000000000 --- a/managed/testdata/supervisord.d/pmm-passive.ini +++ /dev/null @@ -1,89 +0,0 @@ - -[unix_http_server] -chmod = 0700 -username = dummy -password = dummy - -[supervisorctl] -username = dummy -password = dummy - -[program:pmm-update-perform-init] -command = /usr/sbin/pmm-update -run-playbook -playbook=/usr/share/pmm-update/ansible/playbook/tasks/init.yml -directory = / -autorestart = unexpected -priority=-1 -exitcodes = 0 -autostart = true -startretries = 3 -startsecs = 1 -stopsignal = TERM -stopwaitsecs = 300 -stdout_logfile = /srv/logs/pmm-update-perform-init.log -stdout_logfile_maxbytes = 50MB -stdout_logfile_backups = 3 -redirect_stderr = true - -[program:clickhouse] -priority = 2 -command = /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml -autorestart = true -autostart = true -startretries = 10 -startsecs = 1 -stopsignal = TERM -stopwaitsecs = 300 -; config.xml contains settings to log to stdout (console), -; so we delegate logfile managemenet to supervisord -stdout_logfile = /srv/logs/clickhouse-server.log -stdout_logfile_maxbytes = 50MB -stdout_logfile_backups = 2 -redirect_stderr = true - -[program:nginx] -priority = 4 -command = nginx -autorestart = true -autostart = true -startretries = 10 -startsecs = 1 -stopsignal = TERM -stopwaitsecs = 10 -; nginx.conf contains settings to log to /dev/sdtout and /dev/stderr, -; which allows supervisord to manage the logs further. -stdout_logfile = /srv/logs/nginx.log -stdout_logfile_maxbytes = 50MB -stdout_logfile_backups = 2 -redirect_stderr = true - -[program:pmm-managed] -priority = 14 -command = - /usr/sbin/pmm-managed - --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --supervisord-config-dir=/etc/supervisord.d -autorestart = true -autostart = true -startretries = 1000 -startsecs = 1 -stopsignal = TERM -stopwaitsecs = 300 -stdout_logfile = /srv/logs/pmm-managed.log -stdout_logfile_maxbytes = 50MB -stdout_logfile_backups = 2 -redirect_stderr = true - -[program:pmm-update-perform] -command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml -directory = / -autorestart = unexpected -exitcodes = 0 -autostart = false -startretries = 10 -startsecs = 1 -stopsignal = TERM -stopwaitsecs = 300 -stdout_logfile = /srv/logs/pmm-update-perform.log -stdout_logfile_maxbytes = 50MB -stdout_logfile_backups = 3 -redirect_stderr = true From 0aa45ffc13396b30c43dabe1086333f2cc2e5a7e Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 19 Jul 2023 11:58:49 +0300 Subject: [PATCH 55/77] PMM-12078 Fix tests and linters. --- managed/cmd/pmm-managed-starlark/main_test.go | 8 +++++++- managed/services/agents/deps.go | 9 --------- managed/services/agents/registry.go | 2 +- managed/testdata/supervisord.d/pmm-db_disabled.ini | 2 +- managed/testdata/supervisord.d/pmm-db_enabled.ini | 2 +- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/managed/cmd/pmm-managed-starlark/main_test.go b/managed/cmd/pmm-managed-starlark/main_test.go index f52f76ce86..fcabbb8f0a 100644 --- a/managed/cmd/pmm-managed-starlark/main_test.go +++ b/managed/cmd/pmm-managed-starlark/main_test.go @@ -17,10 +17,12 @@ package main import ( "bytes" + "context" "encoding/json" "os" "os/exec" "testing" + "time" "github.com/percona-platform/saas/pkg/check" "github.com/stretchr/testify/assert" @@ -103,8 +105,12 @@ func TestStarlarkSandbox(t *testing.T) { //nolint:tparallel }, } + ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) // since we run the binary as a child process to test it we need to build it first. - err := exec.Command("make", "-C", "../..", "release").Run() + command := exec.CommandContext(ctx, "make", "-C", "../..", "release") + command.Stdout = os.Stdout + command.Stderr = os.Stderr + err := command.Run() require.NoError(t, err) for _, tc := range testCases { diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index 14ab6d6b49..e164663985 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -23,7 +23,6 @@ import ( "github.com/percona/pmm/api/agentpb" qanpb "github.com/percona/pmm/api/qanpb" - "github.com/percona/pmm/managed/services/agents/channel" ) // prometheusService is a subset of methods of victoriametrics.Service used by this package. @@ -69,11 +68,3 @@ type highAvailablityService interface { // TODO: extend by send message BroadcastMessage(message []byte) } - -type communicationChannel interface { - Requests() <-chan *channel.AgentRequest - Send(resp *channel.ServerResponse) - SendAndWaitResponse(payload agentpb.ServerRequestPayload) (agentpb.AgentResponsePayload, error) - Metrics() *channel.Metrics - Wait() error -} diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index b3ca08631f..523e1eb996 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -78,7 +78,7 @@ type localPMMAgentInfo struct { kickChan chan struct{} } -func (p *localPMMAgentInfo) Channel() *channel.Channel { //nolint:ireturn +func (p *localPMMAgentInfo) Channel() *channel.Channel { return p.channel } diff --git a/managed/testdata/supervisord.d/pmm-db_disabled.ini b/managed/testdata/supervisord.d/pmm-db_disabled.ini index b45251ab1f..8e1cb82648 100644 --- a/managed/testdata/supervisord.d/pmm-db_disabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_disabled.ini @@ -77,7 +77,7 @@ redirect_stderr = true priority = 15 command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml autorestart = true -autostart = true +autostart = false startretries = 1000 startsecs = 1 stopsignal = TERM diff --git a/managed/testdata/supervisord.d/pmm-db_enabled.ini b/managed/testdata/supervisord.d/pmm-db_enabled.ini index 4d4ad4afd9..4ce13c4281 100644 --- a/managed/testdata/supervisord.d/pmm-db_enabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_enabled.ini @@ -101,7 +101,7 @@ redirect_stderr = true priority = 15 command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml autorestart = true -autostart = true +autostart = false startretries = 1000 startsecs = 1 stopsignal = TERM From d86b64bd2f79a0c6022f6c2e9c2344c24ac43719 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 21 Jul 2023 11:37:42 +0300 Subject: [PATCH 56/77] PMM-12078 Fix tests and linters. --- managed/cmd/pmm-managed-starlark/main_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/managed/cmd/pmm-managed-starlark/main_test.go b/managed/cmd/pmm-managed-starlark/main_test.go index fcabbb8f0a..72e23b788c 100644 --- a/managed/cmd/pmm-managed-starlark/main_test.go +++ b/managed/cmd/pmm-managed-starlark/main_test.go @@ -105,7 +105,8 @@ func TestStarlarkSandbox(t *testing.T) { //nolint:tparallel }, } - ctx, _ := context.WithTimeout(context.Background(), 120*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + t.Cleanup(cancel) // since we run the binary as a child process to test it we need to build it first. command := exec.CommandContext(ctx, "make", "-C", "../..", "release") command.Stdout = os.Stdout From 13ed0d315c1d066ba52859372df2e489b7347727 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 21 Jul 2023 12:57:37 +0300 Subject: [PATCH 57/77] PMM-12078 Fix starlark test. --- managed/Makefile | 4 ++++ managed/cmd/pmm-managed-starlark/main_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/managed/Makefile b/managed/Makefile index 9412255408..5c3722002d 100644 --- a/managed/Makefile +++ b/managed/Makefile @@ -36,6 +36,10 @@ release: ## Build pmm-managed release binaries env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/... $(PMM_RELEASE_PATH)/pmm-managed --version +release-starlark: + env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-managed-starlark/... + $(PMM_RELEASE_PATH)/pmm-managed-starlark --version + ARCH=$(shell uname -m) release-dev: ## Build pmm-managed binaries for development if [ $(ARCH) = "aarch64" ]; then \ diff --git a/managed/cmd/pmm-managed-starlark/main_test.go b/managed/cmd/pmm-managed-starlark/main_test.go index 72e23b788c..1eaa23d17b 100644 --- a/managed/cmd/pmm-managed-starlark/main_test.go +++ b/managed/cmd/pmm-managed-starlark/main_test.go @@ -108,7 +108,7 @@ func TestStarlarkSandbox(t *testing.T) { //nolint:tparallel ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) t.Cleanup(cancel) // since we run the binary as a child process to test it we need to build it first. - command := exec.CommandContext(ctx, "make", "-C", "../..", "release") + command := exec.CommandContext(ctx, "make", "-C", "../..", "release-starlark") command.Stdout = os.Stdout command.Stderr = os.Stderr err := command.Run() From 2f8fc85becf02b9df627f2061f4bb4337b7d4811 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 31 Aug 2023 00:29:03 +0300 Subject: [PATCH 58/77] PMM-9374 fix supervisord for external VM. --- managed/services/supervisord/supervisord.go | 2 +- managed/utils/envvars/parser.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 23efd6fac4..40ac1f617b 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -579,7 +579,7 @@ func (s *Service) UpdateConfiguration(settings *models.Settings, ssoDetails *mod } for _, tmpl := range templates.Templates() { - if tmpl.Name() == "" { + if tmpl.Name() == "" || (tmpl.Name() == "victoriametrics" && s.vmParams.ExternalVM()) { continue } diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 2f70388254..4a5ac40146 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -157,6 +157,12 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs case "PMM_PUBLIC_ADDRESS": envSettings.PMMPublicAddress = v + case "PMM_VM_URL": + _, err := url.Parse(v) + if err != nil { + err = fmt.Errorf("invalid value %q for environment variable %q", v, k) + } + case "NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY": continue From a79a4fa68c8e68a7245ea4004a0e6ef15283ddad Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 5 Sep 2023 11:59:38 +0300 Subject: [PATCH 59/77] PMM-9374 Fix linters. --- managed/models/victoriametrics_params.go | 2 +- managed/utils/envvars/parser.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/models/victoriametrics_params.go b/managed/models/victoriametrics_params.go index 30a38f8b27..f033b5a4f4 100644 --- a/managed/models/victoriametrics_params.go +++ b/managed/models/victoriametrics_params.go @@ -45,7 +45,7 @@ type VictoriaMetricsParams struct { // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { if !strings.HasSuffix(vmURL, "/") { - vmURL = vmURL + "/" + vmURL += "/" } URL, err := url.Parse(vmURL) diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 4a5ac40146..7847419984 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -158,7 +158,7 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs envSettings.PMMPublicAddress = v case "PMM_VM_URL": - _, err := url.Parse(v) + _, err = url.Parse(v) if err != nil { err = fmt.Errorf("invalid value %q for environment variable %q", v, k) } From 5a6a5a2c4201943ae7879256ae0956d114312b84 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 8 Sep 2023 12:16:23 +0300 Subject: [PATCH 60/77] PMM-9374 Don't run victoria metrics if it's external. --- managed/services/supervisord/supervisord.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 1e66d8bded..7172bbe8fa 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -394,20 +394,10 @@ func (s *Service) UpdateLog(offset uint32) ([]string, uint32, error) { // reload asks supervisord to reload configuration. func (s *Service) reload(name string) error { - // See https://github.com/Supervisor/supervisor/issues/1264 for explanation - // why we do reread + stop/remove/add. - if _, err := s.supervisorctl("reread"); err != nil { s.l.Warn(err) } - if _, err := s.supervisorctl("stop", name); err != nil { - s.l.Warn(err) - } - if _, err := s.supervisorctl("remove", name); err != nil { - s.l.Warn(err) - } - - _, err := s.supervisorctl("add", name) + _, err := s.supervisorctl("update", name) return err } @@ -592,7 +582,7 @@ func (s *Service) UpdateConfiguration(settings *models.Settings, ssoDetails *mod } for _, tmpl := range templates.Templates() { - if tmpl.Name() == "" || (tmpl.Name() == "victoriametrics" && s.vmParams.ExternalVM()) { + if tmpl.Name() == "" { continue } From 5481cb554a9b1c482c973503b758838527d9dd54 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 18 Sep 2023 17:39:22 +0300 Subject: [PATCH 61/77] PMM-12078 revert changes made for active-active connection. --- api/hapb/ha.pb.go | 295 --------------- api/hapb/ha.pb.validate.go | 357 ------------------ api/hapb/ha.proto | 21 -- go.sum | 3 +- managed/cmd/pmm-managed/main.go | 2 +- managed/services/agents/actions.go | 42 +-- managed/services/agents/agents.go | 4 +- managed/services/agents/connection_checker.go | 2 +- managed/services/agents/deps.go | 6 - managed/services/agents/handler.go | 28 +- managed/services/agents/jobs.go | 12 +- managed/services/agents/registry.go | 55 +-- managed/services/agents/state.go | 28 +- managed/services/agents/versioner.go | 2 +- .../highavailability/highavailability.go | 11 - 15 files changed, 76 insertions(+), 792 deletions(-) delete mode 100644 api/hapb/ha.pb.go delete mode 100644 api/hapb/ha.pb.validate.go delete mode 100644 api/hapb/ha.proto diff --git a/api/hapb/ha.pb.go b/api/hapb/ha.pb.go deleted file mode 100644 index 4750b4792e..0000000000 --- a/api/hapb/ha.pb.go +++ /dev/null @@ -1,295 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.31.0 -// protoc (unknown) -// source: hapb/ha.proto - -package hapb - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - - agentpb "github.com/percona/pmm/api/agentpb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Metadata struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Sender string `protobuf:"bytes,1,opt,name=sender,proto3" json:"sender,omitempty"` - Receiver string `protobuf:"bytes,2,opt,name=receiver,proto3" json:"receiver,omitempty"` -} - -func (x *Metadata) Reset() { - *x = Metadata{} - if protoimpl.UnsafeEnabled { - mi := &file_hapb_ha_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Metadata) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Metadata) ProtoMessage() {} - -func (x *Metadata) ProtoReflect() protoreflect.Message { - mi := &file_hapb_ha_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Metadata.ProtoReflect.Descriptor instead. -func (*Metadata) Descriptor() ([]byte, []int) { - return file_hapb_ha_proto_rawDescGZIP(), []int{0} -} - -func (x *Metadata) GetSender() string { - if x != nil { - return x.Sender - } - return "" -} - -func (x *Metadata) GetReceiver() string { - if x != nil { - return x.Receiver - } - return "" -} - -type ClusterMessage struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Metadata *Metadata `protobuf:"bytes,2,opt,name=metadata,proto3" json:"metadata,omitempty"` - // Types that are assignable to Payload: - // - // *ClusterMessage_ServerMessage - // *ClusterMessage_AgentMessage - Payload isClusterMessage_Payload `protobuf_oneof:"payload"` -} - -func (x *ClusterMessage) Reset() { - *x = ClusterMessage{} - if protoimpl.UnsafeEnabled { - mi := &file_hapb_ha_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ClusterMessage) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ClusterMessage) ProtoMessage() {} - -func (x *ClusterMessage) ProtoReflect() protoreflect.Message { - mi := &file_hapb_ha_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ClusterMessage.ProtoReflect.Descriptor instead. -func (*ClusterMessage) Descriptor() ([]byte, []int) { - return file_hapb_ha_proto_rawDescGZIP(), []int{1} -} - -func (x *ClusterMessage) GetId() uint32 { - if x != nil { - return x.Id - } - return 0 -} - -func (x *ClusterMessage) GetMetadata() *Metadata { - if x != nil { - return x.Metadata - } - return nil -} - -func (m *ClusterMessage) GetPayload() isClusterMessage_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *ClusterMessage) GetServerMessage() *agentpb.ServerMessage { - if x, ok := x.GetPayload().(*ClusterMessage_ServerMessage); ok { - return x.ServerMessage - } - return nil -} - -func (x *ClusterMessage) GetAgentMessage() *agentpb.AgentMessage { - if x, ok := x.GetPayload().(*ClusterMessage_AgentMessage); ok { - return x.AgentMessage - } - return nil -} - -type isClusterMessage_Payload interface { - isClusterMessage_Payload() -} - -type ClusterMessage_ServerMessage struct { - ServerMessage *agentpb.ServerMessage `protobuf:"bytes,11,opt,name=server_message,json=serverMessage,proto3,oneof"` -} - -type ClusterMessage_AgentMessage struct { - AgentMessage *agentpb.AgentMessage `protobuf:"bytes,12,opt,name=agent_message,json=agentMessage,proto3,oneof"` -} - -func (*ClusterMessage_ServerMessage) isClusterMessage_Payload() {} - -func (*ClusterMessage_AgentMessage) isClusterMessage_Payload() {} - -var File_hapb_ha_proto protoreflect.FileDescriptor - -var file_hapb_ha_proto_rawDesc = []byte{ - 0x0a, 0x0d, 0x68, 0x61, 0x70, 0x62, 0x2f, 0x68, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x02, 0x68, 0x61, 0x1a, 0x13, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x70, 0x62, 0x2f, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3e, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, - 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x22, 0xd0, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x08, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, - 0x68, 0x61, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3d, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x12, 0x3a, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x48, 0x00, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x5a, 0x0a, 0x06, 0x63, - 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x42, 0x07, 0x48, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, - 0x5a, 0x1f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x65, 0x72, - 0x63, 0x6f, 0x6e, 0x61, 0x2f, 0x70, 0x6d, 0x6d, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x61, 0x70, - 0x62, 0xa2, 0x02, 0x03, 0x48, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x48, 0x61, 0xca, 0x02, 0x02, 0x48, - 0x61, 0xe2, 0x02, 0x0e, 0x48, 0x61, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0xea, 0x02, 0x02, 0x48, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_hapb_ha_proto_rawDescOnce sync.Once - file_hapb_ha_proto_rawDescData = file_hapb_ha_proto_rawDesc -) - -func file_hapb_ha_proto_rawDescGZIP() []byte { - file_hapb_ha_proto_rawDescOnce.Do(func() { - file_hapb_ha_proto_rawDescData = protoimpl.X.CompressGZIP(file_hapb_ha_proto_rawDescData) - }) - return file_hapb_ha_proto_rawDescData -} - -var ( - file_hapb_ha_proto_msgTypes = make([]protoimpl.MessageInfo, 2) - file_hapb_ha_proto_goTypes = []interface{}{ - (*Metadata)(nil), // 0: ha.Metadata - (*ClusterMessage)(nil), // 1: ha.ClusterMessage - (*agentpb.ServerMessage)(nil), // 2: agent.ServerMessage - (*agentpb.AgentMessage)(nil), // 3: agent.AgentMessage - } -) - -var file_hapb_ha_proto_depIdxs = []int32{ - 0, // 0: ha.ClusterMessage.metadata:type_name -> ha.Metadata - 2, // 1: ha.ClusterMessage.server_message:type_name -> agent.ServerMessage - 3, // 2: ha.ClusterMessage.agent_message:type_name -> agent.AgentMessage - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name -} - -func init() { file_hapb_ha_proto_init() } -func file_hapb_ha_proto_init() { - if File_hapb_ha_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_hapb_ha_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Metadata); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_hapb_ha_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ClusterMessage); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_hapb_ha_proto_msgTypes[1].OneofWrappers = []interface{}{ - (*ClusterMessage_ServerMessage)(nil), - (*ClusterMessage_AgentMessage)(nil), - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_hapb_ha_proto_rawDesc, - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_hapb_ha_proto_goTypes, - DependencyIndexes: file_hapb_ha_proto_depIdxs, - MessageInfos: file_hapb_ha_proto_msgTypes, - }.Build() - File_hapb_ha_proto = out.File - file_hapb_ha_proto_rawDesc = nil - file_hapb_ha_proto_goTypes = nil - file_hapb_ha_proto_depIdxs = nil -} diff --git a/api/hapb/ha.pb.validate.go b/api/hapb/ha.pb.validate.go deleted file mode 100644 index ff8e848601..0000000000 --- a/api/hapb/ha.pb.validate.go +++ /dev/null @@ -1,357 +0,0 @@ -// Code generated by protoc-gen-validate. DO NOT EDIT. -// source: hapb/ha.proto - -package hapb - -import ( - "bytes" - "errors" - "fmt" - "net" - "net/mail" - "net/url" - "regexp" - "sort" - "strings" - "time" - "unicode/utf8" - - "google.golang.org/protobuf/types/known/anypb" -) - -// ensure the imports are used -var ( - _ = bytes.MinRead - _ = errors.New("") - _ = fmt.Print - _ = utf8.UTFMax - _ = (*regexp.Regexp)(nil) - _ = (*strings.Reader)(nil) - _ = net.IPv4len - _ = time.Duration(0) - _ = (*url.URL)(nil) - _ = (*mail.Address)(nil) - _ = anypb.Any{} - _ = sort.Sort -) - -// Validate checks the field values on Metadata with the rules defined in the -// proto definition for this message. If any rules are violated, the first -// error encountered is returned, or nil if there are no violations. -func (m *Metadata) Validate() error { - return m.validate(false) -} - -// ValidateAll checks the field values on Metadata with the rules defined in -// the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in MetadataMultiError, or nil -// if none found. -func (m *Metadata) ValidateAll() error { - return m.validate(true) -} - -func (m *Metadata) validate(all bool) error { - if m == nil { - return nil - } - - var errors []error - - // no validation rules for Sender - - // no validation rules for Receiver - - if len(errors) > 0 { - return MetadataMultiError(errors) - } - - return nil -} - -// MetadataMultiError is an error wrapping multiple validation errors returned -// by Metadata.ValidateAll() if the designated constraints aren't met. -type MetadataMultiError []error - -// Error returns a concatenation of all the error messages it wraps. -func (m MetadataMultiError) Error() string { - var msgs []string - for _, err := range m { - msgs = append(msgs, err.Error()) - } - return strings.Join(msgs, "; ") -} - -// AllErrors returns a list of validation violation errors. -func (m MetadataMultiError) AllErrors() []error { return m } - -// MetadataValidationError is the validation error returned by -// Metadata.Validate if the designated constraints aren't met. -type MetadataValidationError struct { - field string - reason string - cause error - key bool -} - -// Field function returns field value. -func (e MetadataValidationError) Field() string { return e.field } - -// Reason function returns reason value. -func (e MetadataValidationError) Reason() string { return e.reason } - -// Cause function returns cause value. -func (e MetadataValidationError) Cause() error { return e.cause } - -// Key function returns key value. -func (e MetadataValidationError) Key() bool { return e.key } - -// ErrorName returns error name. -func (e MetadataValidationError) ErrorName() string { return "MetadataValidationError" } - -// Error satisfies the builtin error interface -func (e MetadataValidationError) Error() string { - cause := "" - if e.cause != nil { - cause = fmt.Sprintf(" | caused by: %v", e.cause) - } - - key := "" - if e.key { - key = "key for " - } - - return fmt.Sprintf( - "invalid %sMetadata.%s: %s%s", - key, - e.field, - e.reason, - cause) -} - -var _ error = MetadataValidationError{} - -var _ interface { - Field() string - Reason() string - Key() bool - Cause() error - ErrorName() string -} = MetadataValidationError{} - -// Validate checks the field values on ClusterMessage with the rules defined in -// the proto definition for this message. If any rules are violated, the first -// error encountered is returned, or nil if there are no violations. -func (m *ClusterMessage) Validate() error { - return m.validate(false) -} - -// ValidateAll checks the field values on ClusterMessage with the rules defined -// in the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in ClusterMessageMultiError, -// or nil if none found. -func (m *ClusterMessage) ValidateAll() error { - return m.validate(true) -} - -func (m *ClusterMessage) validate(all bool) error { - if m == nil { - return nil - } - - var errors []error - - // no validation rules for Id - - if all { - switch v := interface{}(m.GetMetadata()).(type) { - case interface{ ValidateAll() error }: - if err := v.ValidateAll(); err != nil { - errors = append(errors, ClusterMessageValidationError{ - field: "Metadata", - reason: "embedded message failed validation", - cause: err, - }) - } - case interface{ Validate() error }: - if err := v.Validate(); err != nil { - errors = append(errors, ClusterMessageValidationError{ - field: "Metadata", - reason: "embedded message failed validation", - cause: err, - }) - } - } - } else if v, ok := interface{}(m.GetMetadata()).(interface{ Validate() error }); ok { - if err := v.Validate(); err != nil { - return ClusterMessageValidationError{ - field: "Metadata", - reason: "embedded message failed validation", - cause: err, - } - } - } - - switch v := m.Payload.(type) { - case *ClusterMessage_ServerMessage: - if v == nil { - err := ClusterMessageValidationError{ - field: "Payload", - reason: "oneof value cannot be a typed-nil", - } - if !all { - return err - } - errors = append(errors, err) - } - - if all { - switch v := interface{}(m.GetServerMessage()).(type) { - case interface{ ValidateAll() error }: - if err := v.ValidateAll(); err != nil { - errors = append(errors, ClusterMessageValidationError{ - field: "ServerMessage", - reason: "embedded message failed validation", - cause: err, - }) - } - case interface{ Validate() error }: - if err := v.Validate(); err != nil { - errors = append(errors, ClusterMessageValidationError{ - field: "ServerMessage", - reason: "embedded message failed validation", - cause: err, - }) - } - } - } else if v, ok := interface{}(m.GetServerMessage()).(interface{ Validate() error }); ok { - if err := v.Validate(); err != nil { - return ClusterMessageValidationError{ - field: "ServerMessage", - reason: "embedded message failed validation", - cause: err, - } - } - } - - case *ClusterMessage_AgentMessage: - if v == nil { - err := ClusterMessageValidationError{ - field: "Payload", - reason: "oneof value cannot be a typed-nil", - } - if !all { - return err - } - errors = append(errors, err) - } - - if all { - switch v := interface{}(m.GetAgentMessage()).(type) { - case interface{ ValidateAll() error }: - if err := v.ValidateAll(); err != nil { - errors = append(errors, ClusterMessageValidationError{ - field: "AgentMessage", - reason: "embedded message failed validation", - cause: err, - }) - } - case interface{ Validate() error }: - if err := v.Validate(); err != nil { - errors = append(errors, ClusterMessageValidationError{ - field: "AgentMessage", - reason: "embedded message failed validation", - cause: err, - }) - } - } - } else if v, ok := interface{}(m.GetAgentMessage()).(interface{ Validate() error }); ok { - if err := v.Validate(); err != nil { - return ClusterMessageValidationError{ - field: "AgentMessage", - reason: "embedded message failed validation", - cause: err, - } - } - } - - default: - _ = v // ensures v is used - } - - if len(errors) > 0 { - return ClusterMessageMultiError(errors) - } - - return nil -} - -// ClusterMessageMultiError is an error wrapping multiple validation errors -// returned by ClusterMessage.ValidateAll() if the designated constraints -// aren't met. -type ClusterMessageMultiError []error - -// Error returns a concatenation of all the error messages it wraps. -func (m ClusterMessageMultiError) Error() string { - var msgs []string - for _, err := range m { - msgs = append(msgs, err.Error()) - } - return strings.Join(msgs, "; ") -} - -// AllErrors returns a list of validation violation errors. -func (m ClusterMessageMultiError) AllErrors() []error { return m } - -// ClusterMessageValidationError is the validation error returned by -// ClusterMessage.Validate if the designated constraints aren't met. -type ClusterMessageValidationError struct { - field string - reason string - cause error - key bool -} - -// Field function returns field value. -func (e ClusterMessageValidationError) Field() string { return e.field } - -// Reason function returns reason value. -func (e ClusterMessageValidationError) Reason() string { return e.reason } - -// Cause function returns cause value. -func (e ClusterMessageValidationError) Cause() error { return e.cause } - -// Key function returns key value. -func (e ClusterMessageValidationError) Key() bool { return e.key } - -// ErrorName returns error name. -func (e ClusterMessageValidationError) ErrorName() string { return "ClusterMessageValidationError" } - -// Error satisfies the builtin error interface -func (e ClusterMessageValidationError) Error() string { - cause := "" - if e.cause != nil { - cause = fmt.Sprintf(" | caused by: %v", e.cause) - } - - key := "" - if e.key { - key = "key for " - } - - return fmt.Sprintf( - "invalid %sClusterMessage.%s: %s%s", - key, - e.field, - e.reason, - cause) -} - -var _ error = ClusterMessageValidationError{} - -var _ interface { - Field() string - Reason() string - Key() bool - Cause() error - ErrorName() string -} = ClusterMessageValidationError{} diff --git a/api/hapb/ha.proto b/api/hapb/ha.proto deleted file mode 100644 index 4c1b5f6b78..0000000000 --- a/api/hapb/ha.proto +++ /dev/null @@ -1,21 +0,0 @@ -syntax = "proto3"; - -package ha; - -import "agentpb/agent.proto"; - -option go_package = "api/hapb;hapb"; - -message Metadata { - string sender = 1; - string receiver = 2; -} - -message ClusterMessage { - uint32 id = 1; - Metadata metadata = 2; - oneof payload { - agent.ServerMessage server_message = 11; - agent.AgentMessage agent_message = 12; - } -} diff --git a/go.sum b/go.sum index a0e0b1eb23..27a2f9c6c9 100644 --- a/go.sum +++ b/go.sum @@ -205,7 +205,6 @@ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= @@ -533,6 +532,7 @@ github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= @@ -973,7 +973,6 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 07adec8266..d5ef8e79ef 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -976,7 +976,7 @@ func main() { //nolint:cyclop,maintidx jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService) agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams) - agentsHandler := agents.NewHandler(db, ha, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) + agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) actionsService := agents.NewActionsService(qanClient, agentsRegistry) diff --git a/managed/services/agents/actions.go b/managed/services/agents/actions.go index c6e21cec21..c2834b8c3a 100644 --- a/managed/services/agents/actions.go +++ b/managed/services/agents/actions.go @@ -117,7 +117,7 @@ func (s *ActionsService) StartMySQLExplainAction( Timeout: defaultActionTimeout, } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -144,7 +144,7 @@ func (s *ActionsService) StartMySQLShowCreateTableAction(_ context.Context, id, if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -171,7 +171,7 @@ func (s *ActionsService) StartMySQLShowTableStatusAction(_ context.Context, id, if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -198,7 +198,7 @@ func (s *ActionsService) StartMySQLShowIndexAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -219,7 +219,7 @@ func (s *ActionsService) StartPostgreSQLShowCreateTableAction(_ context.Context, if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -240,7 +240,7 @@ func (s *ActionsService) StartPostgreSQLShowIndexAction(_ context.Context, id, p if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -266,7 +266,7 @@ func (s *ActionsService) StartMongoDBExplainAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -293,7 +293,7 @@ func (s *ActionsService) StartMySQLQueryShowAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -320,7 +320,7 @@ func (s *ActionsService) StartMySQLQuerySelectAction(_ context.Context, id, pmmA if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -340,7 +340,7 @@ func (s *ActionsService) StartPostgreSQLQueryShowAction(_ context.Context, id, p if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -361,7 +361,7 @@ func (s *ActionsService) StartPostgreSQLQuerySelectAction(_ context.Context, id, if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -386,7 +386,7 @@ func (s *ActionsService) StartMongoDBQueryGetParameterAction(_ context.Context, if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -411,7 +411,7 @@ func (s *ActionsService) StartMongoDBQueryBuildInfoAction(_ context.Context, id, if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -436,7 +436,7 @@ func (s *ActionsService) StartMongoDBQueryGetCmdLineOptsAction(_ context.Context if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -461,7 +461,7 @@ func (s *ActionsService) StartMongoDBQueryReplSetGetStatusAction(_ context.Conte if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -486,7 +486,7 @@ func (s *ActionsService) StartMongoDBQueryGetDiagnosticDataAction(_ context.Cont if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -505,7 +505,7 @@ func (s *ActionsService) StartPTSummaryAction(_ context.Context, id, pmmAgentID if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(aRequest) + _, err = agent.channel.SendAndWaitResponse(aRequest) return err } @@ -528,7 +528,7 @@ func (s *ActionsService) StartPTPgSummaryAction(_ context.Context, id, pmmAgentI if err != nil { return err } - _, err = pmmAgent.Channel().SendAndWaitResponse(actionRequest) + _, err = pmmAgent.channel.SendAndWaitResponse(actionRequest) return err } @@ -554,7 +554,7 @@ func (s *ActionsService) StartPTMongoDBSummaryAction(_ context.Context, id, pmmA if err != nil { return err } - _, err = pmmAgent.Channel().SendAndWaitResponse(actionRequest) + _, err = pmmAgent.channel.SendAndWaitResponse(actionRequest) return err } @@ -580,7 +580,7 @@ func (s *ActionsService) StartPTMySQLSummaryAction(_ context.Context, id, pmmAge if err != nil { return err } - _, err = pmmAgent.Channel().SendAndWaitResponse(actionRequest) + _, err = pmmAgent.channel.SendAndWaitResponse(actionRequest) return err } @@ -591,6 +591,6 @@ func (s *ActionsService) StopAction(_ context.Context, actionID string) error { if err != nil { return err } - _, err = agent.Channel().SendAndWaitResponse(&agentpb.StopActionRequest{ActionId: actionID}) + _, err = agent.channel.SendAndWaitResponse(&agentpb.StopActionRequest{ActionId: actionID}) return err } diff --git a/managed/services/agents/agents.go b/managed/services/agents/agents.go index b77bbe0e99..0d7ffa4853 100644 --- a/managed/services/agents/agents.go +++ b/managed/services/agents/agents.go @@ -46,7 +46,7 @@ func (a *AgentService) Logs(_ context.Context, pmmAgentID, agentID string, limit return nil, 0, err } - resp, err := agent.Channel().SendAndWaitResponse(&agentpb.AgentLogsRequest{ + resp, err := agent.channel.SendAndWaitResponse(&agentpb.AgentLogsRequest{ AgentId: agentID, Limit: limit, }) @@ -79,7 +79,7 @@ func (a *AgentService) PBMSwitchPITR(pmmAgentID, dsn string, files map[string]st Enabled: enabled, } - _, err = agent.Channel().SendAndWaitResponse(req) + _, err = agent.channel.SendAndWaitResponse(req) return err } diff --git a/managed/services/agents/connection_checker.go b/managed/services/agents/connection_checker.go index a9ea43a17d..b784fd4d85 100644 --- a/managed/services/agents/connection_checker.go +++ b/managed/services/agents/connection_checker.go @@ -91,7 +91,7 @@ func (c *ConnectionChecker) CheckConnectionToService(ctx context.Context, q *ref sanitizedDSN = strings.ReplaceAll(request.Dsn, word, "****") } l.Infof("CheckConnectionRequest: type: %s, DSN: %s timeout: %s.", request.Type, sanitizedDSN, request.Timeout) - resp, err := pmmAgent.Channel().SendAndWaitResponse(request) + resp, err := pmmAgent.channel.SendAndWaitResponse(request) if err != nil { return err } diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index d5597bc322..396517c7d1 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -63,9 +63,3 @@ type victoriaMetricsParams interface { URLFor(path string) (*url.URL, error) URL() string } - -// highAvailablityService is an interface to get information related to High Availability. -type highAvailablityService interface { - // TODO: extend by send message - BroadcastMessage(message []byte) -} diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go index 4300a661fe..fdde5645d0 100644 --- a/managed/services/agents/handler.go +++ b/managed/services/agents/handler.go @@ -37,7 +37,6 @@ import ( // Handler handles agent requests. type Handler struct { db *reform.DB - ha highAvailablityService r *Registry vmdb prometheusService qanClient qanClient @@ -46,12 +45,11 @@ type Handler struct { } // NewHandler creates new agents handler. -func NewHandler(db *reform.DB, ha highAvailablityService, qanClient qanClient, vmdb prometheusService, registry *Registry, - state *StateUpdater, jobsService jobsService, +func NewHandler(db *reform.DB, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater, + jobsService jobsService, ) *Handler { h := &Handler{ db: db, - ha: ha, r: registry, vmdb: vmdb, qanClient: qanClient, @@ -79,7 +77,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { // run pmm-agent state update loop for the current agent. go h.state.runStateChangeHandler(ctx, agent) - h.state.RequestStateUpdate(ctx, agent.ID()) + h.state.RequestStateUpdate(ctx, agent.id) ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -88,22 +86,22 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { case <-ticker.C: err := h.r.ping(ctx, agent) if err != nil { - l.Errorf("agent %s ping: %v", agent.ID(), err) + l.Errorf("agent %s ping: %v", agent.id, err) } // see unregister and Kick methods - case <-agent.KickChan(): + case <-agent.kickChan: // already unregistered, no need to call unregister method l.Warn("Kicked.") disconnectReason = "kicked" err = status.Errorf(codes.Aborted, "Kicked.") return err - case req := <-agent.Channel().Requests(): + case req := <-agent.channel.Requests(): if req == nil { disconnectReason = "done" - err = agent.Channel().Wait() - h.r.unregister(agent.ID(), disconnectReason) + err = agent.channel.Wait() + h.r.unregister(agent.id, disconnectReason) if err != nil { l.Error(errors.WithStack(err)) } @@ -112,7 +110,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { switch p := req.Payload.(type) { case *agentpb.Ping: - agent.Channel().Send(&channel.ServerResponse{ + agent.channel.Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.Pong{ CurrentTime: timestamppb.Now(), @@ -125,7 +123,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { l.Errorf("%+v", err) } - agent.Channel().Send(&channel.ServerResponse{ + agent.channel.Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.StateChangedResponse{}, }) @@ -137,7 +135,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { l.Errorf("%+v", err) } - agent.Channel().Send(&channel.ServerResponse{ + agent.channel.Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.QANCollectResponse{}, }) @@ -145,7 +143,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { case *agentpb.ActionResultRequest: // TODO: PMM-3978: In the future we need to merge action parts before send it to storage. - err := models.ChangeActionResult(h.db.Querier, p.ActionId, agent.ID(), p.Error, string(p.Output), p.Done) + err := models.ChangeActionResult(h.db.Querier, p.ActionId, agent.id, p.Error, string(p.Output), p.Done) if err != nil { l.Warnf("Failed to change action: %+v", err) } @@ -154,7 +152,7 @@ func (h *Handler) Run(stream agentpb.Agent_ConnectServer) error { l.Warnf("Action was done with an error: %v.", p.Error) } - agent.Channel().Send(&channel.ServerResponse{ + agent.channel.Send(&channel.ServerResponse{ ID: req.ID, Payload: &agentpb.ActionResultResponse{}, }) diff --git a/managed/services/agents/jobs.go b/managed/services/agents/jobs.go index 3cba7bdfac..a78fffd381 100644 --- a/managed/services/agents/jobs.go +++ b/managed/services/agents/jobs.go @@ -411,7 +411,7 @@ func (s *JobsService) StartMySQLBackupJob(jobID, pmmAgentID string, timeout time return err } - resp, err := agent.Channel().SendAndWaitResponse(req) + resp, err := agent.channel.SendAndWaitResponse(req) if err != nil { return err } @@ -509,7 +509,7 @@ func (s *JobsService) StartMongoDBBackupJob( return err } - resp, err := agentInfo.Channel().SendAndWaitResponse(req) + resp, err := agentInfo.channel.SendAndWaitResponse(req) if err != nil { return err } @@ -559,7 +559,7 @@ func (s *JobsService) StartMySQLRestoreBackupJob( return err } - resp, err := agent.Channel().SendAndWaitResponse(req) + resp, err := agent.channel.SendAndWaitResponse(req) if err != nil { return err } @@ -666,7 +666,7 @@ func (s *JobsService) StartMongoDBRestoreBackupJob( return err } - resp, err := agentInfo.Channel().SendAndWaitResponse(req) + resp, err := agentInfo.channel.SendAndWaitResponse(req) if err != nil { return err } @@ -755,7 +755,7 @@ func (s *JobsService) restartSystemService(agentID string, service agentpb.Start if err != nil { return errors.Wrapf(err, "failed to get information about PMM agent: %s", agentID) } - _, err = agent.Channel().SendAndWaitResponse(req) + _, err = agent.channel.SendAndWaitResponse(req) if err != nil { return errors.Wrapf(err, "failed to restart %s on agent: %s", service, agentID) } @@ -779,7 +779,7 @@ func (s *JobsService) StopJob(jobID string) error { return errors.WithStack(err) } - _, err = agent.Channel().SendAndWaitResponse(&agentpb.StopJobRequest{JobId: jobID}) + _, err = agent.channel.SendAndWaitResponse(&agentpb.StopJobRequest{JobId: jobID}) return err } diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 8489312d06..f841f58420 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -64,42 +64,19 @@ var ( nil) ) -type pmmAgentInfo interface { - Channel() *channel.Channel - ID() string - StateChangeChan() chan struct{} - KickChan() chan struct{} -} - -type localPMMAgentInfo struct { +type pmmAgentInfo struct { channel *channel.Channel id string stateChangeChan chan struct{} kickChan chan struct{} } -func (p *localPMMAgentInfo) Channel() *channel.Channel { - return p.channel -} - -func (p *localPMMAgentInfo) ID() string { - return p.id -} - -func (p *localPMMAgentInfo) StateChangeChan() chan struct{} { - return p.stateChangeChan -} - -func (p *localPMMAgentInfo) KickChan() chan struct{} { - return p.kickChan -} - // Registry keeps track of all connected pmm-agents. type Registry struct { db *reform.DB rw sync.RWMutex - agents map[string]*localPMMAgentInfo // id -> info + agents map[string]*pmmAgentInfo // id -> info roster *roster @@ -114,7 +91,7 @@ type Registry struct { // NewRegistry creates a new registry with given database connection. func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry { - agents := make(map[string]*localPMMAgentInfo) + agents := make(map[string]*pmmAgentInfo) r := &Registry{ db: db, @@ -176,7 +153,7 @@ func (r *Registry) IsConnected(pmmAgentID string) bool { return err == nil } -func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*localPMMAgentInfo, error) { +func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, error) { ctx := stream.Context() l := logger.Get(ctx) r.mConnects.Inc() @@ -221,7 +198,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*localPMMAgentI r.rw.Lock() defer r.rw.Unlock() - agent := &localPMMAgentInfo{ + agent := &pmmAgentInfo{ channel: channel.New(ctx, stream), id: agentMD.ID, stateChangeChan: make(chan struct{}, 1), @@ -278,7 +255,7 @@ func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Quer } // unregister removes pmm-agent with given ID from the registry. -func (r *Registry) unregister(pmmAgentID, disconnectReason string) *localPMMAgentInfo { +func (r *Registry) unregister(pmmAgentID, disconnectReason string) *pmmAgentInfo { r.mDisconnects.WithLabelValues(disconnectReason).Inc() r.rw.Lock() @@ -298,10 +275,10 @@ func (r *Registry) unregister(pmmAgentID, disconnectReason string) *localPMMAgen } // ping sends Ping message to given Agent, waits for Pong and observes round-trip time and clock drift. -func (r *Registry) ping(ctx context.Context, agent pmmAgentInfo) error { +func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error { l := logger.Get(ctx) start := time.Now() - resp, err := agent.Channel().SendAndWaitResponse(&agentpb.Ping{}) + resp, err := agent.channel.SendAndWaitResponse(&agentpb.Ping{}) if err != nil { return err } @@ -394,13 +371,13 @@ func (r *Registry) Kick(ctx context.Context, pmmAgentID string) { l.Debugf("pmm-agent with ID %q will be kicked in a moment.", pmmAgentID) // see Run method - close(agent.KickChan()) + close(agent.kickChan) // Do not close agent.stateChangeChan to avoid breaking RequestStateUpdate; // closing agent.kickChan is enough to exit runStateChangeHandler goroutine. } -func (r *Registry) get(pmmAgentID string) (pmmAgentInfo, error) { //nolint:ireturn +func (r *Registry) get(pmmAgentID string) (*pmmAgentInfo, error) { r.rw.RLock() pmmAgent := r.agents[pmmAgentID] r.rw.RUnlock() @@ -424,12 +401,12 @@ func (r *Registry) Collect(ch chan<- prom.Metric) { r.rw.RLock() for _, agent := range r.agents { - m := agent.Channel().Metrics() + m := agent.channel.Metrics() - ch <- prom.MustNewConstMetric(mSentDesc, prom.CounterValue, m.Sent, agent.ID()) - ch <- prom.MustNewConstMetric(mRecvDesc, prom.CounterValue, m.Recv, agent.ID()) - ch <- prom.MustNewConstMetric(mResponsesDesc, prom.GaugeValue, m.Responses, agent.ID()) - ch <- prom.MustNewConstMetric(mRequestsDesc, prom.GaugeValue, m.Requests, agent.ID()) + ch <- prom.MustNewConstMetric(mSentDesc, prom.CounterValue, m.Sent, agent.id) + ch <- prom.MustNewConstMetric(mRecvDesc, prom.CounterValue, m.Recv, agent.id) + ch <- prom.MustNewConstMetric(mResponsesDesc, prom.GaugeValue, m.Responses, agent.id) + ch <- prom.MustNewConstMetric(mRequestsDesc, prom.GaugeValue, m.Requests, agent.id) } r.rw.RUnlock() @@ -442,7 +419,7 @@ func (r *Registry) Collect(ch chan<- prom.Metric) { func (r *Registry) KickAll(ctx context.Context) { for _, agentInfo := range r.agents { - r.Kick(ctx, agentInfo.ID()) + r.Kick(ctx, agentInfo.id) } } diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index 12744c1995..d4020c966a 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -68,7 +68,7 @@ func (u *StateUpdater) RequestStateUpdate(ctx context.Context, pmmAgentID string } select { - case agent.StateChangeChan() <- struct{}{}: + case agent.stateChangeChan <- struct{}{}: default: } } @@ -95,8 +95,8 @@ func (u *StateUpdater) UpdateAgentsState(ctx context.Context) error { } // runStateChangeHandler runs pmm-agent state update loop for given pmm-agent until ctx is canceled or agent is kicked. -func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgentInfo) { - l := logger.Get(ctx).WithField("agent_id", agent.ID()) +func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgentInfo) { + l := logger.Get(ctx).WithField("agent_id", agent.id) l.Info("Starting runStateChangeHandler ...") defer l.Info("Done runStateChangeHandler.") @@ -104,7 +104,7 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgent // stateChangeChan, state update loop, and RequestStateUpdate method ensure that state // is reloaded when requested, but several requests are batched together to avoid too often reloads. // That allows the caller to just call RequestStateUpdate when it seems fit. - if cap(agent.StateChangeChan()) != 1 { + if cap(agent.stateChangeChan) != 1 { panic("stateChangeChan should have capacity 1") } @@ -113,10 +113,10 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgent case <-ctx.Done(): return - case <-agent.KickChan(): + case <-agent.kickChan: return - case <-agent.StateChangeChan(): + case <-agent.stateChangeChan: // batch several update requests together by delaying the first one sleepCtx, sleepCancel := context.WithTimeout(ctx, updateBatchDelay) <-sleepCtx.Done() @@ -130,7 +130,7 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgent err := u.sendSetStateRequest(nCtx, agent) if err != nil { l.Error(err) - u.RequestStateUpdate(ctx, agent.ID()) + u.RequestStateUpdate(ctx, agent.id) } cancel() } @@ -138,7 +138,7 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgent } // sendSetStateRequest sends SetStateRequest to given pmm-agent. -func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentInfo) error { //nolint:cyclop +func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentInfo) error { //nolint:cyclop l := logger.Get(ctx) start := time.Now() defer func() { @@ -146,7 +146,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentIn l.Warnf("sendSetStateRequest took %s.", dur) } }() - pmmAgent, err := models.FindAgentByID(u.db.Querier, agent.ID()) + pmmAgent, err := models.FindAgentByID(u.db.Querier, agent.id) if err != nil { return errors.Wrap(err, "failed to get PMM Agent") } @@ -155,7 +155,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentIn return errors.Wrapf(err, "failed to parse PMM agent version %q", *pmmAgent.Version) } - agents, err := models.FindAgents(u.db.Querier, models.AgentFilters{PMMAgentID: agent.ID()}) + agents, err := models.FindAgents(u.db.Querier, models.AgentFilters{PMMAgentID: agent.id}) if err != nil { return errors.Wrap(err, "failed to collect agents") } @@ -178,9 +178,9 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentIn case models.PMMAgentType: continue case models.VMAgentType: - scrapeCfg, err := u.vmdb.BuildScrapeConfigForVMAgent(agent.ID()) + scrapeCfg, err := u.vmdb.BuildScrapeConfigForVMAgent(agent.id) if err != nil { - return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.ID()) + return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.id) } agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg), u.vmParams) @@ -262,7 +262,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentIn } if len(rdsExporters) != 0 { - groupID := u.r.roster.add(agent.ID(), rdsGroup, rdsExporters) + groupID := u.r.roster.add(agent.id, rdsGroup, rdsExporters) c, err := rdsExporterConfig(rdsExporters, redactMode, pmmAgentVersion) if err != nil { return err @@ -276,7 +276,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent pmmAgentIn } l.Debugf("sendSetStateRequest:\n%s", proto.MarshalTextString(state)) - resp, err := agent.Channel().SendAndWaitResponse(state) + resp, err := agent.channel.SendAndWaitResponse(state) if err != nil { return err } diff --git a/managed/services/agents/versioner.go b/managed/services/agents/versioner.go index 279bca993b..d91709128f 100644 --- a/managed/services/agents/versioner.go +++ b/managed/services/agents/versioner.go @@ -157,7 +157,7 @@ func (s *VersionerService) GetVersions(pmmAgentID string, softwareList []Softwar } request := &agentpb.GetVersionsRequest{Softwares: softwareRequest} - response, err := agent.Channel().SendAndWaitResponse(request) + response, err := agent.channel.SendAndWaitResponse(request) if err != nil { return nil, errors.WithStack(err) } diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index 10eb1a1703..cac427f257 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -27,9 +27,7 @@ import ( "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/sirupsen/logrus" - "google.golang.org/protobuf/proto" - "github.com/percona/pmm/api/hapb" "github.com/percona/pmm/managed/models" ) @@ -53,15 +51,6 @@ type Service struct { func (s *Service) Apply(logEntry *raft.Log) interface{} { s.l.Printf("raft: got a message: %s", string(logEntry.Data)) - var m hapb.ClusterMessage - err := proto.Unmarshal(logEntry.Data, &m) - if err != nil { - return err - } - switch m.Payload.(type) { - case *hapb.ClusterMessage_ServerMessage: - case *hapb.ClusterMessage_AgentMessage: - } s.receivedMessages <- logEntry.Data return nil } From 9f98df580b6e9fad3cc07c98a4a818758a916b86 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 18 Sep 2023 20:49:05 +0300 Subject: [PATCH 62/77] PMM-12078 fix linter. --- Makefile.include | 2 +- managed/services/agents/log_level.go | 6 +++--- managed/services/server/server.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile.include b/Makefile.include index 0e33e35406..7c59505c85 100644 --- a/Makefile.include +++ b/Makefile.include @@ -47,7 +47,7 @@ gen-api: ## Generate PMM API bin/buf generate -v api - for API in api/agentlocalpb api/serverpb api/inventorypb api/hapb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/node api/managementpb/service api/platformpb api/userpb; do \ + for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/node api/managementpb/service api/platformpb api/userpb; do \ set -x ; \ bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \ bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \ diff --git a/managed/services/agents/log_level.go b/managed/services/agents/log_level.go index 57f236c1f0..58f62b6803 100644 --- a/managed/services/agents/log_level.go +++ b/managed/services/agents/log_level.go @@ -22,11 +22,11 @@ import ( "github.com/percona/pmm/version" ) -// Log level available in exporters with pmm 2.28 +// Log level available in exporters with pmm 2.28. var exporterLogLevelCommandVersion = version.MustParse("2.27.99") -// withLogLevel - append CLI args --log.level -// mysqld_exporter, node_exporter and postgres_exporter don't support --log.level=fatal +// withLogLevel - append CLI args --log.level. +// mysqld_exporter, node_exporter and postgres_exporter don't support --log.level=fatal. func withLogLevel(args []string, logLevel *string, pmmAgentVersion *version.Parsed, supportLogLevelFatal bool) []string { level := pointer.GetString(logLevel) diff --git a/managed/services/server/server.go b/managed/services/server/server.go index ae582ee907..a689dbf293 100644 --- a/managed/services/server/server.go +++ b/managed/services/server/server.go @@ -245,7 +245,7 @@ func (s *Server) Readiness(ctx context.Context, req *serverpb.ReadinessRequest) // LeaderHealthCheck checks if the instance is the leader in a cluster. // Returns an error if the instance isn't the leader. -// It's used for HA purpose +// It's used for HA purpose. func (s *Server) LeaderHealthCheck(ctx context.Context, req *serverpb.LeaderHealthCheckRequest) (*serverpb.LeaderHealthCheckResponse, error) { if s.haService.IsLeader() { return &serverpb.LeaderHealthCheckResponse{}, nil From 993e4d5ddf0fc41c276accdb65f06b8b0d959013 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 18 Sep 2023 21:56:10 +0300 Subject: [PATCH 63/77] Update log_level.go --- managed/services/agents/log_level.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/agents/log_level.go b/managed/services/agents/log_level.go index 58f62b6803..2fa6ac409c 100644 --- a/managed/services/agents/log_level.go +++ b/managed/services/agents/log_level.go @@ -25,8 +25,8 @@ import ( // Log level available in exporters with pmm 2.28. var exporterLogLevelCommandVersion = version.MustParse("2.27.99") -// withLogLevel - append CLI args --log.level. -// mysqld_exporter, node_exporter and postgres_exporter don't support --log.level=fatal. +// withLogLevel - append CLI args --log.level +// mysqld_exporter, node_exporter and postgres_exporter don't support --log.level=fatal func withLogLevel(args []string, logLevel *string, pmmAgentVersion *version.Parsed, supportLogLevelFatal bool) []string { level := pointer.GetString(logLevel) From f61f01686b821a513164a9c9f7828b6dbfd25bc0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 4 Oct 2023 00:38:24 +0300 Subject: [PATCH 64/77] Update managed/services/highavailability/highavailability.go Co-authored-by: Artem Gavrilov --- managed/services/highavailability/highavailability.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/services/highavailability/highavailability.go b/managed/services/highavailability/highavailability.go index cac427f257..b994524948 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/highavailability/highavailability.go @@ -268,7 +268,7 @@ func (s *Service) runLeaderObserver(ctx context.Context) { if isLeader { s.services.StartAllServices(ctx) // This node is the leader - s.l.Printf("I am a leader!") + s.l.Printf("I am the leader!") peers := s.memberlist.Members() for _, peer := range peers { if peer.Name == s.params.NodeID { From 6f7ae1de9f9914f995cc5fa4bcbd7f87c20dcb2f Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 4 Oct 2023 00:38:33 +0300 Subject: [PATCH 65/77] Update managed/services/highavailability/leaderservice.go Co-authored-by: Artem Gavrilov --- managed/services/highavailability/leaderservice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/highavailability/leaderservice.go index 05d909e4ca..7886a6fb74 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/highavailability/leaderservice.go @@ -57,9 +57,9 @@ type ContextService struct { id string startFunc func(context.Context) error - cancel context.CancelFunc m sync.Mutex + cancel context.CancelFunc } func NewContextService(id string, startFunc func(context.Context) error) *ContextService { From 8bf58db60b2a80162d1e03b3f1cc0c2c92bb06a0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 4 Oct 2023 00:45:11 +0300 Subject: [PATCH 66/77] PMM-12078 address comments. --- managed/cmd/pmm-managed/main.go | 46 +++++++++---------- .../highavailability.go | 11 +++-- .../{highavailability => ha}/leaderservice.go | 2 +- .../{highavailability => ha}/services.go | 22 ++++----- managed/services/server/deps.go | 2 +- 5 files changed, 42 insertions(+), 41 deletions(-) rename managed/services/{highavailability => ha}/highavailability.go (95%) rename managed/services/{highavailability => ha}/leaderservice.go (98%) rename managed/services/{highavailability => ha}/services.go (82%) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index d5ef8e79ef..d3bb4e4271 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -84,7 +84,7 @@ import ( "github.com/percona/pmm/managed/services/config" "github.com/percona/pmm/managed/services/dbaas" "github.com/percona/pmm/managed/services/grafana" - "github.com/percona/pmm/managed/services/highavailability" + "github.com/percona/pmm/managed/services/ha" "github.com/percona/pmm/managed/services/inventory" inventorygrpc "github.com/percona/pmm/managed/services/inventory/grpc" "github.com/percona/pmm/managed/services/management" @@ -186,7 +186,7 @@ func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) { type gRPCServerDeps struct { db *reform.DB - ha *highavailability.Service + ha *ha.Service vmdb *victoriametrics.Service platformClient *platformClient.Client server *server.Server @@ -247,7 +247,7 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { ) if l.Logger.GetLevel() >= logrus.DebugLevel { - l.Debug("Reflection and channels are enabled.") + l.Debug("Reflection and channelz are enabled.") reflection.Register(gRPCServer) channelz.RegisterChannelzServiceToServer(gRPCServer) @@ -311,7 +311,7 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { k8sServer := managementdbaas.NewKubernetesServer(deps.db, deps.dbaasClient, deps.versionServiceClient, deps.grafanaClient) - deps.ha.AddLeaderService(highavailability.NewContextService("dbaas-register", func(ctx context.Context) error { + deps.ha.AddLeaderService(ha.NewContextService("dbaas-register", func(ctx context.Context) error { deps.dbaasInitializer.RegisterKubernetesServer(k8sServer) return nil })) @@ -540,7 +540,7 @@ func runDebugServer(ctx context.Context) { type setupDeps struct { sqlDB *sql.DB - ha *highavailability.Service + ha *ha.Service supervisord *supervisord.Service vmdb *victoriametrics.Service vmalert *vmalert.Service @@ -743,7 +743,7 @@ func main() { //nolint:cyclop,maintidx haAdvertiseAddress := kingpin.Flag("ha-advertise-address", "HA Advertise address"). Envar("PERCONA_TEST_HA_ADVERTISE_ADDRESS"). String() - haGossipNodes := kingpin.Flag("ha-peers", "HA Peers"). + haPeers := kingpin.Flag("ha-peers", "HA Peers"). Envar("PERCONA_TEST_HA_PEERS"). String() haRaftPort := kingpin.Flag("ha-raft-port", "HA raft port"). @@ -789,8 +789,8 @@ func main() { //nolint:cyclop,maintidx defer l.Info("Done.") var nodes []string - if *haGossipNodes != "" { - nodes = strings.Split(*haGossipNodes, ",") + if *haPeers != "" { + nodes = strings.Split(*haPeers, ",") } haParams := &models.HAParams{ Enabled: *haEnabled, @@ -802,7 +802,7 @@ func main() { //nolint:cyclop,maintidx GossipPort: *haGossipPort, GrafanaGossipPort: *grafanaGossipPort, } - ha := highavailability.New(haParams) + haService := ha.New(haParams) cfg := config.NewService() if err := cfg.Load(); err != nil { @@ -860,7 +860,7 @@ func main() { //nolint:cyclop,maintidx } defer sqlDB.Close() //nolint:errcheck - if ha.Bootstrap() { + if haService.Bootstrap() { migrateDB(ctx, sqlDB, setupParams) } @@ -869,7 +869,7 @@ func main() { //nolint:cyclop,maintidx prom.MustRegister(reformL) db := reform.NewDB(sqlDB, postgresql.Dialect, reformL) - if ha.Bootstrap() { + if haService.Bootstrap() { // Generate unique PMM Server ID if it's not already set. err = models.SetPMMServerID(db) if err != nil { @@ -896,7 +896,7 @@ func main() { //nolint:cyclop,maintidx agentsRegistry := agents.NewRegistry(db, vmParams) // TODO remove once PMM cluster will be Active-Active - ha.AddLeaderService(highavailability.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() { + haService.AddLeaderService(ha.NewStandardService("agentsRegistry", func(ctx context.Context) error { return nil }, func() { agentsRegistry.KickAll(ctx) })) @@ -939,7 +939,7 @@ func main() { //nolint:cyclop,maintidx }, gRPCMessageMaxSize) - ha.AddLeaderService(highavailability.NewStandardService("pmm-agent-runner", func(ctx context.Context) error { + haService.AddLeaderService(ha.NewStandardService("pmm-agent-runner", func(ctx context.Context) error { return supervisord.StartSupervisedService("pmm-agent") }, func() { err := supervisord.StopSupervisedService("pmm-agent") @@ -999,7 +999,7 @@ func main() { //nolint:cyclop,maintidx l.Fatalf("Could not create templates service: %s", err) } // We should collect templates before rules service created, because it will regenerate rule files on startup. - if ha.Bootstrap() { + if haService.Bootstrap() { templatesService.CollectTemplates(ctx) } rulesService := ia.NewRulesService(db, templatesService, vmalert, alertManager) @@ -1041,7 +1041,7 @@ func main() { //nolint:cyclop,maintidx RulesService: rulesService, DBaaSInitializer: dbaasInitializer, Emailer: emailer, - HAService: ha, + HAService: haService, } server, err := server.NewServer(serverParams) @@ -1077,7 +1077,7 @@ func main() { //nolint:cyclop,maintidx // try synchronously once, then retry in the background deps := &setupDeps{ sqlDB: sqlDB, - ha: ha, + ha: haService, supervisord: supervisord, vmdb: vmdb, vmalert: vmalert, @@ -1138,7 +1138,7 @@ func main() { //nolint:cyclop,maintidx alertManager.Run(ctx) }() - ha.AddLeaderService(highavailability.NewContextService("checks", func(ctx context.Context) error { + haService.AddLeaderService(ha.NewContextService("checks", func(ctx context.Context) error { checksService.Run(ctx) return nil })) @@ -1150,18 +1150,18 @@ func main() { //nolint:cyclop,maintidx }() wg.Add(1) - ha.AddLeaderService(highavailability.NewContextService("telemetry", func(ctx context.Context) error { + haService.AddLeaderService(ha.NewContextService("telemetry", func(ctx context.Context) error { defer wg.Done() telemetry.Run(ctx) return nil })) - ha.AddLeaderService(highavailability.NewContextService("scheduler", func(ctx context.Context) error { + haService.AddLeaderService(ha.NewContextService("scheduler", func(ctx context.Context) error { schedulerService.Run(ctx) return nil })) - ha.AddLeaderService(highavailability.NewContextService("versionCache", func(ctx context.Context) error { + haService.AddLeaderService(ha.NewContextService("versionCache", func(ctx context.Context) error { versionCache.Run(ctx) return nil })) @@ -1172,7 +1172,7 @@ func main() { //nolint:cyclop,maintidx runGRPCServer(ctx, &gRPCServerDeps{ db: db, - ha: ha, + ha: haService, vmdb: vmdb, platformClient: platformClient, server: server, @@ -1225,7 +1225,7 @@ func main() { //nolint:cyclop,maintidx runDebugServer(ctx) }() - ha.AddLeaderService(highavailability.NewContextService("cleaner", func(ctx context.Context) error { + haService.AddLeaderService(ha.NewContextService("cleaner", func(ctx context.Context) error { cleaner.Run(ctx, cleanInterval, cleanOlderThan) return nil })) @@ -1251,7 +1251,7 @@ func main() { //nolint:cyclop,maintidx wg.Add(1) go func() { defer wg.Done() - err := ha.Run(ctx) + err := haService.Run(ctx) if err != nil { l.Panicf("cannot start high availability service: %+v", err) } diff --git a/managed/services/highavailability/highavailability.go b/managed/services/ha/highavailability.go similarity index 95% rename from managed/services/highavailability/highavailability.go rename to managed/services/ha/highavailability.go index b994524948..cec144b4d8 100644 --- a/managed/services/highavailability/highavailability.go +++ b/managed/services/ha/highavailability.go @@ -13,14 +13,15 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -// Package highavailability contains everything related to high availability. -package highavailability +// Package ha contains everything related to high availability. +package ha import ( "context" "fmt" "io" "net" + "strconv" "sync" "time" @@ -82,7 +83,7 @@ func (s *Service) Run(ctx context.Context) error { defer s.wg.Done() for { select { - case <-s.services.ServiceAdded(): + case <-s.services.Refresh(): if s.IsLeader() { s.services.StartAllServices(ctx) } @@ -106,11 +107,11 @@ func (s *Service) Run(ctx context.Context) error { raftConfig.LogLevel = "DEBUG" // Create a new Raft transport - raa, err := net.ResolveTCPAddr("", fmt.Sprintf("%s:%d", s.params.AdvertiseAddress, s.params.RaftPort)) + raa, err := net.ResolveTCPAddr("", net.JoinHostPort(s.params.AdvertiseAddress, strconv.Itoa(s.params.RaftPort))) if err != nil { return err } - raftTrans, err := raft.NewTCPTransport(fmt.Sprintf("0.0.0.0:%d", s.params.RaftPort), raa, 3, 10*time.Second, nil) + raftTrans, err := raft.NewTCPTransport(net.JoinHostPort("0.0.0.0", strconv.Itoa(s.params.RaftPort)), raa, 3, 10*time.Second, nil) if err != nil { return err } diff --git a/managed/services/highavailability/leaderservice.go b/managed/services/ha/leaderservice.go similarity index 98% rename from managed/services/highavailability/leaderservice.go rename to managed/services/ha/leaderservice.go index 7886a6fb74..b5f39fca7b 100644 --- a/managed/services/highavailability/leaderservice.go +++ b/managed/services/ha/leaderservice.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package highavailability +package ha import ( "context" diff --git a/managed/services/highavailability/services.go b/managed/services/ha/services.go similarity index 82% rename from managed/services/highavailability/services.go rename to managed/services/ha/services.go index 90b3e2d04f..fad2c7002d 100644 --- a/managed/services/highavailability/services.go +++ b/managed/services/ha/services.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package highavailability +package ha import ( "context" @@ -24,23 +24,23 @@ import ( ) type services struct { - wg *sync.WaitGroup + wg sync.WaitGroup + rw sync.Mutex all map[string]LeaderService running map[string]LeaderService - serviceAdded chan struct{} + refresh chan struct{} l *logrus.Entry } func newServices() *services { return &services{ - wg: &sync.WaitGroup{}, - all: make(map[string]LeaderService), - running: make(map[string]LeaderService), - serviceAdded: make(chan struct{}), - l: logrus.WithField("component", "ha-services"), + all: make(map[string]LeaderService), + running: make(map[string]LeaderService), + refresh: make(chan struct{}), + l: logrus.WithField("component", "ha-services"), } } @@ -54,7 +54,7 @@ func (s *services) Add(service LeaderService) error { } s.all[id] = service select { - case s.serviceAdded <- struct{}{}: + case s.refresh <- struct{}{}: default: } return nil @@ -98,8 +98,8 @@ func (s *services) StopRunningServices() { } } -func (s *services) ServiceAdded() chan struct{} { - return s.serviceAdded +func (s *services) Refresh() chan struct{} { + return s.refresh } func (s *services) Wait() { diff --git a/managed/services/server/deps.go b/managed/services/server/deps.go index 57a95dceea..5261a8d6bd 100644 --- a/managed/services/server/deps.go +++ b/managed/services/server/deps.go @@ -133,7 +133,7 @@ type templatesService interface { CollectTemplates(ctx context.Context) } -// haService is a subset of methods of highavailability.Service used by this package. +// haService is a subset of methods of ha.Service used by this package. // We use it instead of real type for testing and to avoid dependency cycle. type haService interface { IsLeader() bool From a8b3ca15df73cc8df8fcdadf33297e75e08e00e2 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 4 Oct 2023 12:53:50 +0300 Subject: [PATCH 67/77] PMM-12078 fix linter. --- managed/services/ha/leaderservice.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/ha/leaderservice.go b/managed/services/ha/leaderservice.go index b5f39fca7b..3c12923f72 100644 --- a/managed/services/ha/leaderservice.go +++ b/managed/services/ha/leaderservice.go @@ -58,8 +58,8 @@ type ContextService struct { startFunc func(context.Context) error - m sync.Mutex - cancel context.CancelFunc + m sync.Mutex + cancel context.CancelFunc } func NewContextService(id string, startFunc func(context.Context) error) *ContextService { From d3f45c81e8164fe75047abd64557610564a9f3d4 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 9 Oct 2023 12:19:12 +0300 Subject: [PATCH 68/77] PMM-12078 fix license headers. --- managed/models/params.go | 2 +- managed/services/ha/leaderservice.go | 2 +- managed/services/ha/services.go | 2 +- managed/services/supervisord/pmm_config_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/managed/models/params.go b/managed/models/params.go index 588ae8efa9..8121686d70 100644 --- a/managed/models/params.go +++ b/managed/models/params.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017 Percona LLC +// Copyright (C) 2023 Percona LLC // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by diff --git a/managed/services/ha/leaderservice.go b/managed/services/ha/leaderservice.go index 3c12923f72..9ec5185629 100644 --- a/managed/services/ha/leaderservice.go +++ b/managed/services/ha/leaderservice.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017 Percona LLC +// Copyright (C) 2023 Percona LLC // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by diff --git a/managed/services/ha/services.go b/managed/services/ha/services.go index fad2c7002d..eabc233c8a 100644 --- a/managed/services/ha/services.go +++ b/managed/services/ha/services.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017 Percona LLC +// Copyright (C) 2023 Percona LLC // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by diff --git a/managed/services/supervisord/pmm_config_test.go b/managed/services/supervisord/pmm_config_test.go index ff95d36cd4..baaf279b58 100644 --- a/managed/services/supervisord/pmm_config_test.go +++ b/managed/services/supervisord/pmm_config_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2017 Percona LLC +// Copyright (C) 2023 Percona LLC // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by From c7937fde9ffa168bbf61e6b7ea6b42b026a66125 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 18 Oct 2023 12:50:06 +0300 Subject: [PATCH 69/77] PMM-12078 Some improvements regarding env variables. --- docker-compose.yml | 38 +++++++++---------- managed/cmd/pmm-managed/main.go | 20 +++++----- .../tasks/roles/initialization/tasks/main.yml | 36 +++++++++++------- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6fd3ae5379..543cfa923d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -236,13 +236,13 @@ services: - GO_VERSION=1.20 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - - PERCONA_TEST_HA_ENABLE=1 - - PERCONA_TEST_HA_BOOTSTRAP=1 - - PERCONA_TEST_HA_NODE_ID=pmm-server-active - - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 - - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9094 + - PMM_TEST_HA_ENABLE=1 + - PMM_TEST_HA_BOOTSTRAP=1 + - PMM_TEST_HA_NODE_ID=pmm-server-active + - PMM_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5 + - PMM_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 + - PMM_TEST_HA_GOSSIP_PORT=9096 + - PMM_TEST_HA_GRAFANA_GOSSIP_PORT=9094 extra_hosts: - host.docker.internal:host-gateway @@ -320,12 +320,12 @@ services: - GO_VERSION=1.20 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - - PERCONA_TEST_HA_ENABLE=1 - - PERCONA_TEST_HA_NODE_ID=pmm-server-passive - - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 - - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9094 + - PMM_TEST_HA_ENABLE=1 + - PMM_TEST_HA_NODE_ID=pmm-server-passive + - PMM_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6 + - PMM_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 + - PMM_TEST_HA_GOSSIP_PORT=9096 + - PMM_TEST_HA_GRAFANA_GOSSIP_PORT=9094 extra_hosts: - host.docker.internal:host-gateway @@ -403,12 +403,12 @@ services: - GO_VERSION=1.20 - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest - - PERCONA_TEST_HA_ENABLE=1 - - PERCONA_TEST_HA_NODE_ID=pmm-server-passive-2 - - PERCONA_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11 - - PERCONA_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 - - PERCONA_TEST_HA_GOSSIP_PORT=9096 - - PERCONA_TEST_GRAFANA_GOSSIP_PORT=9094 + - PMM_TEST_HA_ENABLE=1 + - PMM_TEST_HA_NODE_ID=pmm-server-passive-2 + - PMM_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11 + - PMM_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2 + - PMM_TEST_HA_GOSSIP_PORT=9096 + - PMM_TEST_HA_GRAFANA_GOSSIP_PORT=9094 extra_hosts: - host.docker.internal:host-gateway diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 17e40b2860..845fa8789e 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -732,30 +732,30 @@ func main() { //nolint:cyclop,maintidx String() haEnabled := kingpin.Flag("ha-enable", "Enable HA"). - Envar("PERCONA_TEST_HA_ENABLE"). + Envar("PMM_TEST_HA_ENABLE"). Bool() haBootstrap := kingpin.Flag("ha-bootstrap", "Bootstrap HA cluster"). - Envar("PERCONA_TEST_HA_BOOTSTRAP"). + Envar("PMM_TEST_HA_BOOTSTRAP"). Bool() haNodeID := kingpin.Flag("ha-node-id", "HA Node ID"). - Envar("PERCONA_TEST_HA_NODE_ID"). + Envar("PMM_TEST_HA_NODE_ID"). String() haAdvertiseAddress := kingpin.Flag("ha-advertise-address", "HA Advertise address"). - Envar("PERCONA_TEST_HA_ADVERTISE_ADDRESS"). + Envar("PMM_TEST_HA_ADVERTISE_ADDRESS"). String() haPeers := kingpin.Flag("ha-peers", "HA Peers"). - Envar("PERCONA_TEST_HA_PEERS"). + Envar("PMM_TEST_HA_PEERS"). String() haRaftPort := kingpin.Flag("ha-raft-port", "HA raft port"). - Envar("PERCONA_TEST_HA_RAFT_PORT"). + Envar("PMM_TEST_HA_RAFT_PORT"). Default("9760"). Int() haGossipPort := kingpin.Flag("ha-gossip-port", "HA gossip port"). - Envar("PERCONA_TEST_HA_GOSSIP_PORT"). + Envar("PMM_TEST_HA_GOSSIP_PORT"). Default("9761"). Int() - grafanaGossipPort := kingpin.Flag("grafana-gossip-port", "Grafana gossip port"). - Envar("PERCONA_TEST_GRAFANA_GOSSIP_PORT"). + haGrafanaGossipPort := kingpin.Flag("ha-grafana-gossip-port", "HA Grafana gossip port"). + Envar("PMM_TEST_HA_GRAFANA_GOSSIP_PORT"). Default("9762"). Int() @@ -800,7 +800,7 @@ func main() { //nolint:cyclop,maintidx Nodes: nodes, RaftPort: *haRaftPort, GossipPort: *haGossipPort, - GrafanaGossipPort: *grafanaGossipPort, + GrafanaGossipPort: *haGrafanaGossipPort, } haService := ha.New(haParams) diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 2476b13d68..96f6f4f24e 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -80,20 +80,28 @@ name: postgres when: is_postgres_11.stat.exists -- name: Create grafana database in postgres - postgresql_db: - name: grafana - state: present - -- name: Create grafana user in postgres - postgresql_user: - db: grafana - name: grafana - password: grafana - priv: 'ALL' - expires: infinity - state: present - when: not ansible_check_mode +- name: Create grafana DB + block: + - name: Create grafana database in postgres + postgresql_db: + name: grafana + state: present + + - name: Create grafana user in postgres + postgresql_user: + db: grafana + name: grafana + password: grafana + priv: 'ALL' + expires: infinity + state: present + when: not ansible_check_mode + when: + +- name: Upgrade grafana database (Get the latest schema) + command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords + changed_when: True + when: {{ lookup('env','PMM_TEST_HA_BOOTSTRAP') | default('USERVAL',False) }} - name: Create working directory for Alertmanager file: path=/srv/alertmanager/data state=directory owner=pmm group=pmm From 717f9a09694dfb9042f12a7ec1c646e8f81165a0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 19 Oct 2023 14:44:02 +0300 Subject: [PATCH 70/77] PMM-12078 Some improvements regarding env variables. --- .../ansible/playbook/tasks/roles/initialization/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 96f6f4f24e..c755aeba8f 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -101,7 +101,7 @@ - name: Upgrade grafana database (Get the latest schema) command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords changed_when: True - when: {{ lookup('env','PMM_TEST_HA_BOOTSTRAP') | default('USERVAL',False) }} + when: "{{ lookup('env','PMM_TEST_HA_BOOTSTRAP') | default('USERVAL',False) }}" - name: Create working directory for Alertmanager file: path=/srv/alertmanager/data state=directory owner=pmm group=pmm From 2a3df44a2ff415e6853e0a802454b804ceb01292 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 19 Oct 2023 15:45:50 +0300 Subject: [PATCH 71/77] PMM-12078 Some improvements regarding env variables. --- .../ansible/playbook/tasks/roles/initialization/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index c755aeba8f..cacc921876 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -101,7 +101,7 @@ - name: Upgrade grafana database (Get the latest schema) command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords changed_when: True - when: "{{ lookup('env','PMM_TEST_HA_BOOTSTRAP') | default('USERVAL',False) }}" + when: lookup('env','PMM_TEST_HA_BOOTSTRAP') != '' - name: Create working directory for Alertmanager file: path=/srv/alertmanager/data state=directory owner=pmm group=pmm From 225f228454ce9fee0e5b016d559b3c31358419a3 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 19 Oct 2023 21:46:54 +0300 Subject: [PATCH 72/77] PMM-12078 Don't create grafana DB for external DB. --- docker-compose.yml | 10 +++++++++- .../playbook/tasks/roles/initialization/tasks/main.yml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 543cfa923d..72cfbcf0e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,8 @@ services: networks: ha: ipv4_address: 172.20.0.7 + volumes: + - chdata:/var/lib/clickhouse # Volume for ClickHouse data victoriametrics: profiles: @@ -168,6 +170,7 @@ services: ha: ipv4_address: 172.20.0.3 volumes: + - pgdata:/var/lib/postgresql/data # Volume for PostgreSQL data - ./managed/testdata/pg/conf/:/conf/ - ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/ @@ -274,6 +277,7 @@ services: - go-modules:/root/go/pkg/mod - root-cache:/root/.cache - ./managed/testdata/pg/certs/:/certs/ + - ./update/ansible:/usr/share/pmm-update/ansible pmm-server-passive: profiles: @@ -357,6 +361,7 @@ services: - go-modules:/root/go/pkg/mod - root-cache:/root/.cache - ./managed/testdata/pg/certs/:/certs/ + - ./update/ansible:/usr/share/pmm-update/ansible pmm-server-passive-2: profiles: @@ -440,10 +445,13 @@ services: - go-modules:/root/go/pkg/mod - root-cache:/root/.cache - ./managed/testdata/pg/certs/:/certs/ + - ./update/ansible:/usr/share/pmm-update/ansible volumes: + chdata: # Volume for ClickHouse data + vmdata: # Volume for VictoriaMetrics data + pgdata: # Volume for PostgreSQL data go-modules: - vmdata: {} root-cache: networks: diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index cacc921876..73b1613bb9 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -96,7 +96,7 @@ expires: infinity state: present when: not ansible_check_mode - when: + when: lookup('env','GF_DATABASE_URL') == '' and lookup('env','GF_DATABASE_HOST') == '' - name: Upgrade grafana database (Get the latest schema) command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords From 37944388af2b208c206990e4f1a172321c115d1b Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Fri, 20 Oct 2023 13:31:46 +0300 Subject: [PATCH 73/77] PMM-12078 Update db versions. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 72cfbcf0e1..e97c626c6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,7 +104,7 @@ services: ch: profiles: - pmm-ha - image: ${CH_IMAGE:-clickhouse/clickhouse-server:22.6.9.11-alpine} + image: ${CH_IMAGE:-clickhouse/clickhouse-server:23.8.2.7-alpine} platform: linux/amd64 hostname: ${CH_HOSTNAME:-ch} ports: @@ -119,7 +119,7 @@ services: profiles: - pmm-ha hostname: ${VM_HOSTNAME:-victoriametrics} - image: victoriametrics/victoria-metrics:v1.88.1 + image: victoriametrics/victoria-metrics:v1.93.4 ports: - 8428:8428 - 8089:8089 From 007033b1a95e0775debbd35c020b045d40a7239f Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 31 Oct 2023 15:30:15 +0300 Subject: [PATCH 74/77] PMM-12078 fix go mod. --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index f421d06fcd..7600814ca6 100644 --- a/go.mod +++ b/go.mod @@ -122,7 +122,6 @@ require ( github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect From 28289454b991c341eb9b6a8cd51863d9ecd80ded Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 27 Nov 2023 18:59:09 +0300 Subject: [PATCH 75/77] PMM-12078 fix linter --- .../json/client/server/leader_health_check_responses.go | 9 +++++---- api/serverpb/json/serverpb.json | 4 ++-- api/swagger/swagger-dev.json | 4 ++-- api/swagger/swagger.json | 4 ++-- go.mod | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/api/serverpb/json/client/server/leader_health_check_responses.go b/api/serverpb/json/client/server/leader_health_check_responses.go index bdbe4dbab2..25152bbebd 100644 --- a/api/serverpb/json/client/server/leader_health_check_responses.go +++ b/api/serverpb/json/client/server/leader_health_check_responses.go @@ -250,7 +250,7 @@ LeaderHealthCheckDefaultBodyDetailsItems0 `Any` contains an arbitrary serialized // foo = any.unpack(Foo.getDefaultInstance()); // } // -// Example 3: Pack and unpack a message in Python. +// Example 3: Pack and unpack a message in Python. // // foo = Foo(...) // any = Any() @@ -260,7 +260,7 @@ LeaderHealthCheckDefaultBodyDetailsItems0 `Any` contains an arbitrary serialized // any.Unpack(foo) // ... // -// Example 4: Pack and unpack a message in Go +// Example 4: Pack and unpack a message in Go // // foo := &pb.Foo{...} // any, err := anypb.New(foo) @@ -280,7 +280,7 @@ LeaderHealthCheckDefaultBodyDetailsItems0 `Any` contains an arbitrary serialized // name "y.z". // // JSON -// +// ==== // The JSON representation of an `Any` value uses the regular // representation of the deserialized, embedded message, with an // additional field `@type` which contains the type URL. Example: @@ -332,7 +332,8 @@ type LeaderHealthCheckDefaultBodyDetailsItems0 struct { // // Note: this functionality is not currently available in the official // protobuf release, and it is not used for type URLs beginning with - // type.googleapis.com. + // type.googleapis.com. As of May 2023, there are no widely used type server + // implementations and no plans to implement one. // // Schemes other than `http`, `https` (or the empty scheme) might be // used with implementation specific semantics. diff --git a/api/serverpb/json/serverpb.json b/api/serverpb/json/serverpb.json index 93d235d5e6..932f000ca7 100644 --- a/api/serverpb/json/serverpb.json +++ b/api/serverpb/json/serverpb.json @@ -1361,11 +1361,11 @@ "details": { "type": "array", "items": { - "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\nExample 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\nExample 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", + "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\n Example 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\n Example 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n====\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", "type": "object", "properties": { "@type": { - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", "type": "string", "x-order": 0 } diff --git a/api/swagger/swagger-dev.json b/api/swagger/swagger-dev.json index eca0c4aae7..20fda6ab9d 100644 --- a/api/swagger/swagger-dev.json +++ b/api/swagger/swagger-dev.json @@ -17867,11 +17867,11 @@ "details": { "type": "array", "items": { - "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\nExample 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\nExample 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", + "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\n Example 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\n Example 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n====\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", "type": "object", "properties": { "@type": { - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", "type": "string", "x-order": 0 } diff --git a/api/swagger/swagger.json b/api/swagger/swagger.json index cc457d2797..9baf6b7a01 100644 --- a/api/swagger/swagger.json +++ b/api/swagger/swagger.json @@ -15023,11 +15023,11 @@ "details": { "type": "array", "items": { - "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\nExample 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\nExample 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", + "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n Foo foo = ...;\n Any any;\n any.PackFrom(foo);\n ...\n if (any.UnpackTo(\u0026foo)) {\n ...\n }\n\nExample 2: Pack and unpack a message in Java.\n\n Foo foo = ...;\n Any any = Any.pack(foo);\n ...\n if (any.is(Foo.class)) {\n foo = any.unpack(Foo.class);\n }\n // or ...\n if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n foo = any.unpack(Foo.getDefaultInstance());\n }\n\n Example 3: Pack and unpack a message in Python.\n\n foo = Foo(...)\n any = Any()\n any.Pack(foo)\n ...\n if any.Is(Foo.DESCRIPTOR):\n any.Unpack(foo)\n ...\n\n Example 4: Pack and unpack a message in Go\n\n foo := \u0026pb.Foo{...}\n any, err := anypb.New(foo)\n if err != nil {\n ...\n }\n ...\n foo := \u0026pb.Foo{}\n if err := any.UnmarshalTo(foo); err != nil {\n ...\n }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n====\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n package google.profile;\n message Person {\n string first_name = 1;\n string last_name = 2;\n }\n\n {\n \"@type\": \"type.googleapis.com/google.profile.Person\",\n \"firstName\": \u003cstring\u003e,\n \"lastName\": \u003cstring\u003e\n }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n {\n \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n \"value\": \"1.212s\"\n }", "type": "object", "properties": { "@type": { - "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", + "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n URL, or have them precompiled into a binary to avoid any\n lookup. Therefore, binary compatibility needs to be preserved\n on changes to types. (Use versioned type names to manage\n breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics.", "type": "string", "x-order": 0 } diff --git a/go.mod b/go.mod index 84a012bf2e..7c0add89df 100644 --- a/go.mod +++ b/go.mod @@ -123,8 +123,8 @@ require ( github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/minio-go v6.0.14+incompatible // indirect From 9363230250a78a2b7b600babe4987422090cbff0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 27 Nov 2023 19:41:49 +0300 Subject: [PATCH 76/77] PMM-12078 fix linter --- .../ansible/playbook/tasks/roles/initialization/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml index ce0ca54321..5113096993 100644 --- a/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/update/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -128,7 +128,7 @@ - name: Upgrade grafana database (Get the latest schema) command: grafana cli --homepath=/usr/share/grafana admin data-migration encrypt-datasource-passwords changed_when: True - when: lookup('env','PMM_TEST_HA_BOOTSTRAP') != '' + when: lookup('env','PMM_TEST_HA_BOOTSTRAP') != '' and not pmm_current_version is version(pmm_image_version, '>=') - name: Create working directory for Alertmanager file: path=/srv/alertmanager/data state=directory owner=pmm group=pmm From 82c8fb4928a37ffa4000dd73dc4ab4112b38641a Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 27 Nov 2023 19:51:09 +0300 Subject: [PATCH 77/77] PMM-12078 fix linter --- update/.ansible-lint | 1 + 1 file changed, 1 insertion(+) diff --git a/update/.ansible-lint b/update/.ansible-lint index 29c64847a3..0e1628d74b 100644 --- a/update/.ansible-lint +++ b/update/.ansible-lint @@ -11,3 +11,4 @@ warn_list: # don't move to 'skip_list' - it will silence them completely - no-changed-when # Commands should not change things if nothing needs doing - parser-error # AnsibleParserError - ignore-errors # Use failed_when and specify error conditions instead of using ignore_errors + - empty-string-compare # Don't compare to empty string