From 59d57d51db8bc867a569a88ad9fc9fe5cabce301 Mon Sep 17 00:00:00 2001 From: husharp Date: Wed, 31 Jul 2024 11:40:54 +0800 Subject: [PATCH] add pdms name Signed-off-by: husharp --- components/playground/instance/pd.go | 16 +++- embed/templates/scripts/run_scheduling.sh.tpl | 3 + embed/templates/scripts/run_tso.sh.tpl | 3 + pkg/cluster/manager/manager_test.go | 15 ++++ pkg/cluster/manager/transfer_test.go | 28 +++++++ pkg/cluster/spec/scheduling.go | 38 +++++---- pkg/cluster/spec/spec.go | 10 +++ pkg/cluster/spec/tso.go | 38 +++++---- pkg/cluster/spec/validate.go | 34 ++++++++ pkg/cluster/template/scripts/pdms_test.go | 80 +++++++++++++++++++ pkg/cluster/template/scripts/scheduling.go | 1 + pkg/cluster/template/scripts/tso.go | 1 + pkg/tidbver/tidbver.go | 5 ++ 13 files changed, 239 insertions(+), 33 deletions(-) create mode 100644 pkg/cluster/template/scripts/pdms_test.go diff --git a/components/playground/instance/pd.go b/components/playground/instance/pd.go index 3c1d6f8636..67b2e53ed2 100644 --- a/components/playground/instance/pd.go +++ b/components/playground/instance/pd.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) @@ -83,7 +84,14 @@ func (inst *PDInstance) InitCluster(pds []*PDInstance) *PDInstance { // Name return the name of pd. func (inst *PDInstance) Name() string { - return fmt.Sprintf("pd-%d", inst.ID) + switch inst.Role { + case PDRoleTSO: + return fmt.Sprintf("tso-%d", inst.ID) + case PDRoleScheduling: + return fmt.Sprintf("scheduling-%d", inst.ID) + default: + return fmt.Sprintf("pd-%d", inst.ID) + } } // Start calls set inst.cmd and Start @@ -142,6 +150,9 @@ func (inst *PDInstance) Start(ctx context.Context) error { fmt.Sprintf("--log-file=%s", inst.LogFile()), fmt.Sprintf("--config=%s", configPath), } + if tidbver.PDSupportMicroServicesWithName(inst.Version.String()) { + args = append(args, fmt.Sprintf("--name=%s", uid)) + } case PDRoleScheduling: endpoints := pdEndpoints(inst.pds, true) args = []string{ @@ -153,6 +164,9 @@ func (inst *PDInstance) Start(ctx context.Context) error { fmt.Sprintf("--log-file=%s", inst.LogFile()), fmt.Sprintf("--config=%s", configPath), } + if tidbver.PDSupportMicroServicesWithName(inst.Version.String()) { + args = append(args, fmt.Sprintf("--name=%s", uid)) + } } inst.Process = &process{cmd: PrepareCommand(ctx, inst.BinPath, args, nil, inst.Dir)} diff --git a/embed/templates/scripts/run_scheduling.sh.tpl b/embed/templates/scripts/run_scheduling.sh.tpl index a15b1ba4f7..2ba72fed0f 100644 --- a/embed/templates/scripts/run_scheduling.sh.tpl +++ b/embed/templates/scripts/run_scheduling.sh.tpl @@ -11,6 +11,9 @@ cd "${DEPLOY_DIR}" || exit 1 exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services scheduling\ {{- else}} exec env GODEBUG=madvdontneed=1 bin/pd-server services scheduling \ +{{- end}} +{{- if .Name}} + --name="{{.Name}}" \ {{- end}} --backend-endpoints="{{.BackendEndpoints}}" \ --listen-addr="{{.ListenURL}}" \ diff --git a/embed/templates/scripts/run_tso.sh.tpl b/embed/templates/scripts/run_tso.sh.tpl index 0d6486d73e..177b676aff 100644 --- a/embed/templates/scripts/run_tso.sh.tpl +++ b/embed/templates/scripts/run_tso.sh.tpl @@ -11,6 +11,9 @@ cd "${DEPLOY_DIR}" || exit 1 exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services tso\ {{- else}} exec env GODEBUG=madvdontneed=1 bin/pd-server services tso \ +{{- end}} +{{- if .Name}} + --name="{{.Name}}" \ {{- end}} --backend-endpoints="{{.BackendEndpoints}}" \ --listen-addr="{{.ListenURL}}" \ diff --git a/pkg/cluster/manager/manager_test.go b/pkg/cluster/manager/manager_test.go index e80d1d1c64..b82b6eb785 100644 --- a/pkg/cluster/manager/manager_test.go +++ b/pkg/cluster/manager/manager_test.go @@ -85,6 +85,21 @@ pd_servers: assert.Nil(err) err = validateNewTopo(&topo) assert.NotNil(err) + + topo = spec.Specification{} + err = yaml.Unmarshal([]byte(` +global: + user: "test4" + deploy_dir: "test-deploy" + data_dir: "test-data" +tso_servers: + - host: 172.16.5.53 +scheduling_servers: + - host: 172.16.5.54 +`), &topo) + assert.Nil(err) + err = validateNewTopo(&topo) + assert.Nil(err) } func TestDeduplicateCheckResult(t *testing.T) { diff --git a/pkg/cluster/manager/transfer_test.go b/pkg/cluster/manager/transfer_test.go index bef179dd0b..d4b4581f6a 100644 --- a/pkg/cluster/manager/transfer_test.go +++ b/pkg/cluster/manager/transfer_test.go @@ -53,4 +53,32 @@ func TestRenderSpec(t *testing.T) { dir, err = renderSpec("{{.DataDir}}", s, "test-pd") assert.Nil(t, err) assert.NotEmpty(t, dir) + + s = &spec.TSOInstance{BaseInstance: spec.BaseInstance{ + InstanceSpec: &spec.TSOSpec{ + Host: "172.16.5.140", + SSHPort: 22, + Name: "tso-1", + DeployDir: "/home/test/deploy/tso-3379", + DataDir: "/home/test/deploy/tso-3379/data", + }, + }} + // s.BaseInstance.InstanceSpec + dir, err = renderSpec("{{.DataDir}}", s, "test-tso") + assert.Nil(t, err) + assert.NotEmpty(t, dir) + + s = &spec.SchedulingInstance{BaseInstance: spec.BaseInstance{ + InstanceSpec: &spec.SchedulingSpec{ + Host: "172.16.5.140", + SSHPort: 22, + Name: "scheduling-1", + DeployDir: "/home/test/deploy/scheduling-3379", + DataDir: "/home/test/deploy/scheduling-3379/data", + }, + }} + // s.BaseInstance.InstanceSpec + dir, err = renderSpec("{{.DataDir}}", s, "test-scheduling") + assert.Nil(t, err) + assert.NotEmpty(t, dir) } diff --git a/pkg/cluster/spec/scheduling.go b/pkg/cluster/spec/scheduling.go index efbe7def00..3da3975f2d 100644 --- a/pkg/cluster/spec/scheduling.go +++ b/pkg/cluster/spec/scheduling.go @@ -25,26 +25,29 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) // SchedulingSpec represents the scheduling topology specification in topology.yaml type SchedulingSpec struct { - Host string `yaml:"host"` - ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` - ListenHost string `yaml:"listen_host,omitempty"` - AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` - SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` - IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` - Port int `yaml:"port" default:"3379"` - DeployDir string `yaml:"deploy_dir,omitempty"` - DataDir string `yaml:"data_dir,omitempty"` - LogDir string `yaml:"log_dir,omitempty"` - Source string `yaml:"source,omitempty" validate:"source:editable"` - NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` - Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` - Arch string `yaml:"arch,omitempty"` - OS string `yaml:"os,omitempty"` + Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` + ListenHost string `yaml:"listen_host,omitempty"` + AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` + SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` + IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` + // Use Name to get the name with a default value if it's empty. + Name string `yaml:"name,omitempty"` + Port int `yaml:"port" default:"3379"` + DeployDir string `yaml:"deploy_dir,omitempty"` + DataDir string `yaml:"data_dir,omitempty"` + LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` + NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` + Arch string `yaml:"arch,omitempty"` + OS string `yaml:"os,omitempty"` } // Status queries current status of the instance @@ -200,7 +203,6 @@ func (c *SchedulingComponent) Instances() []Instance { // SchedulingInstance represent the scheduling instance type SchedulingInstance struct { - Name string BaseInstance topo Topology } @@ -229,6 +231,7 @@ func (i *SchedulingInstance) InitConfig( pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS)) } cfg := &scripts.SchedulingScript{ + Name: spec.Name, ListenURL: fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(i.GetListenHost(), spec.Port)), AdvertiseListenURL: spec.GetAdvertiseListenURL(enableTLS), BackendEndpoints: strings.Join(pds, ","), @@ -237,6 +240,9 @@ func (i *SchedulingInstance) InitConfig( LogDir: paths.Log, NumaNode: spec.NumaNode, } + if !tidbver.PDSupportMicroServicesWithName(version) { + cfg.Name = "" + } fp := filepath.Join(paths.Cache, fmt.Sprintf("run_scheduling_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index ffef492dda..99bd0e87dd 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -679,10 +679,20 @@ func setCustomDefaults(globalOptions *GlobalOptions, field reflect.Value) error } field.Field(j).Set(reflect.ValueOf(globalOptions.SSHPort)) case "Name": + // Only PD related components have `Name` field, if field.Field(j).String() != "" { continue } host := reflect.Indirect(field).FieldByName("Host").String() + // `TSO` and `Scheduling` components use `Port` filed + if reflect.Indirect(field).FieldByName("Port").IsValid() { + port := reflect.Indirect(field).FieldByName("Port").Int() + // field.String() is + role := strings.Split(strings.Split(field.Type().String(), ".")[1], "Spec")[0] + component := strings.ToLower(role) + field.Field(j).Set(reflect.ValueOf(fmt.Sprintf("%s-%s-%d", component, host, port))) + continue + } clientPort := reflect.Indirect(field).FieldByName("ClientPort").Int() field.Field(j).Set(reflect.ValueOf(fmt.Sprintf("pd-%s-%d", host, clientPort))) case "DataDir": diff --git a/pkg/cluster/spec/tso.go b/pkg/cluster/spec/tso.go index a84069f6ee..e309f1d652 100644 --- a/pkg/cluster/spec/tso.go +++ b/pkg/cluster/spec/tso.go @@ -25,26 +25,29 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) // TSOSpec represents the TSO topology specification in topology.yaml type TSOSpec struct { - Host string `yaml:"host"` - ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` - ListenHost string `yaml:"listen_host,omitempty"` - AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` - SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` - IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` - Port int `yaml:"port" default:"3379"` - DeployDir string `yaml:"deploy_dir,omitempty"` - DataDir string `yaml:"data_dir,omitempty"` - LogDir string `yaml:"log_dir,omitempty"` - Source string `yaml:"source,omitempty" validate:"source:editable"` - NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` - Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` - Arch string `yaml:"arch,omitempty"` - OS string `yaml:"os,omitempty"` + Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` + ListenHost string `yaml:"listen_host,omitempty"` + AdvertiseListenAddr string `yaml:"advertise_listen_addr,omitempty"` + SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` + IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` + // Use Name to get the name with a default value if it's empty. + Name string `yaml:"name,omitempty"` + Port int `yaml:"port" default:"3379"` + DeployDir string `yaml:"deploy_dir,omitempty"` + DataDir string `yaml:"data_dir,omitempty"` + LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` + NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` + Arch string `yaml:"arch,omitempty"` + OS string `yaml:"os,omitempty"` } // Status queries current status of the instance @@ -200,7 +203,6 @@ func (c *TSOComponent) Instances() []Instance { // TSOInstance represent the TSO instance type TSOInstance struct { - Name string BaseInstance topo Topology } @@ -229,6 +231,7 @@ func (i *TSOInstance) InitConfig( pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS)) } cfg := &scripts.TSOScript{ + Name: spec.Name, ListenURL: fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(i.GetListenHost(), spec.Port)), AdvertiseListenURL: spec.GetAdvertiseListenURL(enableTLS), BackendEndpoints: strings.Join(pds, ","), @@ -237,6 +240,9 @@ func (i *TSOInstance) InitConfig( LogDir: paths.Log, NumaNode: spec.NumaNode, } + if !tidbver.PDSupportMicroServicesWithName(version) { + cfg.Name = "" + } fp := filepath.Join(paths.Cache, fmt.Sprintf("run_tso_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { diff --git a/pkg/cluster/spec/validate.go b/pkg/cluster/spec/validate.go index 99384cbec7..ce73682c01 100644 --- a/pkg/cluster/spec/validate.go +++ b/pkg/cluster/spec/validate.go @@ -984,6 +984,38 @@ func (s *Specification) validatePDNames() error { return nil } +func (s *Specification) validateTSONames() error { + // check tso server name + tsoNames := set.NewStringSet() + for _, tso := range s.TSOServers { + if tso.Name == "" { + continue + } + + if tsoNames.Exist(tso.Name) { + return errors.Errorf("component tso_servers.name is not supported duplicated, the name %s is duplicated", tso.Name) + } + tsoNames.Insert(tso.Name) + } + return nil +} + +func (s *Specification) validateSchedulingNames() error { + // check scheduling server name + schedulingNames := set.NewStringSet() + for _, scheduling := range s.SchedulingServers { + if scheduling.Name == "" { + continue + } + + if schedulingNames.Exist(scheduling.Name) { + return errors.Errorf("component scheduling_servers.name is not supported duplicated, the name %s is duplicated", scheduling.Name) + } + schedulingNames.Insert(scheduling.Name) + } + return nil +} + func (s *Specification) validateTiFlashConfigs() error { c := FindComponent(s, ComponentTiFlash) for _, ins := range c.Instances() { @@ -1063,6 +1095,8 @@ func (s *Specification) Validate() error { s.dirConflictsDetect, s.validateUserGroup, s.validatePDNames, + s.validateTSONames, + s.validateSchedulingNames, s.validateTiSparkSpec, s.validateTiFlashConfigs, s.validateMonitorAgent, diff --git a/pkg/cluster/template/scripts/pdms_test.go b/pkg/cluster/template/scripts/pdms_test.go new file mode 100644 index 0000000000..414f693f1f --- /dev/null +++ b/pkg/cluster/template/scripts/pdms_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scripts + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestScheduling(t *testing.T) { + assert := require.New(t) + conf, err := os.CreateTemp("", "scheduling.conf") + assert.Nil(err) + defer os.Remove(conf.Name()) + + cfg := &SchedulingScript{ + Name: "scheduling-0", + ListenURL: "127.0.0.1", + AdvertiseListenURL: "127.0.0.2", + BackendEndpoints: "127.0.0.3", + DeployDir: "/deploy", + DataDir: "/data", + LogDir: "/log", + } + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err := os.ReadFile(conf.Name()) + assert.Nil(err) + assert.True(strings.Contains(string(content), "--name")) + + cfg.Name = "" + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err = os.ReadFile(conf.Name()) + assert.Nil(err) + assert.False(strings.Contains(string(content), "--name")) +} + +func TestTSO(t *testing.T) { + assert := require.New(t) + conf, err := os.CreateTemp("", "tso.conf") + assert.Nil(err) + defer os.Remove(conf.Name()) + + cfg := &TSOScript{ + Name: "tso-0", + ListenURL: "127.0.0.1", + AdvertiseListenURL: "127.0.0.2", + BackendEndpoints: "127.0.0.3", + DeployDir: "/deploy", + DataDir: "/data", + LogDir: "/log", + } + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err := os.ReadFile(conf.Name()) + assert.Nil(err) + assert.True(strings.Contains(string(content), "--name")) + + cfg.Name = "" + err = cfg.ConfigToFile(conf.Name()) + assert.Nil(err) + content, err = os.ReadFile(conf.Name()) + assert.Nil(err) + assert.False(strings.Contains(string(content), "--name")) +} diff --git a/pkg/cluster/template/scripts/scheduling.go b/pkg/cluster/template/scripts/scheduling.go index 6167d9336e..76142485a2 100644 --- a/pkg/cluster/template/scripts/scheduling.go +++ b/pkg/cluster/template/scripts/scheduling.go @@ -24,6 +24,7 @@ import ( // SchedulingScript represent the data to generate scheduling config type SchedulingScript struct { + Name string ListenURL string AdvertiseListenURL string BackendEndpoints string diff --git a/pkg/cluster/template/scripts/tso.go b/pkg/cluster/template/scripts/tso.go index 0197b82c38..91c3bfe1d0 100644 --- a/pkg/cluster/template/scripts/tso.go +++ b/pkg/cluster/template/scripts/tso.go @@ -24,6 +24,7 @@ import ( // TSOScript represent the data to generate tso config type TSOScript struct { + Name string ListenURL string AdvertiseListenURL string BackendEndpoints string diff --git a/pkg/tidbver/tidbver.go b/pkg/tidbver/tidbver.go index c659304a85..c811510cfc 100644 --- a/pkg/tidbver/tidbver.go +++ b/pkg/tidbver/tidbver.go @@ -104,6 +104,11 @@ func PDSupportMicroServices(version string) bool { return semver.Compare(version, "v7.3.0") >= 0 || strings.Contains(version, "nightly") } +// PDSupportMicroServicesWithName return if the given version of PD supports micro services with name. +func PDSupportMicroServicesWithName(version string) bool { + return semver.Compare(version, "v8.3.0") >= 0 || strings.Contains(version, "nightly") +} + // TiCDCSupportConfigFile return if given version of TiCDC support config file func TiCDCSupportConfigFile(version string) bool { // config support since v4.0.13, ignore v5.0.0-rc