Skip to content

Commit

Permalink
remove resource manager
Browse files Browse the repository at this point in the history 
Signed-off-by: Ryan Leung <[email protected]>
  • Loading branch information
@rleungx
rleungx committed Mar 12, 2024
1 parent 9e11d0e commit 7c18c80
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 83 deletions.
2 changes: 1 addition & 1 deletion components/playground/instance/instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func logIfErr(err error) {
func pdEndpoints(pds []*PDInstance, isHTTP bool) []string {
var endpoints []string
for _, pd := range pds {
if pd.Role == PDRoleTSO || pd.Role == PDRoleScheduling || pd.Role == PDRoleResourceManager {
if pd.Role == PDRoleTSO || pd.Role == PDRoleScheduling {
continue
}
if isHTTP {
Expand Down
22 changes: 5 additions & 17 deletions components/playground/instance/pd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ const (
PDRoleTSO PDRole = "tso"
// PDRoleScheduling is the role of PD scheduling
PDRoleScheduling PDRole = "scheduling"
// PDRoleResourceManager is the role of PD resource manager
PDRoleResourceManager PDRole = "resource_manager"
)

// PDInstance represent a running pd-server
Expand Down Expand Up @@ -159,19 +157,6 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
if inst.ConfigPath != "" {
args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
}
case PDRoleResourceManager:
endpoints := pdEndpoints(inst.pds, true)
args = []string{
"services",
"resource-manager",
fmt.Sprintf("--listen-addr=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)),
fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
fmt.Sprintf("--log-file=%s", inst.LogFile()),
}
if inst.ConfigPath != "" {
args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
}
}

var err error
Expand All @@ -186,14 +171,17 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error

// Component return the component name.
func (inst *PDInstance) Component() string {
if inst.Role == PDRoleNormal {
if inst.Role == PDRoleNormal || inst.Role == PDRoleAPI {
return "pd"
}
return fmt.Sprintf("pd %s", inst.Role)
return string(inst.Role)
}

// LogFile return the log file.
func (inst *PDInstance) LogFile() string {
if inst.Role == PDRoleNormal || inst.Role == PDRoleAPI {
return filepath.Join(inst.Dir, "pd.log")
}
return filepath.Join(inst.Dir, fmt.Sprintf("%s.log", string(inst.Role)))
}

Expand Down
50 changes: 18 additions & 32 deletions components/playground/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,9 @@ type BootOptions struct {
Mode string `yaml:"mode"`
PDMode string `yaml:"pd_mode"`
Version string `yaml:"version"`
PD instance.Config `yaml:"pd"` // ignored when pd_mode == ms
PDAPI instance.Config `yaml:"pd_api"` // Only available when pd_mode == ms
PDTSO instance.Config `yaml:"pd_tso"` // Only available when pd_mode == ms
PDScheduling instance.Config `yaml:"pd_scheduling"` // Only available when pd_mode == ms
PDRM instance.Config `yaml:"pd_rm"` // Only available when pd_mode == ms
PD instance.Config `yaml:"pd"` // will change to api when pd_mode == ms
TSO instance.Config `yaml:"tso"` // Only available when pd_mode == ms
Scheduling instance.Config `yaml:"scheduling"` // Only available when pd_mode == ms
TiProxy instance.Config `yaml:"tiproxy"`
TiDB instance.Config `yaml:"tidb"`
TiKV instance.Config `yaml:"tikv"`
Expand Down Expand Up @@ -289,6 +287,8 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
rootCmd.Flags().IntVar(&options.TiDB.Num, "db", 0, "TiDB instance number")
rootCmd.Flags().IntVar(&options.TiKV.Num, "kv", 0, "TiKV instance number")
rootCmd.Flags().IntVar(&options.PD.Num, "pd", 0, "PD instance number")
rootCmd.Flags().IntVar(&options.TSO.Num, "tso", 0, "TSO instance number")
rootCmd.Flags().IntVar(&options.Scheduling.Num, "scheduling", 0, "Scheduling instance number")
rootCmd.Flags().IntVar(&options.TiProxy.Num, "tiproxy", 0, "TiProxy instance number")
rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-disagg this will set instance number for both Write Node and Compute Node")
rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-disagg, take precedence over --tiflash")
Expand All @@ -298,11 +298,6 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
rootCmd.Flags().IntVar(&options.Pump.Num, "pump", 0, "Pump instance number")
rootCmd.Flags().IntVar(&options.Drainer.Num, "drainer", 0, "Drainer instance number")

rootCmd.Flags().IntVar(&options.PDAPI.Num, "pd.api", 0, "PD API instance number")
rootCmd.Flags().IntVar(&options.PDTSO.Num, "pd.tso", 0, "PD TSO instance number")
rootCmd.Flags().IntVar(&options.PDScheduling.Num, "pd.scheduling", 0, "PD scheduling instance number")
rootCmd.Flags().IntVar(&options.PDRM.Num, "pd.rm", 0, "PD resource manager instance number")

rootCmd.Flags().IntVar(&options.TiDB.UpTimeout, "db.timeout", 60, "TiDB max wait time in seconds for starting, 0 means no limit")
rootCmd.Flags().IntVar(&options.TiFlash.UpTimeout, "tiflash.timeout", 120, "TiFlash max wait time in seconds for starting, 0 means no limit")
rootCmd.Flags().IntVar(&options.TiProxy.UpTimeout, "tiproxy.timeout", 60, "TiProxy max wait time in seconds for starting, 0 means no limit")
Expand All @@ -322,6 +317,8 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
rootCmd.Flags().StringVar(&options.TiDB.ConfigPath, "db.config", "", "TiDB instance configuration file")
rootCmd.Flags().StringVar(&options.TiKV.ConfigPath, "kv.config", "", "TiKV instance configuration file")
rootCmd.Flags().StringVar(&options.PD.ConfigPath, "pd.config", "", "PD instance configuration file")
rootCmd.Flags().StringVar(&options.TSO.ConfigPath, "tso.config", "", "TSO instance configuration file")
rootCmd.Flags().StringVar(&options.Scheduling.ConfigPath, "scheduling.config", "", "Scheduling instance configuration file")
rootCmd.Flags().StringVar(&options.TiProxy.ConfigPath, "tiproxy.config", "", "TiProxy instance configuration file")
rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-disagg this will set config file for both Write Node and Compute Node")
rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-disagg, take precedence over --tiflash.config")
Expand All @@ -331,14 +328,11 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
rootCmd.Flags().StringVar(&options.TiCDC.ConfigPath, "ticdc.config", "", "TiCDC instance configuration file")
rootCmd.Flags().StringVar(&options.TiKVCDC.ConfigPath, "kvcdc.config", "", "TiKV-CDC instance configuration file")

rootCmd.Flags().StringVar(&options.PDAPI.ConfigPath, "pd.api.config", "", "PD API instance configuration file")
rootCmd.Flags().StringVar(&options.PDTSO.ConfigPath, "pd.tso.config", "", "PD TSO instance configuration file")
rootCmd.Flags().StringVar(&options.PDScheduling.ConfigPath, "pd.scheduling.config", "", "PD scheduling instance configuration file")
rootCmd.Flags().StringVar(&options.PDRM.ConfigPath, "pd.rm.config", "", "PD resource manager instance configuration file")

rootCmd.Flags().StringVar(&options.TiDB.BinPath, "db.binpath", "", "TiDB instance binary path")
rootCmd.Flags().StringVar(&options.TiKV.BinPath, "kv.binpath", "", "TiKV instance binary path")
rootCmd.Flags().StringVar(&options.PD.BinPath, "pd.binpath", "", "PD instance binary path")
rootCmd.Flags().StringVar(&options.TSO.BinPath, "tso.binpath", "", "TSO instance binary path")
rootCmd.Flags().StringVar(&options.Scheduling.BinPath, "scheduling.binpath", "", "Scheduling instance binary path")
rootCmd.Flags().StringVar(&options.TiProxy.BinPath, "tiproxy.binpath", "", "TiProxy instance binary path")
rootCmd.Flags().StringVar(&options.TiProxy.Version, "tiproxy.version", "", "TiProxy instance version")
rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-disagg this will set binary path for both Write Node and Compute Node")
Expand All @@ -349,11 +343,6 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset].
rootCmd.Flags().StringVar(&options.Pump.BinPath, "pump.binpath", "", "Pump instance binary path")
rootCmd.Flags().StringVar(&options.Drainer.BinPath, "drainer.binpath", "", "Drainer instance binary path")

rootCmd.Flags().StringVar(&options.PDAPI.BinPath, "pd.api.binpath", "", "PD API instance binary path")
rootCmd.Flags().StringVar(&options.PDTSO.BinPath, "pd.tso.binpath", "", "PD TSO instance binary path")
rootCmd.Flags().StringVar(&options.PDScheduling.BinPath, "pd.scheduling.binpath", "", "PD scheduling instance binary path")
rootCmd.Flags().StringVar(&options.PDRM.BinPath, "pd.rm.binpath", "", "PD resource manager instance binary path")

rootCmd.Flags().StringVar(&options.TiKVCDC.Version, "kvcdc.version", "", "TiKV-CDC instance version")

rootCmd.Flags().StringVar(&options.DisaggOpts.S3Endpoint, "disagg.s3_endpoint", "127.0.0.1:9000", "Object store URL for the disaggregated TiFlash, available when --mode=tidb-disagg")
Expand Down Expand Up @@ -413,18 +402,15 @@ func populateDefaultOpt(flagSet *pflag.FlagSet) error {
case "pd":
defaultInt(&options.PD.Num, "pd", 1)
case "ms":
defaultInt(&options.PDAPI.Num, "pd.api", 1)
defaultStr(&options.PDAPI.BinPath, "pd.api.binpath", options.PD.BinPath)
defaultStr(&options.PDAPI.ConfigPath, "pd.api.config", options.PD.ConfigPath)
defaultInt(&options.PDTSO.Num, "pd.tso", 1)
defaultStr(&options.PDTSO.BinPath, "pd.tso.binpath", options.PD.BinPath)
defaultStr(&options.PDTSO.ConfigPath, "pd.tso.config", options.PD.ConfigPath)
defaultInt(&options.PDScheduling.Num, "pd.scheduling", 1)
defaultStr(&options.PDScheduling.BinPath, "pd.scheduling.binpath", options.PD.BinPath)
defaultStr(&options.PDScheduling.ConfigPath, "pd.scheduling.config", options.PD.ConfigPath)
defaultInt(&options.PDRM.Num, "pd.rm", 1)
defaultStr(&options.PDRM.BinPath, "pd.rm.binpath", options.PD.BinPath)
defaultStr(&options.PDRM.ConfigPath, "pd.rm.config", options.PD.ConfigPath)
defaultInt(&options.PD.Num, "pd", 1)
defaultStr(&options.PD.BinPath, "pd.binpath", options.PD.BinPath)
defaultStr(&options.PD.ConfigPath, "pd.config", options.PD.ConfigPath)
defaultInt(&options.TSO.Num, "tso", 1)
defaultStr(&options.TSO.BinPath, "tso.binpath", options.PD.BinPath)
defaultStr(&options.TSO.ConfigPath, "tso.config", options.PD.ConfigPath)
defaultInt(&options.Scheduling.Num, "scheduling", 1)
defaultStr(&options.Scheduling.BinPath, "scheduling.binpath", options.PD.BinPath)
defaultStr(&options.Scheduling.ConfigPath, "scheduling.config", options.PD.ConfigPath)
default:
return errors.Errorf("Unknown --pd.mode %s", options.PDMode)
}
Expand Down
41 changes: 9 additions & 32 deletions components/playground/playground.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ type Playground struct {
pds []*instance.PDInstance
tsos []*instance.PDInstance
schedulings []*instance.PDInstance
rms []*instance.PDInstance
tikvs []*instance.TiKVInstance
tidbs []*instance.TiDBInstance
tiflashs []*instance.TiFlashInstance
Expand Down Expand Up @@ -446,7 +445,7 @@ func (p *Playground) startInstance(ctx context.Context, inst instance.Instance)
var err error
boundVersion := p.bindVersion(inst.Component(), p.bootOptions.Version)
component := inst.Component()
if strings.HasPrefix(component, "pd") {
if component == "tso" || component == "scheduling" {
component = string(instance.PDRoleNormal)
}
version, err = environment.GlobalEnv().V1Repository().ResolveComponentVersion(component, boundVersion)
Expand Down Expand Up @@ -617,12 +616,6 @@ func (p *Playground) WalkInstances(fn func(componentID string, ins instance.Inst
return err
}
}
for _, ins := range p.rms {
err := fn(spec.ComponentPD, ins)
if err != nil {
return err
}
}
for _, ins := range p.tikvs {
err := fn(spec.ComponentTiKV, ins)
if err != nil {
Expand Down Expand Up @@ -705,9 +698,9 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif

id := p.allocID(componentID)
dir := filepath.Join(dataDir, fmt.Sprintf("%s-%d", componentID, id))
if componentID == string(instance.PDRoleNormal) && pdRole != instance.PDRoleNormal {
id = p.allocID(fmt.Sprintf("%s-%s", componentID, pdRole))
dir = filepath.Join(dataDir, fmt.Sprintf("%s-%s-%d", componentID, pdRole, id))
if componentID == string(instance.PDRoleNormal) && (pdRole != instance.PDRoleNormal && pdRole != instance.PDRoleAPI) {
id = p.allocID(string(pdRole))
dir = filepath.Join(dataDir, fmt.Sprintf("%s-%d", pdRole, id))
}
if err = utils.MkdirAll(dir, 0755); err != nil {
return nil, err
Expand Down Expand Up @@ -736,8 +729,6 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif
p.tsos = append(p.tsos, inst)
} else if pdRole == instance.PDRoleScheduling {
p.schedulings = append(p.schedulings, inst)
} else if pdRole == instance.PDRoleResourceManager {
p.rms = append(p.rms, inst)
}
case spec.ComponentTiDB:
inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.enableBinlog(), p.bootOptions.Mode == "tidb-disagg")
Expand Down Expand Up @@ -907,10 +898,8 @@ func (p *Playground) bindVersion(comp string, version string) (bindVersion strin
func (p *Playground) bootCluster(ctx context.Context, env *environment.Environment, options *BootOptions) error {
for _, cfg := range []*instance.Config{
&options.PD,
&options.PDAPI,
&options.PDTSO,
&options.PDScheduling,
&options.PDRM,
&options.TSO,
&options.Scheduling,
&options.TiProxy,
&options.TiDB,
&options.TiKV,
Expand Down Expand Up @@ -1017,10 +1006,9 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme
return fmt.Errorf("PD cluster doesn't support microservices mode in version %s", options.Version)
}
instances = append([]InstancePair{
{spec.ComponentPD, instance.PDRoleAPI, instance.TiFlashRoleNormal, options.PDAPI},
{spec.ComponentPD, instance.PDRoleTSO, instance.TiFlashRoleNormal, options.PDTSO},
{spec.ComponentPD, instance.PDRoleScheduling, instance.TiFlashRoleNormal, options.PDScheduling},
{spec.ComponentPD, instance.PDRoleResourceManager, instance.TiFlashRoleNormal, options.PDRM}},
{spec.ComponentPD, instance.PDRoleAPI, instance.TiFlashRoleNormal, options.PD},
{spec.ComponentPD, instance.PDRoleTSO, instance.TiFlashRoleNormal, options.TSO},
{spec.ComponentPD, instance.PDRoleScheduling, instance.TiFlashRoleNormal, options.Scheduling}},
instances...,
)
}
Expand Down Expand Up @@ -1127,7 +1115,6 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme
var (
tsoAddr []string
apiAddr []string
rmAddr []string
schedulingAddr []string
)
for _, api := range p.pds {
Expand All @@ -1139,18 +1126,13 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme
for _, scheduling := range p.schedulings {
schedulingAddr = append(schedulingAddr, scheduling.Addr())
}
for _, rm := range p.rms {
rmAddr = append(rmAddr, rm.Addr())
}

fmt.Printf("PD API Endpoints: ")
colorCmd.Printf("%s\n", strings.Join(apiAddr, ","))
fmt.Printf("PD TSO Endpoints: ")
colorCmd.Printf("%s\n", strings.Join(tsoAddr, ","))
fmt.Printf("PD Scheduling Endpoints: ")
colorCmd.Printf("%s\n", strings.Join(schedulingAddr, ","))
fmt.Printf("PD Resource Manager Endpoints: ")
colorCmd.Printf("%s\n", strings.Join(rmAddr, ","))
} else {
var pdAddrs []string
for _, pd := range p.pds {
Expand Down Expand Up @@ -1293,11 +1275,6 @@ func (p *Playground) terminate(sig syscall.Signal) {
kill(inst.Component(), inst.Pid(), inst.Wait)
}
}
for _, inst := range p.rms {
if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil {
kill(inst.Component(), inst.Pid(), inst.Wait)
}
}
for _, inst := range p.tiproxys {
if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil {
kill(inst.Component(), inst.Pid(), inst.Wait)
Expand Down
2 changes: 1 addition & 1 deletion pkg/cluster/executor/ssh.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func (e *EasySSHExecutor) Execute(ctx context.Context, cmd string, sudo bool, ti

// set a basic PATH in case it's empty on login
cmd = fmt.Sprintf("PATH=$PATH:/bin:/sbin:/usr/bin:/usr/sbin; %s", cmd)

if e.Locale != "" {
cmd = fmt.Sprintf("export LANG=%s; %s", e.Locale, cmd)
}
Expand Down

0 comments on commit 7c18c80

Please sign in to comment.